From 38b17fe361dadd9a98a7ca95bd5c6e39bfb3f2ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2020 04:45:38 +0000 Subject: [PATCH 001/259] chore(deps-dev): bump @testing-library/cypress from 6.0.1 to 7.0.1 Bumps [@testing-library/cypress](https://github.com/kentcdodds/cypress-testing-library) from 6.0.1 to 7.0.1. - [Release notes](https://github.com/kentcdodds/cypress-testing-library/releases) - [Changelog](https://github.com/testing-library/cypress-testing-library/blob/master/CHANGELOG.md) - [Commits](https://github.com/kentcdodds/cypress-testing-library/compare/v6.0.1...v7.0.1) Signed-off-by: dependabot[bot] --- packages/app/package.json | 2 +- yarn.lock | 21 ++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index e8dfb94046..a6875ba86f 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -46,7 +46,7 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@testing-library/cypress": "^6.0.0", + "@testing-library/cypress": "^7.0.1", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/yarn.lock b/yarn.lock index a3b64f0650..d17878a727 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4467,16 +4467,15 @@ dependencies: defer-to-connect "^2.0.0" -"@testing-library/cypress@^6.0.0": - version "6.0.1" - resolved "https://registry.npmjs.org/@testing-library/cypress/-/cypress-6.0.1.tgz#c924a69617db6a403c498abcb63759b79318fc76" - integrity sha512-hcPu2OnVuSTX1yDubEe7TBRD6mP2VgyopP1WTBIrJP79INPZdgOAX+TMNH68uZ/r5b4C7100IB4hjPIEQ+/Xog== +"@testing-library/cypress@^7.0.1": + version "7.0.1" + resolved "https://registry.npmjs.org/@testing-library/cypress/-/cypress-7.0.1.tgz#2843033acaefe96cb4cf789f16e98d957383e59d" + integrity sha512-LtggqG/7Hdc1EiKdmqXQwxWOO3ET1dkZtq0S8mIe8o+xaOtaVLrdCn0dE8Bi4Aj7z3w51w6wN9STdYymnUPlnQ== dependencies: "@babel/runtime" "^7.11.2" "@testing-library/dom" "^7.22.2" - "@types/testing-library__cypress" "^5.0.6" -"@testing-library/dom@^7.11.0", "@testing-library/dom@^7.17.1", "@testing-library/dom@^7.22.2": +"@testing-library/dom@^7.17.1", "@testing-library/dom@^7.22.2": version "7.23.0" resolved "https://registry.npmjs.org/@testing-library/dom/-/dom-7.23.0.tgz#c54c0fa53705ad867bcefb52fc0c96487fbc10f6" integrity sha512-H5m090auYH+obdZmsaYLrSWC5OauWD2CvNbz88KBxQJoXgkJzbU0DpAG8BS7Evj5WqCC3nAAKrLS6vw0ljUYLg== @@ -5600,14 +5599,6 @@ dependencies: "@types/estree" "*" -"@types/testing-library__cypress@^5.0.6": - version "5.0.6" - resolved "https://registry.npmjs.org/@types/testing-library__cypress/-/testing-library__cypress-5.0.6.tgz#9015f575c1a98f05996a4fe769071134ee488c26" - integrity sha512-TUp5wfanU7zUZigKqIeQDChnHQ1MEzbYqrI5iCQMFiesWNOASWm/el1lFBh1JPqmd6GkdDdDiHYJnkqd9le2ww== - dependencies: - "@testing-library/dom" "^7.11.0" - cypress "*" - "@types/testing-library__jest-dom@^5.9.1": version "5.9.1" resolved "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.9.1.tgz#aba5ee062b7880f69c212ef769389f30752806e5" @@ -9338,7 +9329,7 @@ cyclist@^1.0.1: resolved "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= -cypress@*, cypress@^4.2.0: +cypress@^4.2.0: version "4.12.1" resolved "https://registry.npmjs.org/cypress/-/cypress-4.12.1.tgz#0ead1b9f4c0917d69d8b57f996b6e01fe693b6ec" integrity sha512-9SGIPEmqU8vuRA6xst2CMTYd9sCFCxKSzrHt0wr+w2iAQMCIIsXsQ5Gplns1sT6LDbZcmLv6uehabAOl3fhc9Q== From 17e7efdc9f47e41723e858013862b529da9cb6de Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Thu, 8 Oct 2020 15:06:32 +0200 Subject: [PATCH 002/259] remove on close from component creation modal --- .../src/components/JobStatusModal/JobStatusModal.tsx | 10 ++-------- .../MultistepJsonForm/MultistepJsonForm.tsx | 12 ++++++++---- .../src/components/TemplatePage/TemplatePage.tsx | 2 -- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx index c531e0ec2c..149f7a47d7 100644 --- a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx +++ b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx @@ -30,18 +30,12 @@ import { entityRoute } from '@backstage/plugin-catalog'; import { generatePath } from 'react-router-dom'; type Props = { - onClose: () => void; onComplete: (job: Job) => void; jobId: string; entity: TemplateEntityV1alpha1 | null; }; -export const JobStatusModal = ({ - onClose, - jobId, - onComplete, - entity, -}: Props) => { +export const JobStatusModal = ({ jobId, onComplete, entity }: Props) => { const job = useJobPolling(jobId); const [dialogTitle, setDialogTitle] = useState('Creating component...'); @@ -54,7 +48,7 @@ export const JobStatusModal = ({ }, [job, onComplete, setDialogTitle]); return ( - + {dialogTitle} {!job ? ( diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx index f0ba720b39..9700d43c00 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx +++ b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx @@ -27,7 +27,7 @@ import { } from '@material-ui/core'; import { FormProps, IChangeEvent, withTheme } from '@rjsf/core'; import { Theme as MuiTheme } from '@rjsf/material-ui'; -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; const Form = withTheme(MuiTheme); type Step = { @@ -54,6 +54,7 @@ export const MultistepJsonForm = ({ onFinish, }: Props) => { const [activeStep, setActiveStep] = useState(0); + const [formDataEvent, setFormDataEvent] = useState({ formData: {} }); const handleReset = () => { setActiveStep(0); @@ -62,18 +63,21 @@ export const MultistepJsonForm = ({ const handleNext = () => setActiveStep(Math.min(activeStep + 1, steps.length)); const handleBack = () => setActiveStep(Math.max(activeStep - 1, 0)); - + useEffect(() => { + onChange(formDataEvent as IChangeEvent); + }, [formDataEvent, onChange]); return ( <> {steps.map(({ label, schema, ...formProps }) => ( {label} - +
setFormDataEvent(e)} schema={schema as FormProps['schema']} onSubmit={e => { if (e.errors.length === 0) handleNext(); diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index e68c795bd6..19196d39ad 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -93,7 +93,6 @@ export const TemplatePage = () => { setFormState({ ...formState, ...e.formData }); const [jobId, setJobId] = useState(null); - const handleClose = () => setJobId(null); const handleCreate = async () => { try { @@ -161,7 +160,6 @@ export const TemplatePage = () => { )} From 00b7b66bd8107cc006e0dbb9dbe086bd75764d9f Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sat, 7 Nov 2020 06:30:16 -0500 Subject: [PATCH 003/259] Fix typo in support button example (#3262) --- packages/core/src/layout/Page/Page.stories.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/layout/Page/Page.stories.tsx b/packages/core/src/layout/Page/Page.stories.tsx index 3b1dd07c7a..1d3aaea412 100644 --- a/packages/core/src/layout/Page/Page.stories.tsx +++ b/packages/core/src/layout/Page/Page.stories.tsx @@ -185,7 +185,7 @@ const ExampleContentHeader = ({ selectedTab }: { selectedTab?: number }) => ( title={selectedTab !== undefined ? tabs[selectedTab].label : 'Header'} > - This Plugin is an example. This text could provide usefull information for + This Plugin is an example. This text could provide useful information for the user. From a41730c6ead2ebf297b13be794af0240d986ad03 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sat, 7 Nov 2020 06:31:11 -0500 Subject: [PATCH 004/259] [plugin-jenkins] Add tooltip to Jenkins build rerun (#3251) * Add tooltop to Jenkins build rerun * Add tooltip to spelling --- .changeset/calm-emus-push.md | 5 +++++ .github/styles/vocab.txt | 1 + .../src/components/BuildsPage/lib/CITable/CITable.tsx | 10 ++++++---- 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 .changeset/calm-emus-push.md diff --git a/.changeset/calm-emus-push.md b/.changeset/calm-emus-push.md new file mode 100644 index 0000000000..b83f442c8a --- /dev/null +++ b/.changeset/calm-emus-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-jenkins': minor +--- + +Add tooltip for Jenkins rerun button diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index ecb8fdf3c1..3c5fa34adc 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -191,6 +191,7 @@ toc tolerations Tolerations toolsets +tooltip touchpoints ui upvote diff --git a/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx b/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx index 23f9af445c..5d5d7d9fcd 100644 --- a/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx +++ b/plugins/jenkins/src/components/BuildsPage/lib/CITable/CITable.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import React, { FC } from 'react'; -import { Box, IconButton, Link, Typography } from '@material-ui/core'; +import { Box, IconButton, Link, Typography, Tooltip } from '@material-ui/core'; import RetryIcon from '@material-ui/icons/Replay'; import GitHubIcon from '@material-ui/icons/GitHub'; import { generatePath, Link as RouterLink } from 'react-router-dom'; @@ -179,9 +179,11 @@ const generatedColumns: TableColumn[] = [ title: 'Actions', sorting: false, render: (row: Partial) => ( - - - + + + + + ), width: '10%', }, From 63834ee3383a8ab548c6fe566bb30bf2ddae1d52 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 6 Nov 2020 15:45:16 +0100 Subject: [PATCH 005/259] microsite/static: add confugration meta-schema based on JSONSchema draft-07 --- microsite/static/schema/config-v1 | 170 ++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 microsite/static/schema/config-v1 diff --git a/microsite/static/schema/config-v1 b/microsite/static/schema/config-v1 new file mode 100644 index 0000000000..f85f2844cc --- /dev/null +++ b/microsite/static/schema/config-v1 @@ -0,0 +1,170 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://backstage.io/schema/config-v1", + "title": "Backstage Configuration Meta-Schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [{ "$ref": "#" }, { "$ref": "#/definitions/schemaArray" }], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [{ "$ref": "#" }, { "$ref": "#/definitions/stringArray" }] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" }, + "visibility": { + "type": "string", + "enum": ["frontend", "backend", "secret"] + } + }, + "default": true +} From 8a39c5bd1c14951af5e3d3ed15108cbf16f660fd Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 9 Nov 2020 09:33:42 -0500 Subject: [PATCH 006/259] Big Query -> BigQuery --- app-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index 4746e822da..6f82af7772 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -245,7 +245,7 @@ costInsights: name: Cloud Storage icon: storage bigQuery: - name: Big Query + name: BigQuery icon: search metrics: DAU: From c8b24b5bb36675f3337efd52bb16e010af36ae87 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 9 Nov 2020 09:34:11 -0500 Subject: [PATCH 007/259] custom tooltip; export custom bar chart components --- .../cost-insights/src/api/CostInsightsApi.ts | 4 +- plugins/cost-insights/src/client.ts | 109 ++++++++- .../src/components/BarChart/BarChart.test.tsx | 28 +-- .../src/components/BarChart/BarChart.tsx | 77 +++--- .../src/components/BarChart/BarChartLabel.tsx | 11 +- .../BarChartLegend.test.tsx} | 17 +- .../BarChartLegend.tsx} | 51 ++-- .../src/components/BarChart/BarChartTick.tsx | 10 +- .../BarChartTooltip.test.tsx} | 21 +- .../components/BarChart/BarChartTooltip.tsx | 79 ++++++ .../BarChartTooltipItem.tsx} | 19 +- .../src/components/BarChart/helpers.tsx | 42 ++++ .../src/components/BarChart/index.ts | 9 + .../src/components/CostGrowth/CostGrowth.tsx | 2 +- .../CostGrowth/CostGrowthIndicator.test.tsx | 59 +++++ .../CostGrowth/CostGrowthIndicator.tsx | 68 ++++++ .../src/components/CostGrowth/index.ts | 2 + .../CostOverviewCard/CostOverviewChart.tsx | 70 +++--- .../CostOverviewCard/CostOverviewTooltip.tsx | 38 --- .../ProductInsights/ProductInsights.tsx | 1 + .../ProductEntityDialog.tsx | 188 +++++++++++++++ .../ProductInsightsCard.test.tsx | 38 +-- .../ProductInsightsCard.tsx | 66 ++--- .../ProductInsightsChart.tsx | 225 ++++++++++++++++++ .../components/ProductInsightsCard/index.ts | 1 + .../ProjectGrowthAlertCard.tsx | 31 +-- .../ProjectGrowthAlertChart.tsx | 62 +++++ .../ProjectGrowthInstructionsPage.tsx | 79 +++--- .../ResourceGrowthBarChart.test.tsx | 41 ---- .../ResourceGrowthBarChart.tsx | 87 ------- .../ResourceGrowthBarChart/index.ts | 17 -- .../ResourceGrowthBarChartLegend.test.tsx | 59 ----- .../ResourceGrowthBarChartLegend/index.ts | 17 -- .../src/components/Tooltip/Tooltip.tsx | 52 ---- .../src/components/Tooltip/index.ts | 20 -- .../UnlabeledDataflowAlertCard.tsx | 42 ++-- .../UnlabeledDataflowBarChart.tsx | 71 ------ .../UnlabeledDataflowBarChartLegend.tsx | 57 ----- plugins/cost-insights/src/components/index.ts | 1 - plugins/cost-insights/src/types/Alert.ts | 5 +- plugins/cost-insights/src/types/Entity.ts | 65 ++++- plugins/cost-insights/src/types/Product.ts | 10 - plugins/cost-insights/src/types/Theme.ts | 1 + .../cost-insights/src/utils/change.test.ts | 62 +++++ plugins/cost-insights/src/utils/change.ts | 20 +- plugins/cost-insights/src/utils/graphs.ts | 79 ++++++ plugins/cost-insights/src/utils/graphs.tsx | 36 --- plugins/cost-insights/src/utils/mockData.ts | 19 +- plugins/cost-insights/src/utils/sort.ts | 1 + plugins/cost-insights/src/utils/styles.ts | 99 +++++++- 50 files changed, 1425 insertions(+), 843 deletions(-) rename plugins/cost-insights/src/components/{UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.test.tsx => BarChart/BarChartLegend.test.tsx} (56%) rename plugins/cost-insights/src/components/{ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.tsx => BarChart/BarChartLegend.tsx} (55%) rename plugins/cost-insights/src/components/{Tooltip/Tooltip.test.tsx => BarChart/BarChartTooltip.test.tsx} (74%) create mode 100644 plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx rename plugins/cost-insights/src/components/{Tooltip/TooltipItem.tsx => BarChart/BarChartTooltipItem.tsx} (76%) create mode 100644 plugins/cost-insights/src/components/BarChart/helpers.tsx create mode 100644 plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx create mode 100644 plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.tsx delete mode 100644 plugins/cost-insights/src/components/CostOverviewCard/CostOverviewTooltip.tsx create mode 100644 plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx create mode 100644 plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx create mode 100644 plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx delete mode 100644 plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.test.tsx delete mode 100644 plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.tsx delete mode 100644 plugins/cost-insights/src/components/ResourceGrowthBarChart/index.ts delete mode 100644 plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.test.tsx delete mode 100644 plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/index.ts delete mode 100644 plugins/cost-insights/src/components/Tooltip/Tooltip.tsx delete mode 100644 plugins/cost-insights/src/components/Tooltip/index.ts delete mode 100644 plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChart.tsx delete mode 100644 plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.tsx create mode 100644 plugins/cost-insights/src/utils/change.test.ts create mode 100644 plugins/cost-insights/src/utils/graphs.ts delete mode 100644 plugins/cost-insights/src/utils/graphs.tsx diff --git a/plugins/cost-insights/src/api/CostInsightsApi.ts b/plugins/cost-insights/src/api/CostInsightsApi.ts index be5cd54fdb..5493130f78 100644 --- a/plugins/cost-insights/src/api/CostInsightsApi.ts +++ b/plugins/cost-insights/src/api/CostInsightsApi.ts @@ -19,9 +19,9 @@ import { Alert, Cost, Duration, + Entity, Group, Project, - ProductCost, Maybe, MetricData, } from '../types'; @@ -119,7 +119,7 @@ export type CostInsightsApi = { group: string, duration: Duration, project: Maybe, - ): Promise; + ): Promise; /** * Get current cost alerts for a given group. These show up as Action Items for the group on the diff --git a/plugins/cost-insights/src/client.ts b/plugins/cost-insights/src/client.ts index f7671bcce0..afdd9aff74 100644 --- a/plugins/cost-insights/src/client.ts +++ b/plugins/cost-insights/src/client.ts @@ -25,10 +25,10 @@ import { DateAggregation, DEFAULT_DATE_FORMAT, Duration, + Entity, Group, Maybe, MetricData, - ProductCost, Project, ProjectGrowthData, Trendline, @@ -198,10 +198,11 @@ export class ExampleCostInsightsClient implements CostInsightsApi { group: string, duration: Duration, project: Maybe, - ): Promise { + ): Promise { const projectProductInsights = await this.request( { product, group, duration, project }, { + id: 'project-product', aggregation: [80_000, 110_000], change: { ratio: 0.375, @@ -211,25 +212,47 @@ export class ExampleCostInsightsClient implements CostInsightsApi { { id: null, // entities with null ids will be appear as "Unlabeled" in product panels aggregation: [45_000, 50_000], + change: { + ratio: 0.111, + amount: 5_000, + }, + entities: [], }, { id: 'entity-a', aggregation: [15_000, 20_000], + change: { + ratio: 0.333, + amount: 5_000, + }, + entities: [], }, { id: 'entity-b', aggregation: [20_000, 30_000], + change: { + ratio: 0.5, + amount: 10_000, + }, + entities: [], }, { - id: 'entity-e', + id: 'entity-c', aggregation: [0, 10_000], + change: { + ratio: 10_000, + amount: 10_000, + }, + entities: [], }, ], }, ); - const productInsights: ProductCost = await this.request( + + const productInsights: Entity = await this.request( { product, group, duration, project }, { + id: 'product', aggregation: [200_000, 250_000], change: { ratio: 0.2, @@ -238,39 +261,103 @@ export class ExampleCostInsightsClient implements CostInsightsApi { entities: [ { id: null, // entities with null ids will be appear as "Unlabeled" in product panels - aggregation: [15_000, 30_000], + aggregation: [36_000, 42_000], + change: { + ratio: 0.1666, + amount: 6_000, + }, + entities: [], }, { id: 'entity-a', aggregation: [15_000, 20_000], + change: { + ratio: -0.33333333, + amount: 5_000, + }, + entities: [], }, { id: 'entity-b', aggregation: [20_000, 30_000], + change: { + ratio: 0.5, + amount: 10_000, + }, + entities: [], }, { id: 'entity-c', aggregation: [18_000, 25_000], + change: { + ratio: 0.38, + amount: 7_000, + }, + entities: [], }, { id: 'entity-d', - aggregation: [36_000, 42_000], + aggregation: [15_000, 30_000], + change: { + ratio: 1, + amount: 15_000, + }, + entities: [], }, { id: 'entity-e', aggregation: [0, 10_000], + entities: [], + change: { + ratio: 10_000, + amount: 10_000, + }, }, { id: 'entity-f', aggregation: [17_000, 19_000], + change: { + ratio: 0.118, + amount: 2_000, + }, + entities: [], }, { id: 'entity-g', - aggregation: [49_000, 30_000], - }, - { - id: 'entity-h', - aggregation: [0, 34_000], + aggregation: [80_000, 60_000], + change: { + ratio: -0.25, + amount: -20_000, + }, + entities: [ + { + id: 'vCPU Time Batch Belgium', + aggregation: [15_000, 15_000], + change: { + ratio: 0, + amount: 0, + }, + entities: [], + }, + { + id: 'RAM Time Belgium', + aggregation: [15_000, 30_000], + change: { + ratio: 1, + amount: 15_000, + }, + entities: [], + }, + { + id: 'Local Disk Time PD Standard Belgium', + aggregation: [50_000, 15_000], + change: { + ratio: -0.7, + amount: -35_000, + }, + entities: [], + }, + ], }, ], }, diff --git a/plugins/cost-insights/src/components/BarChart/BarChart.test.tsx b/plugins/cost-insights/src/components/BarChart/BarChart.test.tsx index a095324e6e..4e40a5ddda 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChart.test.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChart.test.tsx @@ -15,56 +15,40 @@ */ import React from 'react'; -import { TooltipPayload } from 'recharts'; import { fireEvent } from '@testing-library/react'; import { BarChart, BarChartProps } from './BarChart'; -import { BarChartData, ResourceData } from '../../types'; +import { ResourceData } from '../../types'; import { createMockEntity } from '../../utils/mockData'; import { resourceSort } from '../../utils/sort'; import { renderInTestApp } from '@backstage/test-utils'; -import { TooltipItemProps } from '../Tooltip'; -import { costInsightsLightTheme } from '../../utils/styles'; + +// Disable responsive container console warnings generated by recharts; doesn't disable React warnings. +jest.spyOn(console, 'warn').mockImplementation(() => {}); const MockEntities = [...Array(10)].map((_, index) => - createMockEntity(() => ({ + createMockEntity(mock => ({ + ...mock, id: `test-id-${index + 1}`, // grow resource costs linearly for testing aggregation: [index * 1000, (index + 1) * 1000], })), ); -const MockBarChartData: BarChartData = { - previousFill: costInsightsLightTheme.palette.yellow, - currentFill: costInsightsLightTheme.palette.darkBlue, - previousName: 'Last Period', - currentName: 'Current Period', -}; - const MockResources: ResourceData[] = MockEntities.map(entity => ({ name: entity.id, previous: entity.aggregation[0], current: entity.aggregation[1], })); -const MockTooltipItem = (payload: TooltipPayload): TooltipItemProps => ({ - label: payload.name, - value: payload.value as string, - fill: payload.fill as string, -}); - const renderWithProps = ({ responsive = false, displayAmount = 6, - barChartData = MockBarChartData, - getTooltipItem = MockTooltipItem, resources = MockResources, }: BarChartProps) => { return renderInTestApp( , ); diff --git a/plugins/cost-insights/src/components/BarChart/BarChart.tsx b/plugins/cost-insights/src/components/BarChart/BarChart.tsx index aa1fd5c601..30e8ad71f8 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChart.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChart.tsx @@ -20,51 +20,59 @@ import { BarChart as RechartsBarChart, CartesianGrid, ContentRenderer, - TooltipProps, + TooltipProps as RechartsTooltipProps, + RechartsFunction, ResponsiveContainer, Tooltip as RechartsTooltip, XAxis, YAxis, - TooltipPayload, } from 'recharts'; import { Box, useTheme } from '@material-ui/core'; import { BarChartTick } from './BarChartTick'; import { BarChartStepper } from './BarChartStepper'; -import { Tooltip, TooltipItemProps } from '../Tooltip'; - import { currencyFormatter } from '../../utils/formatters'; import { BarChartData, - Maybe, ResourceData, DataKey, CostInsightsTheme, } from '../../types'; -import { notEmpty } from '../../utils/assert'; import { useBarChartStyles } from '../../utils/styles'; import { resourceSort } from '../../utils/sort'; export type BarChartProps = { + resources: ResourceData[]; responsive?: boolean; displayAmount?: number; - barChartData: BarChartData; - getTooltipItem: (payload: TooltipPayload) => Maybe; - resources: ResourceData[]; + options?: Partial; + tooltip?: ContentRenderer; + onClick?: RechartsFunction; + onMouseMove?: RechartsFunction; }; export const BarChart = ({ + resources, responsive = true, displayAmount = 6, - barChartData, - getTooltipItem, - resources, + options = {}, + tooltip, + onClick, + onMouseMove, }: BarChartProps) => { const theme = useTheme(); const styles = useBarChartStyles(theme); const [activeChart, setActiveChart] = useState(false); const [stepWindow, setStepWindow] = useState(() => [0, displayAmount]); - const { previousFill, currentFill, previousName, currentName } = barChartData; + const data = Object.assign( + { + previousFill: theme.palette.lightBlue, + currentFill: theme.palette.darkBlue, + previousName: 'Previous', + currentName: 'Current', + }, + options, + ); const [stepStart, stepEnd] = stepWindow; const steps = Math.ceil(resources.length / displayAmount); @@ -95,24 +103,11 @@ export const BarChart = ({ [setStepWindow, resources, displayAmount], ); - const handleChartEnter = () => setActiveChart(true); - - const handleChartLeave = () => setActiveChart(false); - - const renderTooltipContent: ContentRenderer = ({ - label, - payload, - }) => { - if (!(payload && typeof label === 'string')) return [null, null]; - const items = payload.map(getTooltipItem).filter(notEmpty); - return ; - }; - return ( setActiveChart(false)} + onMouseEnter={() => setActiveChart(true)} data-testid="bar-chart-wrapper" > {/* Setting fixed values for height and width generates a console warning in testing but enables ResponsiveContainer to render its children. */} @@ -121,16 +116,22 @@ export const BarChart = ({ width={responsive ? '100%' : styles.container.width} > - + {tooltip && ( + + )} 0, globalResourcesMax]} - tick={{ fill: styles.axis.fill }} + tick={styles.axis} /> diff --git a/plugins/cost-insights/src/components/BarChart/BarChartLabel.tsx b/plugins/cost-insights/src/components/BarChart/BarChartLabel.tsx index 0a21f1ed46..c4402893f3 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChartLabel.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartLabel.tsx @@ -15,7 +15,7 @@ */ import React, { PropsWithChildren } from 'react'; -import { Box } from '@material-ui/core'; +import { Box, Typography } from '@material-ui/core'; import { useBarChartLabelStyles } from '../../utils/styles'; type BarChartLabel = { @@ -23,6 +23,7 @@ type BarChartLabel = { y: number; height: number; width: number; + details?: JSX.Element; }; export const BarChartLabel = ({ @@ -30,11 +31,11 @@ export const BarChartLabel = ({ y, height, width, + details, children, }: PropsWithChildren) => { const classes = useBarChartLabelStyles(); const translateX = width * -0.5; - const childArray = React.Children.toArray(children); return ( - {childArray[0]} - {childArray.slice(1)} + + {children} + + {details} ); diff --git a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.test.tsx b/plugins/cost-insights/src/components/BarChart/BarChartLegend.test.tsx similarity index 56% rename from plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.test.tsx rename to plugins/cost-insights/src/components/BarChart/BarChartLegend.test.tsx index ab38ba8d44..3074af716e 100644 --- a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.test.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartLegend.test.tsx @@ -15,20 +15,15 @@ */ import React from 'react'; -import { UnlabeledDataflowBarChartLegend } from './UnlabeledDataflowBarChartLegend'; import { renderInTestApp } from '@backstage/test-utils'; +import { BarChartLegend } from './BarChartLegend'; -describe('', () => { - it('Displays the correct text', async () => { +describe('', () => { + it(`Should display the correct cost start and end`, async () => { const rendered = await renderInTestApp( - , + , ); - expect(rendered.getByText('Total Unlabeled Cost')).toBeInTheDocument(); - expect(rendered.getByText('Total Labeled Cost')).toBeInTheDocument(); - expect(rendered.getByText('$9,843')).toBeInTheDocument(); - expect(rendered.getByText('$2,309')).toBeInTheDocument(); + expect(rendered.getByText(/\$1,000/)).toBeInTheDocument(); + expect(rendered.queryByText(/\$5,000/)).toBeInTheDocument(); }); }); diff --git a/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.tsx b/plugins/cost-insights/src/components/BarChart/BarChartLegend.tsx similarity index 55% rename from plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.tsx rename to plugins/cost-insights/src/components/BarChart/BarChartLegend.tsx index 6b5ce17c95..6db0e13572 100644 --- a/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartLegend.tsx @@ -14,47 +14,58 @@ * limitations under the License. */ -import React from 'react'; +import React, { PropsWithChildren } from 'react'; import { Box, useTheme } from '@material-ui/core'; import { LegendItem } from '../LegendItem'; -import { CostGrowth } from '../CostGrowth'; import { currencyFormatter } from '../../utils/formatters'; -import { ChangeStatistic, CostInsightsTheme, Duration } from '../../types'; +import { CostInsightsTheme } from '../../types'; +import { useBarChartLayoutStyles as useStyles } from '../../utils/styles'; -export type ResourceGrowthBarChartLegendProps = { - change: ChangeStatistic; - duration: Duration; +type BarChartLegendOptions = { previousName: string; + previousFill: string; currentName: string; - costStart: number; - costEnd: number; + currentFill: string; }; -export const ResourceGrowthBarChartLegend = ({ - change, - duration, - previousName, - currentName, +export type BarChartLegendProps = { + costStart: number; + costEnd: number; + options?: Partial; +}; + +export const BarChartLegend = ({ costStart, costEnd, -}: ResourceGrowthBarChartLegendProps) => { + options = {}, + children, +}: PropsWithChildren) => { const theme = useTheme(); + const classes = useStyles(); + + const data = Object.assign( + { + previousName: 'Previous', + previousFill: theme.palette.lightBlue, + currentName: 'Current', + currentFill: theme.palette.darkBlue, + }, + options, + ); return ( - + - + {currencyFormatter.format(costStart)} - + {currencyFormatter.format(costEnd)} - - - + {children} ); }; diff --git a/plugins/cost-insights/src/components/BarChart/BarChartTick.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTick.tsx index 005753fce6..145e5f805b 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChartTick.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartTick.tsx @@ -26,6 +26,7 @@ type BarChartTickProps = { value: any; }; visibleTicksCount: number; + details?: JSX.Element; }; export const BarChartTick = ({ @@ -35,11 +36,18 @@ export const BarChartTick = ({ width, payload, visibleTicksCount, + details, }: BarChartTickProps) => { const gutterWidth = 5; const labelWidth = width / visibleTicksCount - gutterWidth * 2; return ( - + {!payload.value ? 'Unlabeled' : payload.value} ); diff --git a/plugins/cost-insights/src/components/Tooltip/Tooltip.test.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.test.tsx similarity index 74% rename from plugins/cost-insights/src/components/Tooltip/Tooltip.test.tsx rename to plugins/cost-insights/src/components/BarChart/BarChartTooltip.test.tsx index 2f1f8fe7e1..b30714ca58 100644 --- a/plugins/cost-insights/src/components/Tooltip/Tooltip.test.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.test.tsx @@ -16,10 +16,11 @@ import React from 'react'; import { renderInTestApp } from '@backstage/test-utils'; -import { Tooltip } from './Tooltip'; +import { BarChartTooltip } from './BarChartTooltip'; +import { BarChartTooltipItem } from './BarChartTooltipItem'; import { CostInsightsThemeProvider } from '../CostInsightsPage/CostInsightsThemeProvider'; -const mockTooltipItems = [ +const items = [ { label: 'Cost', value: '$1,000,000', @@ -32,22 +33,14 @@ const mockTooltipItems = [ }, ]; -describe('', () => { - it('renders without exploding', async () => { - const rendered = await renderInTestApp( - - - , - ); - expect( - rendered.container.querySelector('.tooltip-content'), - ).toBeInTheDocument(); - }); +const tooltipItems = () => + items.map(item => ); +describe('', () => { it('formats label and tooltip item text correctly', async () => { const rendered = await renderInTestApp( - + {tooltipItems} , ); expect(rendered.getByText('05/16/2020')).toBeInTheDocument(); diff --git a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx new file mode 100644 index 0000000000..2cb09b724e --- /dev/null +++ b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { ReactNode, PropsWithChildren } from 'react'; +import { Box, Divider, Typography } from '@material-ui/core'; +import { useTooltipStyles as useStyles } from '../../utils/styles'; + +export type BarChartTooltipProps = { + title: string; + content?: ReactNode | string; + subtitle?: ReactNode; + topRight?: ReactNode; + actions?: ReactNode; +}; + +export const BarChartTooltip = ({ + title, + content, + subtitle, + topRight, + actions, + children, +}: PropsWithChildren) => { + const classes = useStyles(); + + return ( + + + + {title} + {subtitle && ( + + {subtitle} + + )} + + {topRight} + + {content && ( + + + {content} + + + )} + + {children} + + {actions && ( + <> + + + {actions} + + + )} + + ); +}; diff --git a/plugins/cost-insights/src/components/Tooltip/TooltipItem.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTooltipItem.tsx similarity index 76% rename from plugins/cost-insights/src/components/Tooltip/TooltipItem.tsx rename to plugins/cost-insights/src/components/BarChart/BarChartTooltipItem.tsx index a084a0fd1d..153182560f 100644 --- a/plugins/cost-insights/src/components/Tooltip/TooltipItem.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartTooltipItem.tsx @@ -19,15 +19,18 @@ import { Box, Typography } from '@material-ui/core'; import LensIcon from '@material-ui/icons/Lens'; import { useTooltipStyles as useStyles } from '../../utils/styles'; -export type TooltipItemProps = { - value: string; - label: string; +export type TooltipItem = { fill: string; + label: string; + value: string; }; -export const TooltipItem = ({ fill, label, value }: TooltipItemProps) => { +export type BarChartTooltipItemProps = { + item: TooltipItem; +}; + +export const BarChartTooltipItem = ({ item }: BarChartTooltipItemProps) => { const classes = useStyles(); - const style = { fill: fill }; return ( { > - + - {label} + {item.label} - {value} + {item.value} ); }; diff --git a/plugins/cost-insights/src/components/BarChart/helpers.tsx b/plugins/cost-insights/src/components/BarChart/helpers.tsx new file mode 100644 index 0000000000..66389af771 --- /dev/null +++ b/plugins/cost-insights/src/components/BarChart/helpers.tsx @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { + ContentRenderer, + TooltipProps as RechartsTooltipProps, +} from 'recharts'; +import { BarChartTooltip } from './BarChartTooltip'; +import { BarChartTooltipItem } from './BarChartTooltipItem'; +import { notEmpty } from '../../utils/assert'; +import { isInvalid, titleOf, tooltipItemOf } from '../../utils/graphs'; + +export const renderDefaultTooltip: ContentRenderer = ({ + label, + payload = [], +}) => { + if (isInvalid({ label, payload })) return null; + + const title = titleOf(label); + const items = payload.map(tooltipItemOf).filter(notEmpty); + return ( + + {items.map((item, index) => ( + + ))} + + ); +}; diff --git a/plugins/cost-insights/src/components/BarChart/index.ts b/plugins/cost-insights/src/components/BarChart/index.ts index 4525bd24f3..76f859636a 100644 --- a/plugins/cost-insights/src/components/BarChart/index.ts +++ b/plugins/cost-insights/src/components/BarChart/index.ts @@ -16,3 +16,12 @@ export { BarChart } from './BarChart'; export type { BarChartProps } from './BarChart'; +export { BarChartLegend } from './BarChartLegend'; +export type { BarChartLegendProps } from './BarChartLegend'; +export { BarChartTooltip } from './BarChartTooltip'; +export type { BarChartTooltipProps } from './BarChartTooltip'; +export { BarChartTooltipItem } from './BarChartTooltipItem'; +export type { + TooltipItem, + BarChartTooltipItemProps, +} from './BarChartTooltipItem'; diff --git a/plugins/cost-insights/src/components/CostGrowth/CostGrowth.tsx b/plugins/cost-insights/src/components/CostGrowth/CostGrowth.tsx index 89e44ee185..01a0874636 100644 --- a/plugins/cost-insights/src/components/CostGrowth/CostGrowth.tsx +++ b/plugins/cost-insights/src/components/CostGrowth/CostGrowth.tsx @@ -49,7 +49,7 @@ export const CostGrowth = ({ change, duration }: CostGrowthProps) => { const converted = amount / (currency.rate ?? rate); // Determine if growth is significant enough to highlight - const growth = growthOf(engineers, change.ratio); + const growth = growthOf(change.ratio, engineers); const classes = classnames({ [styles.excess]: growth === GrowthType.Excess, [styles.savings]: growth === GrowthType.Savings, diff --git a/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx b/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx new file mode 100644 index 0000000000..5aabb09c7d --- /dev/null +++ b/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { renderInTestApp } from '@backstage/test-utils'; +import { CostGrowthIndicator } from './CostGrowthIndicator'; +import { ChangeThreshold, EngineerThreshold } from '../../types'; + +describe.each` + ratio | amount | ariaLabel + ${-0.1} | ${undefined} | ${'savings'} + ${0.01} | ${undefined} | ${'excess'} + ${ChangeThreshold.lower} | ${EngineerThreshold} | ${'savings'} + ${ChangeThreshold.lower - 0.01} | ${EngineerThreshold} | ${'savings'} + ${ChangeThreshold.lower - 0.01} | ${EngineerThreshold + 0.1} | ${'savings'} + ${ChangeThreshold.upper} | ${EngineerThreshold} | ${'excess'} + ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold} | ${'excess'} + ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold + 0.1} | ${'excess'} +`('growthOf', ({ ratio, amount, ariaLabel }) => { + it(`should display the correct indicator for ${ + ariaLabel ? ariaLabel : 'negligible' + }`, async () => { + const { getByLabelText } = await renderInTestApp( + , + ); + expect(getByLabelText(ariaLabel)).toBeInTheDocument(); + }); +}); + +describe.each` + ratio | amount + ${0} | ${undefined} + ${ChangeThreshold.lower} | ${0} + ${ChangeThreshold.lower + 0.01} | ${EngineerThreshold} + ${ChangeThreshold.lower + 0.01} | ${EngineerThreshold + 0.1} + ${ChangeThreshold.lower - 0.01} | ${EngineerThreshold - 0.1} + ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold - 0.1} +`('growthOf', ({ ratio, amount }) => { + it('should display the correct indicator for negligible growth', async () => { + const { queryByLabelText } = await renderInTestApp( + , + ); + expect(queryByLabelText('savings')).not.toBeInTheDocument(); + expect(queryByLabelText('excess')).not.toBeInTheDocument(); + }); +}); diff --git a/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.tsx b/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.tsx new file mode 100644 index 0000000000..e220fdb8e7 --- /dev/null +++ b/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.tsx @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import classnames from 'classnames'; +import { Typography, TypographyProps } from '@material-ui/core'; +import { default as ArrowDropUp } from '@material-ui/icons/ArrowDropUp'; +import { default as ArrowDropDown } from '@material-ui/icons/ArrowDropDown'; +import { growthOf } from '../../utils/change'; +import { GrowthType } from '../../types'; +import { useCostGrowthStyles as useStyles } from '../../utils/styles'; + +export type CostGrowthIndicatorProps = TypographyProps & { + ratio: number; + amount?: number; + formatter?: (amount: number) => string; +}; + +export const CostGrowthIndicator = ({ + ratio, + amount, + formatter, + className, + ...props +}: CostGrowthIndicatorProps) => { + const classes = useStyles(); + const growth = growthOf(ratio, amount); + + const classNames = classnames(classes.indicator, className, { + [classes.savings]: growth === GrowthType.Savings, + [classes.excess]: growth === GrowthType.Excess, + }); + + // Display cost as a factor of engineer cost growth and percentage growth + if (typeof amount === 'number') { + return ( + + {formatter ? formatter(amount) : amount} + {growth === GrowthType.Savings && ( + + )} + {growth === GrowthType.Excess && } + + ); + } + + // Display cost as a factor of percent change + return ( + + {formatter ? formatter(ratio) : ratio} + {ratio < 0 && } + {ratio > 0 && } + + ); +}; diff --git a/plugins/cost-insights/src/components/CostGrowth/index.ts b/plugins/cost-insights/src/components/CostGrowth/index.ts index da17ae8709..569c343380 100644 --- a/plugins/cost-insights/src/components/CostGrowth/index.ts +++ b/plugins/cost-insights/src/components/CostGrowth/index.ts @@ -16,3 +16,5 @@ export { CostGrowth } from './CostGrowth'; export type { CostGrowthProps } from './CostGrowth'; +export { CostGrowthIndicator } from './CostGrowthIndicator'; +export type { CostGrowthIndicatorProps } from './CostGrowthIndicator'; diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx index e442b9a64a..1bd945241a 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx @@ -14,32 +14,35 @@ * limitations under the License. */ import React from 'react'; +import moment from 'moment'; import { useTheme } from '@material-ui/core'; import { ComposedChart, + ContentRenderer, + TooltipProps, XAxis, YAxis, - Tooltip, + Tooltip as RechartsTooltip, CartesianGrid, Area, Line, ResponsiveContainer, - TooltipPayload, } from 'recharts'; import { ChartData, Cost, + DEFAULT_DATE_FORMAT, Maybe, Metric, MetricData, CostInsightsTheme, } from '../../types'; +import { BarChartTooltip, BarChartTooltipItem } from '../BarChart'; import { overviewGraphTickFormatter, formatGraphValue, + isInvalid, } from '../../utils/graphs'; -import { CostOverviewTooltip } from './CostOverviewTooltip'; -import { TooltipItemProps } from '../Tooltip'; import { useCostOverviewStyles as useStyles } from '../../utils/styles'; import { groupByDate, toDataMax, trendFrom } from '../../utils/charts'; import { aggregationSort } from '../../utils/sort'; @@ -91,22 +94,39 @@ export const CostOverviewChart = ({ : {}), })); - function tooltipFormatter(payload: TooltipPayload): TooltipItemProps { - return { - label: - payload.dataKey === data.dailyCost.dataKey - ? data.dailyCost.name - : data.metric.name, - value: - payload.dataKey === data.dailyCost.dataKey - ? formatGraphValue(payload.value as number, data.dailyCost.format) - : formatGraphValue(payload.value as number, data.metric.format), - fill: - payload.dataKey === data.dailyCost.dataKey - ? theme.palette.blue - : theme.palette.magenta, - }; - } + const tooltipRenderer: ContentRenderer = ({ + label, + payload = [], + }) => { + if (isInvalid({ label, payload })) return null; + + const dataKeys = [data.dailyCost.dataKey, data.metric.dataKey]; + const title = moment(label).format(DEFAULT_DATE_FORMAT); + const items = payload + .filter(p => dataKeys.includes(p.dataKey as string)) + .map(p => ({ + label: + p.dataKey === data.dailyCost.dataKey + ? data.dailyCost.name + : data.metric.name, + value: + p.dataKey === data.dailyCost.dataKey + ? formatGraphValue(p.value as number, data.dailyCost.format) + : formatGraphValue(p.value as number, data.metric.format), + fill: + p.dataKey === data.dailyCost.dataKey + ? theme.palette.blue + : theme.palette.magenta, + })); + + return ( + + {items.map((item, index) => ( + + ))} + + ); + }; return ( )} - - } - animationDuration={100} - /> + ); diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewTooltip.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewTooltip.tsx deleted file mode 100644 index e0a9b9a9ea..0000000000 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewTooltip.tsx +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import React from 'react'; -import moment from 'moment'; -import { TooltipPayload, TooltipProps } from 'recharts'; -import { Tooltip, TooltipItemProps } from '../../components/Tooltip'; -import { DEFAULT_DATE_FORMAT } from '../../types'; - -export type CostOverviewTooltipProps = TooltipProps & { - dataKeys: Array; - format: (payload: TooltipPayload) => TooltipItemProps; -}; - -export const CostOverviewTooltip = ({ - label, - payload, - dataKeys, - format, -}: CostOverviewTooltipProps) => { - const tooltipLabel = moment(label).format(DEFAULT_DATE_FORMAT); - const items = payload - ?.filter((p: TooltipPayload) => dataKeys.includes(p.dataKey as string)) - .map(p => format(p)); - return ; -}; diff --git a/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx b/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx index 16146cb045..c5d39a5139 100644 --- a/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx +++ b/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx @@ -21,6 +21,7 @@ import { useConfig } from '../../hooks'; export const ProductInsights = ({}) => { const config = useConfig(); + return ( <> diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx new file mode 100644 index 0000000000..764a18d305 --- /dev/null +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx @@ -0,0 +1,188 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import classnames from 'classnames'; +import { Table, TableColumn } from '@backstage/core'; +import { Dialog, IconButton, Typography } from '@material-ui/core'; +import { default as CloseButton } from '@material-ui/icons/Close'; +import { CostGrowthIndicator } from '../CostGrowth'; +import { currencyFormatter, formatPercent } from '../../utils/formatters'; +import { useEntityDialogStyles as useStyles } from '../../utils/styles'; +import { BarChartOptions, Entity } from '../../types'; + +function createRenderer(col: keyof RowData, classes: Record) { + return function render(rowData: {}): JSX.Element { + const row = rowData as RowData; + const rowStyles = classnames(classes.row, { + [classes.rowTotal]: row.id === 'total', + [classes.colFirst]: col === 'sku', + [classes.colLast]: col === 'ratio', + }); + + switch (col) { + case 'previous': + case 'current': + return ( + + {currencyFormatter.format(row[col])} + + ); + case 'ratio': + return ( + formatPercent(Math.abs(amount))} + /> + ); + default: + return {row.sku}; + } + }; +} + +// material-table does not support fixed rows. Override the sorting algorithm +// to force Total row to bottom by default or when a user sort toggles a column. +function createSorter(field?: keyof Omit) { + return function rowSort(data1: {}, data2: {}): number { + const a = data1 as RowData; + const b = data2 as RowData; + if (a.id === 'total') return 1; + if (b.id === 'total') return 1; + if (field === 'sku') return a.sku.localeCompare(b.sku); + + return field + ? a[field] - b[field] + : b.previous + b.current - (a.previous - a.current); + }; +} + +const defaultEntity: Entity = { + id: null, + aggregation: [0, 0], + change: { ratio: 0, amount: 0 }, + entities: [], +}; + +type RowData = { + id: string; + sku: string; + previous: number; + current: number; + ratio: number; +}; + +type ProductEntityDialogOptions = Partial< + Pick +>; + +type ProductEntityDialogProps = { + open: boolean; + entity?: Entity; + options?: ProductEntityDialogOptions; + onClose: () => void; +}; + +export const ProductEntityDialog = ({ + open, + entity = defaultEntity, + options = {}, + onClose, +}: ProductEntityDialogProps) => { + const classes = useStyles(); + + const data = Object.assign( + { + previousName: 'Previous', + currentName: 'Current', + }, + options, + ); + + const firstColClasses = classnames(classes.column, classes.colFirst); + const lastColClasses = classnames(classes.column, classes.colLast); + + const columns: TableColumn[] = [ + { + field: 'sku', + title: SKU, + render: createRenderer('sku', classes), + customSort: createSorter('sku'), + }, + { + field: 'previous', + title: ( + {data.previousName} + ), + align: 'right', + render: createRenderer('previous', classes), + customSort: createSorter('previous'), + }, + { + field: 'current', + title: ( + {data.currentName} + ), + align: 'right', + render: createRenderer('current', classes), + customSort: createSorter('current'), + }, + { + field: 'ratio', + title: M/M, + align: 'right', + render: createRenderer('ratio', classes), + customSort: createSorter('ratio'), + }, + ]; + + const rowData: RowData[] = entity.entities + .map(e => ({ + id: e.id || 'Unknown', + sku: e.id || 'Unknown', + previous: e.aggregation[0], + current: e.aggregation[1], + ratio: e.change.ratio, + })) + .concat({ + id: 'total', + sku: 'Total', + previous: entity.aggregation[0], + current: entity.aggregation[1], + ratio: entity.change.ratio, + }) + .sort(createSorter()); + + return ( + + + + + + + ); +}; diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.test.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.test.tsx index 9d1610e298..cdbbb79806 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.test.tsx +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.test.tsx @@ -20,7 +20,6 @@ import { ProductInsightsCard } from './ProductInsightsCard'; import { CostInsightsApi } from '../../api'; import { createMockEntity, - createMockProductCost, mockDefaultLoadingState, MockComputeEngine, MockProductFilters, @@ -35,16 +34,14 @@ import { MockScrollProvider, MockLoadingProvider, } from '../../utils/tests'; -import { Duration, Product, ProductCost, ProductPeriod } from '../../types'; +import { Duration, Entity, Product, ProductPeriod } from '../../types'; -const costInsightsApi = ( - productCost: ProductCost, -): Partial => ({ - getProductInsights: () => - Promise.resolve(productCost) as Promise, +const costInsightsApi = (entity: Entity): Partial => ({ + getProductInsights: () => Promise.resolve(entity), }); -const mockProductCost = createMockProductCost(() => ({ +const mockProductCost = createMockEntity(() => ({ + id: 'test-id', entities: [], aggregation: [3000, 4000], change: { @@ -54,12 +51,12 @@ const mockProductCost = createMockProductCost(() => ({ })); const renderProductInsightsCardInTestApp = async ( - productCost: ProductCost, + entity: Entity, product: Product, duration: Duration, ) => await renderInTestApp( - + @@ -96,27 +93,30 @@ describe('', () => { }); it('Should render the right subheader for products with cost data', async () => { - const productCost = { + const entity = { ...mockProductCost, entities: [...Array(1000)].map(createMockEntity), }; const rendered = await renderProductInsightsCardInTestApp( - productCost, + entity, MockComputeEngine, Duration.P1M, ); const subheader = 'entities, sorted by cost'; - const subheaderRgx = new RegExp( - `${productCost.entities.length} ${subheader}`, - ); + const subheaderRgx = new RegExp(`${entity.entities.length} ${subheader}`); expect(rendered.getByText(subheaderRgx)).toBeInTheDocument(); }); it('Should render the right subheader if there is no cost data or change data', async () => { - const productCost = { entities: [], aggregation: [0, 0] } as ProductCost; + const entity: Entity = { + id: 'test-id', + entities: [], + aggregation: [0, 0], + change: { ratio: 0, amount: 0 }, + }; const subheader = `There are no ${MockComputeEngine.name} costs within this timeframe for your team's projects.`; const rendered = await renderProductInsightsCardInTestApp( - productCost, + entity, MockComputeEngine, Duration.P1M, ); @@ -138,12 +138,12 @@ describe('', () => { 'Should display the correct relative time', ({ duration, periodStartText, periodEndText }) => { it(`Should display the correct relative time for ${duration}`, async () => { - const productCost = { + const entity = { ...mockProductCost, entities: [...Array(3)].map(createMockEntity), }; const rendered = await renderProductInsightsCardInTestApp( - productCost, + entity, MockComputeEngine, duration, ); diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.tsx index 840c99522a..da304d42a2 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.tsx +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsCard.tsx @@ -16,23 +16,15 @@ import React, { useCallback, useEffect, useState } from 'react'; import { InfoCard, useApi } from '@backstage/core'; -import { Box } from '@material-ui/core'; import Alert from '@material-ui/lab/Alert'; import { costInsightsApiRef } from '../../api'; +import { ProductInsightsChart } from './ProductInsightsChart'; import { PeriodSelect } from '../PeriodSelect'; -import { ResourceGrowthBarChart } from '../ResourceGrowthBarChart'; -import { ResourceGrowthBarChartLegend } from '../ResourceGrowthBarChartLegend'; -import { - useFilters, - useLastCompleteBillingDate, - useLoading, - useScroll, -} from '../../hooks'; +import { useFilters, useLoading, useScroll } from '../../hooks'; import { useProductInsightsCardStyles as useStyles } from '../../utils/styles'; import { mapFiltersToProps, mapLoadingToProps } from './selector'; -import { Duration, Maybe, Product, ProductCost } from '../../types'; +import { Duration, Maybe, Product, Entity } from '../../types'; import { pluralOf } from '../../utils/grammar'; -import { formatPeriod } from '../../utils/formatters'; type ProductInsightsCardProps = { product: Product; @@ -42,8 +34,7 @@ export const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => { const client = useApi(costInsightsApiRef); const classes = useStyles(); const { ScrollAnchor } = useScroll(product.kind); - const lastCompleteBillingDate = useLastCompleteBillingDate(); - const [resource, setResource] = useState>(null); + const [entity, setEntity] = useState>(null); const [error, setError] = useState>(null); const { group, product: productFilter, setProduct, project } = useFilters( @@ -57,38 +48,24 @@ export const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => { // eslint-disable-next-line react-hooks/exhaustive-deps const dispatchLoadingProduct = useCallback(dispatchLoading, [product.kind]); - const amount = resource?.entities?.length || 0; - const hasCostsWithinTimeframe = resource?.change && !!amount; - - const previousName = formatPeriod( - productFilter.duration, - lastCompleteBillingDate, - false, - ); - const currentName = formatPeriod( - productFilter.duration, - lastCompleteBillingDate, - true, - ); + const amount = entity?.entities?.length || 0; + const hasCostsWithinTimeframe = entity?.change && !!amount; const subheader = amount ? `${amount} ${pluralOf(amount, 'entity', 'entities')}, sorted by cost` : `There are no ${product.name} costs within this timeframe for your team's projects.`; - const costStart = resource?.aggregation[0] || 0; - const costEnd = resource?.aggregation[1] || 0; - useEffect(() => { async function load() { if (loadingProduct) { try { - const p: ProductCost = await client.getProductInsights( + const e: Entity = await client.getProductInsights( product.kind, group!, productFilter!.duration, project, ); - setResource(p); + setEntity(e); } catch (e) { setError(e); } finally { @@ -100,7 +77,7 @@ export const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => { }, [ client, product, - setResource, + setEntity, loadingProduct, dispatchLoadingProduct, productFilter, @@ -135,7 +112,7 @@ export const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => { ); } - if (!resource) { + if (!entity) { return null; } @@ -143,25 +120,10 @@ export const ProductInsightsCard = ({ product }: ProductInsightsCardProps) => { {hasCostsWithinTimeframe && ( - <> - - - - - - - + )} ); diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx new file mode 100644 index 0000000000..507be28500 --- /dev/null +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx @@ -0,0 +1,225 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useEffect, useState } from 'react'; +import { + ContentRenderer, + TooltipProps as RechartsTooltipProps, + RechartsFunction, +} from 'recharts'; +import { Box, Typography } from '@material-ui/core'; +import { default as FullScreenIcon } from '@material-ui/icons/Fullscreen'; +import { useLastCompleteBillingDate } from '../../hooks'; +import { LegendItem } from '../LegendItem'; +import { ProductEntityDialog } from './ProductEntityDialog'; +import { CostGrowth, CostGrowthIndicator } from '../CostGrowth'; +import { + BarChart, + BarChartLegend, + BarChartTooltip, + BarChartTooltipItem, +} from '../BarChart'; +import { pluralOf } from '../../utils/grammar'; +import { findAlways, notEmpty } from '../../utils/assert'; +import { formatPeriod, formatPercent } from '../../utils/formatters'; +import { + titleOf, + tooltipItemOf, + resourceOf, + isInvalid, + isActiveLabel, +} from '../../utils/graphs'; +import { + useProductInsightsChartStyles as useStyles, + useBarChartLayoutStyles as useLayoutStyles, +} from '../../utils/styles'; +import { BarChartOptions, Duration, Entity, Maybe } from '../../types'; + +export type ProductInsightsChartProps = { + entity: Entity; + duration: Duration; +}; + +export const ProductInsightsChart = ({ + entity, + duration, +}: ProductInsightsChartProps) => { + const classes = useStyles(); + const layoutClasses = useLayoutStyles(); + const lastCompleteBillingDate = useLastCompleteBillingDate(); + const [isOpen, setOpen] = useState(false); + const [isClickable, setClickable] = useState(true); + const [selectLabel, setSelected] = useState>(null); + const [activeLabel, setActive] = useState>(null); + + const legendTitle = `Cost ${entity.change.ratio <= 0 ? 'Savings' : 'Growth'}`; + const costStart = entity.aggregation[0]; + const costEnd = entity.aggregation[1]; + const resources = entity.entities.map(resourceOf); + + const options: Partial = { + previousName: formatPeriod(duration, lastCompleteBillingDate, false), + currentName: formatPeriod(duration, lastCompleteBillingDate, true), + }; + + useEffect(() => { + function toggleModal() { + if (selectLabel) { + setOpen(true); + } else { + setOpen(false); + } + } + + toggleModal(); + }, [selectLabel]); + + useEffect(() => { + // disable click if an entity is unlabeled or if it does not have any skus + function toggleClickableEntity() { + if (activeLabel) { + const hasSkus = entity.entities.find(e => e.id === activeLabel) + ?.entities.length; + if (hasSkus) { + setClickable(true); + } else { + setClickable(false); + } + } else { + setClickable(false); + } + } + + toggleClickableEntity(); + }, [activeLabel, entity]); + + const onMouseMove: RechartsFunction = ( + data: Record<'activeLabel', string | undefined>, + ) => { + if (isActiveLabel(data)) { + setActive(data.activeLabel!); + } else { + setActive(null); + } + }; + + const onClick: RechartsFunction = (data: Record<'activeLabel', string>) => { + if (isActiveLabel(data)) { + setSelected(data.activeLabel); + } else { + setSelected(null); + } + }; + + const renderProductInsightsTooltip: ContentRenderer = ({ + label, + payload = [], + }) => { + /* Labels and payloads may be undefined or empty */ + if (isInvalid({ label, payload })) return null; + + /* + * recharts coerces null values to strings + * entity -> resource -> payload + * { id: null } -> { name: null } -> { label: '' } + */ + const id = label === '' ? null : label; + + const title = titleOf(label); + const items = payload.map(tooltipItemOf).filter(notEmpty); + + const activeEntity = findAlways(entity.entities, e => e.id === id); + const ratio = activeEntity.change.ratio; + const skus = activeEntity.entities; + const subtitle = `${skus.length} ${pluralOf(skus.length, 'SKU', 'SKUs')}`; + + if (skus.length) { + return ( + + } + actions={ + + + Click for breakdown + + } + > + {items.map((item, index) => ( + + ))} + + ); + } + + // If an entity doesn't have any skus, there aren't any costs to break down. + return ( + + } + content={ + id + ? null + : `This service is generating costs that can't be attributed to any known entity.` + } + > + {items.map((item, index) => ( + + ))} + + ); + }; + + const barChartProps = isClickable ? { onClick } : {}; + + return ( + + + + + + + + {selectLabel && entity.entities.length && ( + setSelected(null)} + entity={entity.entities.find(e => e.id === selectLabel)} + options={options} + /> + )} + + ); +}; diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/index.ts b/plugins/cost-insights/src/components/ProductInsightsCard/index.ts index 61222f9466..33a402234c 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/index.ts +++ b/plugins/cost-insights/src/components/ProductInsightsCard/index.ts @@ -15,3 +15,4 @@ */ export { ProductInsightsCard } from './ProductInsightsCard'; +export { ProductInsightsChart } from './ProductInsightsChart'; diff --git a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.tsx b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.tsx index 4311bcc1bf..bcda21e0af 100644 --- a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.tsx +++ b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.tsx @@ -15,12 +15,9 @@ */ import React from 'react'; -import moment from 'moment'; -import { Box } from '@material-ui/core'; import { InfoCard } from '@backstage/core'; -import { ResourceGrowthBarChart } from '../ResourceGrowthBarChart'; -import { ResourceGrowthBarChartLegend } from '../ResourceGrowthBarChartLegend'; -import { Duration, ProjectGrowthData } from '../../types'; +import { ProjectGrowthAlertChart } from './ProjectGrowthAlertChart'; +import { ProjectGrowthData } from '../../types'; import { pluralOf } from '../../utils/grammar'; type ProjectGrowthAlertProps = { @@ -28,39 +25,17 @@ type ProjectGrowthAlertProps = { }; export const ProjectGrowthAlertCard = ({ alert }: ProjectGrowthAlertProps) => { - const [costStart, costEnd] = alert.aggregation; - const subheader = ` ${alert.products.length} ${pluralOf(alert.products.length, 'product')}${ alert.products.length > 1 ? ', sorted by cost' : '' }`; - const previousName = moment(alert.periodStart, 'YYYY-[Q]Q').format( - '[Q]Q YYYY', - ); - const currentName = moment(alert.periodEnd, 'YYYY-[Q]Q').format('[Q]Q YYYY'); return ( - - - - - - + ); }; diff --git a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx new file mode 100644 index 0000000000..dffed24992 --- /dev/null +++ b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import moment from 'moment'; +import { Box } from '@material-ui/core'; +import { BarChart, BarChartLegend } from '../BarChart'; +import { LegendItem } from '../LegendItem'; +import { CostGrowth } from '../CostGrowth'; +import { renderDefaultTooltip } from '../BarChart/helpers'; +import { BarChartOptions, Duration, ProjectGrowthData } from '../../types'; +import { useBarChartLayoutStyles as useStyles } from '../../utils/styles'; +import { resourceOf } from '../../utils/graphs'; + +type ProjectGrowthAlertChartProps = { + alert: ProjectGrowthData; +}; + +export const ProjectGrowthAlertChart = ({ + alert, +}: ProjectGrowthAlertChartProps) => { + const classes = useStyles(); + + const costStart = alert.aggregation[0]; + const costEnd = alert.aggregation[1]; + const resourceData = alert.products.map(resourceOf); + + const options: Partial = { + previousName: moment(alert.periodStart, 'YYYY-[Q]Q').format('[Q]Q YYYY'), + currentName: moment(alert.periodEnd, 'YYYY-[Q]Q').format('[Q]Q YYYY'), + }; + + return ( + + + + + + + + + ); +}; diff --git a/plugins/cost-insights/src/components/ProjectGrowthInstructionsPage/ProjectGrowthInstructionsPage.tsx b/plugins/cost-insights/src/components/ProjectGrowthInstructionsPage/ProjectGrowthInstructionsPage.tsx index f2083388ae..ab5d7f3163 100644 --- a/plugins/cost-insights/src/components/ProjectGrowthInstructionsPage/ProjectGrowthInstructionsPage.tsx +++ b/plugins/cost-insights/src/components/ProjectGrowthInstructionsPage/ProjectGrowthInstructionsPage.tsx @@ -15,19 +15,23 @@ */ import React from 'react'; +import moment from 'moment'; import { Box, Typography } from '@material-ui/core'; import { InfoCard } from '@backstage/core'; import { AlertInstructionsLayout } from '../AlertInstructionsLayout'; +import { ProductInsightsChart } from '../ProductInsightsCard'; +import { MockBillingDateProvider } from '../../utils/tests'; import { Alert, + DEFAULT_DATE_FORMAT, Duration, Entity, Product, ProjectGrowthData, } from '../../types'; import { ProjectGrowthAlert } from '../../utils/alerts'; -import { ResourceGrowthBarChartLegend } from '../ResourceGrowthBarChartLegend'; -import { ResourceGrowthBarChart } from '../ResourceGrowthBarChart'; + +const today = moment().format(DEFAULT_DATE_FORMAT); export const ProjectGrowthInstructionsPage = () => { const alertData: ProjectGrowthData = { @@ -54,6 +58,7 @@ export const ProjectGrowthInstructionsPage = () => { }, ], }; + const projectGrowthAlert: Alert = new ProjectGrowthAlert(alertData); const product: Product = { @@ -61,20 +66,34 @@ export const ProjectGrowthInstructionsPage = () => { name: 'Compute Engine', }; - const entities: Entity[] = [ - { - id: 'service-one', - aggregation: [18200, 58500], + const entity: Entity = { + id: 'example-id', + aggregation: [20_000, 60_000], + change: { + ratio: 3, + amount: 40_000, }, - { - id: 'service-two', - aggregation: [1200, 1300], - }, - { - id: 'service-three', - aggregation: [600, 200], - }, - ]; + entities: [ + { + id: 'service-one', + aggregation: [18_200, 58_500], + entities: [], + change: { ratio: 2.21, amount: 40_300 }, + }, + { + id: 'service-two', + aggregation: [1200, 1300], + entities: [], + change: { ratio: 0.083, amount: 100 }, + }, + { + id: 'service-three', + aggregation: [600, 200], + entities: [], + change: { ratio: -0.666, amount: -400 }, + }, + ], + }; return ( @@ -153,28 +172,14 @@ export const ProjectGrowthInstructionsPage = () => { ) that has grown in cost: - {/* ProductInsightsCard without API query / PeriodSelect */} - - - - - - - - - - + + + + + From here, you can dig into commit history or deployment logs to find diff --git a/plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.test.tsx b/plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.test.tsx deleted file mode 100644 index e3d9d61d1b..0000000000 --- a/plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.test.tsx +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { ResourceGrowthBarChart } from './ResourceGrowthBarChart'; -import { renderInTestApp } from '@backstage/test-utils'; -import { createMockEntity } from '../../utils/mockData'; - -const MockResources = [...Array(10)].map((_, index) => - createMockEntity(() => ({ - id: `test-id-${index + 1}`, - // grow resource costs linearly for testing - aggregation: [index * 1000, (index + 1) * 1000], - })), -); - -describe('', () => { - it('Pre-renders without exploding', async () => { - const rendered = await renderInTestApp( - , - ); - expect(rendered.queryByTestId('bar-chart-wrapper')).toBeInTheDocument(); - }); -}); diff --git a/plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.tsx b/plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.tsx deleted file mode 100644 index b4014bf00f..0000000000 --- a/plugins/cost-insights/src/components/ResourceGrowthBarChart/ResourceGrowthBarChart.tsx +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { TooltipPayload } from 'recharts'; -import { currencyFormatter } from '../../utils/formatters'; -import { - AlertCost, - BarChartData, - CostInsightsTheme, - DataKey, - Entity, - Maybe, - ResourceData, -} from '../../types'; -import { BarChart } from '../BarChart'; -import { TooltipItemProps } from '../Tooltip'; -import { useTheme } from '@material-ui/core'; - -export type ResourceGrowthBarChartProps = { - resources: Array; - previousName: string; - currentName: string; -}; - -export const ResourceGrowthBarChart = ({ - resources, - previousName, - currentName, -}: ResourceGrowthBarChartProps) => { - const theme = useTheme(); - const getTooltipItem = (payload: TooltipPayload): Maybe => { - const value = - typeof payload.value === 'number' - ? currencyFormatter.format(payload.value) - : (payload.value as string); - const fill = payload.fill as string; - - switch (payload.dataKey) { - case DataKey.Current: - case DataKey.Previous: - return { - label: payload.name, - value: value, - fill: fill, - }; - default: - return null; - } - }; - - const barChartData: BarChartData = { - previousFill: theme.palette.lightBlue, - currentFill: theme.palette.darkBlue, - previousName: previousName, - currentName: currentName, - }; - - const resourceData: ResourceData[] = resources.map(resource => { - return { - name: resource.id, - previous: resource.aggregation[0], - current: resource.aggregation[1], - }; - }); - - return ( - - ); -}; diff --git a/plugins/cost-insights/src/components/ResourceGrowthBarChart/index.ts b/plugins/cost-insights/src/components/ResourceGrowthBarChart/index.ts deleted file mode 100644 index ef649a17af..0000000000 --- a/plugins/cost-insights/src/components/ResourceGrowthBarChart/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { ResourceGrowthBarChart } from './ResourceGrowthBarChart'; diff --git a/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.test.tsx b/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.test.tsx deleted file mode 100644 index 47285a2363..0000000000 --- a/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/ResourceGrowthBarChartLegend.test.tsx +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React, { PropsWithChildren } from 'react'; -import { renderInTestApp } from '@backstage/test-utils'; -import { ResourceGrowthBarChartLegend } from './ResourceGrowthBarChartLegend'; -import { defaultCurrencies } from '../../utils/currency'; -import { findAlways } from '../../utils/assert'; -import { MockConfigProvider, MockCurrencyProvider } from '../../utils/tests'; -import { Duration } from '../../types'; - -const engineers = findAlways(defaultCurrencies, c => c.kind === null); - -const MockContext = ({ children }: PropsWithChildren<{}>) => ( - - {children} - -); - -describe('', () => { - describe.each` - ratio | amount | costText | engineerTest - ${2.5} | ${300_000} | ${'Cost Growth'} | ${/\~6 engineers/} - ${-2.5} | ${-120_000} | ${'Cost Savings'} | ${/\~2 engineers/} - `( - 'Should display the cost text', - ({ ratio, amount, costText, engineerTest }) => { - it(`Should display the correct cost and engineer text for ${ratio} percent change`, async () => { - const rendered = await renderInTestApp( - - - , - ); - expect(rendered.getByText(costText)).toBeInTheDocument(); - expect(rendered.queryByText(engineerTest)).toBeInTheDocument(); - }); - }, - ); -}); diff --git a/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/index.ts b/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/index.ts deleted file mode 100644 index ee77a483e6..0000000000 --- a/plugins/cost-insights/src/components/ResourceGrowthBarChartLegend/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { ResourceGrowthBarChartLegend } from './ResourceGrowthBarChartLegend'; diff --git a/plugins/cost-insights/src/components/Tooltip/Tooltip.tsx b/plugins/cost-insights/src/components/Tooltip/Tooltip.tsx deleted file mode 100644 index e0ee93c669..0000000000 --- a/plugins/cost-insights/src/components/Tooltip/Tooltip.tsx +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { Box, Typography } from '@material-ui/core'; -import { TooltipItem, TooltipItemProps } from './TooltipItem'; -import { useTooltipStyles } from '../../utils/styles'; - -export type TooltipProps = { - label?: string; - items?: Array; -}; - -export const Tooltip = ({ label, items }: TooltipProps) => { - const classes = useTooltipStyles(); - return ( - - {label && ( - - {label} - - )} - {items && - items.map((item, index) => ( - - ))} - - ); -}; diff --git a/plugins/cost-insights/src/components/Tooltip/index.ts b/plugins/cost-insights/src/components/Tooltip/index.ts deleted file mode 100644 index b26409480e..0000000000 --- a/plugins/cost-insights/src/components/Tooltip/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { Tooltip } from './Tooltip'; -export type { TooltipProps } from './Tooltip'; -export { TooltipItem } from './TooltipItem'; -export type { TooltipItemProps } from './TooltipItem'; diff --git a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx b/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx index 8bc5ebfb6e..5384db566c 100644 --- a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx +++ b/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx @@ -15,12 +15,13 @@ */ import React from 'react'; -import { Box } from '@material-ui/core'; import { InfoCard } from '@backstage/core'; -import { UnlabeledDataflowBarChart } from './UnlabeledDataflowBarChart'; -import { UnlabeledDataflowBarChartLegend } from './UnlabeledDataflowBarChartLegend'; -import { UnlabeledDataflowData } from '../../types'; +import { Box } from '@material-ui/core'; +import { BarChart, BarChartLegend } from '../BarChart'; +import { renderDefaultTooltip } from '../BarChart/helpers'; +import { UnlabeledDataflowData, ResourceData } from '../../types'; import { pluralOf } from '../../utils/grammar'; +import { useBarChartLayoutStyles as useStyles } from '../../utils/styles'; type UnlabeledDataflowAlertProps = { alert: UnlabeledDataflowData; @@ -29,22 +30,35 @@ type UnlabeledDataflowAlertProps = { export const UnlabeledDataflowAlertCard = ({ alert, }: UnlabeledDataflowAlertProps) => { + const classes = useStyles(); const projects = pluralOf(alert.projects.length, 'project'); const subheader = ` Showing costs from ${alert.projects.length} ${projects} with unlabeled Dataflow jobs in the last 30 days. `; + const options = { + previousName: 'Unlabeled Cost', + currentName: 'Labeled Cost', + }; + + const resources: ResourceData[] = alert.projects.map(project => ({ + name: project.id, + previous: project.unlabeledCost, + current: project.labeledCost, + })); + return ( - - - - - - - + + + ); diff --git a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChart.tsx b/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChart.tsx deleted file mode 100644 index faf27ba04f..0000000000 --- a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChart.tsx +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { TooltipPayload } from 'recharts'; -import { BarChart } from '../BarChart'; -import { TooltipItemProps } from '../Tooltip'; -import { - BarChartData, - CostInsightsTheme, - ResourceData, - UnlabeledDataflowAlertProject, -} from '../../types'; -import { currencyFormatter } from '../../utils/formatters'; -import { useTheme } from '@material-ui/core'; - -type UnlabeledDataflowBarChartProps = { - projects: Array; -}; - -export const UnlabeledDataflowBarChart = ({ - projects, -}: UnlabeledDataflowBarChartProps) => { - const theme = useTheme(); - const barChartData: BarChartData = { - previousFill: theme.palette.lightBlue, - currentFill: theme.palette.darkBlue, - previousName: 'Unlabeled Cost', - currentName: 'Labeled Cost', - }; - - const getTooltipItem = (payload: TooltipPayload): TooltipItemProps => { - return { - label: payload.name, - value: - typeof payload.value === 'number' - ? currencyFormatter.format(payload.value) - : (payload.value as string), - fill: payload.fill as string, - }; - }; - - const resources: ResourceData[] = projects.map(project => { - return { - name: project.id, - previous: project.unlabeledCost || 0, - current: project.labeledCost || 0, - }; - }); - - return ( - - ); -}; diff --git a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.tsx b/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.tsx deleted file mode 100644 index aac37c799d..0000000000 --- a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowBarChartLegend.tsx +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { Box, useTheme } from '@material-ui/core'; -import { LegendItem } from '../LegendItem'; -import { currencyFormatter } from '../../utils/formatters'; -import { CostInsightsTheme } from '../../types'; - -type UnlabeledDataflowBarChartLegendProps = { - labeledCost: number; - unlabeledCost: number; -}; - -export const UnlabeledDataflowBarChartLegend = ({ - unlabeledCost, - labeledCost, -}: UnlabeledDataflowBarChartLegendProps) => { - const theme = useTheme(); - return ( - - - - {currencyFormatter.format(unlabeledCost)} - - - - - {currencyFormatter.format(labeledCost)} - - - - ); -}; diff --git a/plugins/cost-insights/src/components/index.ts b/plugins/cost-insights/src/components/index.ts index 5b918144d9..c390666419 100644 --- a/plugins/cost-insights/src/components/index.ts +++ b/plugins/cost-insights/src/components/index.ts @@ -17,4 +17,3 @@ export * from './BarChart'; export * from './CostGrowth'; export * from './LegendItem'; -export * from './Tooltip'; diff --git a/plugins/cost-insights/src/types/Alert.ts b/plugins/cost-insights/src/types/Alert.ts index d6046e3241..c5c0463d99 100644 --- a/plugins/cost-insights/src/types/Alert.ts +++ b/plugins/cost-insights/src/types/Alert.ts @@ -41,13 +41,16 @@ export interface ResourceData { name: Maybe; } -export interface BarChartData { +export interface BarChartOptions { previousFill: string; currentFill: string; previousName: string; currentName: string; } +/** deprecated use BarChartOptions instead */ +export interface BarChartData extends BarChartOptions {} + export enum DataKey { Previous = 'previous', Current = 'current', diff --git a/plugins/cost-insights/src/types/Entity.ts b/plugins/cost-insights/src/types/Entity.ts index f5ac620289..83347f05c3 100644 --- a/plugins/cost-insights/src/types/Entity.ts +++ b/plugins/cost-insights/src/types/Entity.ts @@ -20,5 +20,68 @@ import { Maybe } from './Maybe'; export interface Entity { id: Maybe; aggregation: [number, number]; - change?: Maybe; + entities: Entity[]; + change: ChangeStatistic; } + +/* + An entity is a tree-like structure that represents any unique + product or service that generates cost over a fixed period of time. + An entity could be atomic or composite. An atomic entity is indivisible + and cannot be broken into sub-entities. + + A composite entity can be broken down recursively into sub-entities + that generate cost **over the same time period**. All costs must sum to the root cost. + + Entities with null ids are considered "unlabeled" - costs without attribution. + If an entity is a composite, it may only have one (1) null child but may have any number of + null grandchildren. + + { + id: 'product', + aggregation: [0, 200], + change: { + ratio: 2000, + amount: 200 + }, + entities: [ + { + id: 'service-a', + aggregation: [0, 100], + change: { + ratio: 100, + amount: 100 + }, + entities: [] + }, + { + id: 'service-b', + aggregation: [0, 100], + change: { + ratio: 100, + amount: 100 + }, + entities: [ + { + id: 'service-b-sku-a', + aggregation: [0, 25], + change: { + ratio: 25, + amount: 25 + }, + entities: [] + }, + { + id: null, // Unlabeled cost for service-b + aggregation: [0, 75], + change: { + ratio: 75, + amount: 75 + }, + entities: [] + }, + ] + }, + ] + } +*/ diff --git a/plugins/cost-insights/src/types/Product.ts b/plugins/cost-insights/src/types/Product.ts index d2087644cb..77df579a11 100644 --- a/plugins/cost-insights/src/types/Product.ts +++ b/plugins/cost-insights/src/types/Product.ts @@ -14,17 +14,7 @@ * limitations under the License. */ -import { Entity } from './Entity'; -import { ChangeStatistic } from './ChangeStatistic'; -import { Maybe } from './Maybe'; - export interface Product { kind: string; name: string; } - -export interface ProductCost { - entities?: Array; - aggregation: [number, number]; - change?: Maybe; -} diff --git a/plugins/cost-insights/src/types/Theme.ts b/plugins/cost-insights/src/types/Theme.ts index 9053283351..fecaa17abb 100644 --- a/plugins/cost-insights/src/types/Theme.ts +++ b/plugins/cost-insights/src/types/Theme.ts @@ -30,6 +30,7 @@ type CostInsightsPaletteAdditions = { tooltip: CostInsightsTooltipOptions; navigationText: string; alertBackground: string; + textSecondary: string; }; export type CostInsightsPalette = BackstagePalette & diff --git a/plugins/cost-insights/src/utils/change.test.ts b/plugins/cost-insights/src/utils/change.test.ts new file mode 100644 index 0000000000..daf4538f2c --- /dev/null +++ b/plugins/cost-insights/src/utils/change.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { growthOf } from './change'; +import { GrowthType, ChangeThreshold, EngineerThreshold } from '../types'; + +const GrowthMap = { + [GrowthType.Negligible]: 'negligible growth', + [GrowthType.Savings]: 'cost savings', + [GrowthType.Excess]: 'cost excess', +}; + +describe.each` + ratio | amount | expected + ${0.0} | ${undefined} | ${GrowthType.Negligible} + ${0.0} | ${EngineerThreshold} | ${GrowthType.Negligible} + ${ChangeThreshold.lower} | ${0} | ${GrowthType.Negligible} + ${ChangeThreshold.lower + 0.01} | ${undefined} | ${GrowthType.Negligible} + ${ChangeThreshold.lower + 0.01} | ${EngineerThreshold} | ${GrowthType.Negligible} + ${ChangeThreshold.lower + 0.01} | ${EngineerThreshold + 0.1} | ${GrowthType.Negligible} + ${ChangeThreshold.lower - 0.01} | ${EngineerThreshold - 0.1} | ${GrowthType.Negligible} + ${ChangeThreshold.upper - 0.01} | ${undefined} | ${GrowthType.Negligible} + ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold - 0.1} | ${GrowthType.Negligible} + ${ChangeThreshold.lower} | ${undefined} | ${GrowthType.Savings} + ${ChangeThreshold.lower} | ${EngineerThreshold} | ${GrowthType.Savings} + ${ChangeThreshold.lower - 0.01} | ${undefined} | ${GrowthType.Savings} + ${ChangeThreshold.lower - 0.01} | ${EngineerThreshold} | ${GrowthType.Savings} + ${ChangeThreshold.lower - 0.01} | ${EngineerThreshold + 0.1} | ${GrowthType.Savings} + ${ChangeThreshold.upper} | ${undefined} | ${GrowthType.Excess} + ${ChangeThreshold.upper} | ${EngineerThreshold} | ${GrowthType.Excess} + ${ChangeThreshold.upper + 0.01} | ${undefined} | ${GrowthType.Excess} + ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold} | ${GrowthType.Excess} + ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold + 0.1} | ${GrowthType.Excess} +`( + 'growthOf', + ({ + ratio, + amount, + expected, + }: { + ratio: number; + amount: number; + expected: GrowthType; + }) => { + it(`should display ${GrowthMap[expected]}`, () => { + expect(growthOf(ratio, amount)).toBe(expected); + }); + }, +); diff --git a/plugins/cost-insights/src/utils/change.ts b/plugins/cost-insights/src/utils/change.ts index 1fe3662537..cc797bb3b0 100644 --- a/plugins/cost-insights/src/utils/change.ts +++ b/plugins/cost-insights/src/utils/change.ts @@ -24,14 +24,18 @@ import { } from '../types'; import { aggregationSort } from '../utils/sort'; -// Used by for displaying status colors -export function growthOf(amount: number, ratio: number) { - if (amount >= EngineerThreshold && ratio >= ChangeThreshold.upper) { - return GrowthType.Excess; - } - - if (amount >= EngineerThreshold && ratio <= ChangeThreshold.lower) { - return GrowthType.Savings; +// Used for displaying status colors +export function growthOf(ratio: number, amount?: number) { + if (typeof amount === 'number') { + if (amount >= EngineerThreshold && ratio >= ChangeThreshold.upper) { + return GrowthType.Excess; + } + if (amount >= EngineerThreshold && ratio <= ChangeThreshold.lower) { + return GrowthType.Savings; + } + } else { + if (ratio >= ChangeThreshold.upper) return GrowthType.Excess; + if (ratio <= ChangeThreshold.lower) return GrowthType.Savings; } return GrowthType.Negligible; diff --git a/plugins/cost-insights/src/utils/graphs.ts b/plugins/cost-insights/src/utils/graphs.ts new file mode 100644 index 0000000000..ac06e20587 --- /dev/null +++ b/plugins/cost-insights/src/utils/graphs.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TooltipPayload, TooltipProps } from 'recharts'; +import { AlertCost, DataKey, Entity, ResourceData } from '../types'; +import { + currencyFormatter, + dateFormatter, + lengthyCurrencyFormatter, +} from './formatters'; + +export function formatGraphValue(value: number, format?: string) { + if (format === 'number') { + return value.toLocaleString(); + } + + if (value < 1) { + return lengthyCurrencyFormatter.format(value); + } + + return currencyFormatter.format(value); +} + +export const overviewGraphTickFormatter = (millis: string | number) => + typeof millis === 'number' ? dateFormatter.format(millis) : millis; + +export const tooltipItemOf = (payload: TooltipPayload) => { + const value = + typeof payload.value === 'number' + ? currencyFormatter.format(payload.value) + : (payload.value as string); + const fill = payload.fill as string; + + switch (payload.dataKey) { + case DataKey.Current: + case DataKey.Previous: + return { + label: payload.name, + value: value, + fill: fill, + }; + default: + return null; + } +}; + +export const resourceOf = (entity: Entity | AlertCost): ResourceData => ({ + name: entity.id, + previous: entity.aggregation[0], + current: entity.aggregation[1], +}); + +export const titleOf = (label?: string | number) => { + return label ? String(label) : 'Unlabeled'; +}; + +export const isInvalid = ({ label, payload }: TooltipProps) => { + // null labels are empty strings, which are valid + return label === undefined || !payload || !payload.length; +}; + +export const isActiveLabel = ( + data?: Record<'activeLabel', string | undefined>, +) => { + return data?.activeLabel && data.activeLabel !== ''; +}; diff --git a/plugins/cost-insights/src/utils/graphs.tsx b/plugins/cost-insights/src/utils/graphs.tsx deleted file mode 100644 index 167132cb36..0000000000 --- a/plugins/cost-insights/src/utils/graphs.tsx +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - currencyFormatter, - dateFormatter, - lengthyCurrencyFormatter, -} from './formatters'; - -export function formatGraphValue(value: number, format?: string) { - if (format === 'number') { - return value.toLocaleString(); - } - - if (value < 1) { - return lengthyCurrencyFormatter.format(value); - } - - return currencyFormatter.format(value); -} - -export const overviewGraphTickFormatter = (millis: string | number) => - typeof millis === 'number' ? dateFormatter.format(millis) : millis; diff --git a/plugins/cost-insights/src/utils/mockData.ts b/plugins/cost-insights/src/utils/mockData.ts index 58ba4a4251..ddeb4636b9 100644 --- a/plugins/cost-insights/src/utils/mockData.ts +++ b/plugins/cost-insights/src/utils/mockData.ts @@ -20,7 +20,6 @@ import { Duration, Entity, Product, - ProductCost, ProductFilters, ProjectGrowthData, UnlabeledDataflowAlertProject, @@ -41,6 +40,11 @@ export const createMockEntity = ( const defaultEntity: Entity = { id: 'test-entity', aggregation: [100, 200], + entities: [], + change: { + ratio: 0, + amount: 0, + }, }; if (typeof callback === 'function') { @@ -62,19 +66,6 @@ export const createMockProduct = ( return { ...defaultProduct }; }; -export const createMockProductCost = ( - callback?: mockEntityRenderer, -): ProductCost => { - const defaultProduct: ProductCost = { - entities: [], - aggregation: [0, 0], - }; - if (typeof callback === 'function') { - return callback({ ...defaultProduct }); - } - return { ...defaultProduct }; -}; - export const createMockProjectGrowthData = ( callback?: mockAlertRenderer, ): ProjectGrowthData => { diff --git a/plugins/cost-insights/src/utils/sort.ts b/plugins/cost-insights/src/utils/sort.ts index 944f5fc806..f4fc1987d1 100644 --- a/plugins/cost-insights/src/utils/sort.ts +++ b/plugins/cost-insights/src/utils/sort.ts @@ -19,5 +19,6 @@ export const aggregationSort = ( a: DateAggregation, b: DateAggregation, ): number => a.date.localeCompare(b.date); + export const resourceSort = (a: ResourceData, b: ResourceData) => b.previous + b.current - (a.previous + a.current); diff --git a/plugins/cost-insights/src/utils/styles.ts b/plugins/cost-insights/src/utils/styles.ts index b20c601276..4f8ac7fa98 100644 --- a/plugins/cost-insights/src/utils/styles.ts +++ b/plugins/cost-insights/src/utils/styles.ts @@ -116,6 +116,18 @@ export const useBarChartStyles = (theme: CostInsightsTheme) => ({ }, }); +export const useBarChartLayoutStyles = makeStyles(theme => + createStyles({ + wrapper: { + display: 'flex', + flexDirection: 'column', + }, + legend: { + paddingBottom: theme.spacing(2), + }, + }), +); + export const useBarChartStepperButtonStyles = makeStyles( (theme: CostInsightsTheme) => createStyles({ @@ -146,12 +158,13 @@ export const useBarChartStepperButtonStyles = makeStyles( }), ); -export const useBarChartLabelStyles = makeStyles(() => +export const useBarChartLabelStyles = makeStyles(theme => createStyles({ foreignObject: { textAlign: 'center', }, label: { + fontWeight: theme.typography.fontWeightBold, display: 'block', textDecoration: 'none', overflow: 'hidden', @@ -162,6 +175,11 @@ export const useBarChartLabelStyles = makeStyles(() => marginLeft: 2, fontSize: '1.25em', }, + button: { + textTransform: 'none', + fontWeight: theme.typography.fontWeightBold, + fontSize: theme.typography.fontSize, + }, }), ); @@ -244,6 +262,12 @@ export const useCostGrowthStyles = makeStyles( savings: { color: theme.palette.status.ok, }, + indicator: { + display: 'flex', + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'flex-end', + }, }), ); @@ -356,16 +380,33 @@ export const useTooltipStyles = makeStyles( (theme: CostInsightsTheme) => createStyles({ tooltip: { - padding: theme.spacing(1.5), backgroundColor: theme.palette.tooltip.background, borderRadius: theme.shape.borderRadius, boxShadow: theme.shadows[1], color: theme.palette.tooltip.color, fontSize: theme.typography.fontSize, + width: 250, + }, + actions: { + padding: theme.spacing(2), + }, + header: { + padding: theme.spacing(2), + }, + content: { + padding: theme.spacing(2), }, lensIcon: { fontSize: `.75rem`, }, + divider: { + /* TODO: tooltip divider color should be the inverse of theme. + Dark mode should appear dark, light mode should appear light */ + backgroundColor: darken(theme.palette.divider, 1), + }, + subtitle: { + fontStyle: 'italic', + }, }), ); @@ -446,6 +487,21 @@ export const useProductInsightsCardStyles = makeStyles( }), ); +export const useProductInsightsChartStyles = makeStyles( + (theme: BackstageTheme) => + createStyles({ + indicator: { + fontWeight: theme.typography.fontWeightBold, + fontSize: '1.25rem', + }, + actions: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + }, + }), +); + export const useBackdropStyles = makeStyles( (theme: BackstageTheme) => createStyles({ @@ -463,3 +519,42 @@ export const useSubtleTypographyStyles = makeStyles( }, }), ); + +export const useEntityDialogStyles = makeStyles(theme => + createStyles({ + dialogContent: { + padding: 0, + }, + dialogTitle: { + padding: 0, + }, + closeButton: { + position: 'absolute', + right: theme.spacing(1), + top: theme.spacing(1), + color: theme.palette.grey[500], + zIndex: 100, + }, + row: { + fontSize: theme.typography.fontSize * 1.25, + }, + rowTotal: { + fontWeight: theme.typography.fontWeightBold, + }, + colFirst: { + paddingLeft: theme.spacing(2), + }, + colLast: { + paddingRight: theme.spacing(2), + }, + column: { + fontWeight: theme.typography.fontWeightBold, + }, + growth: { + display: 'flex', + flexDirection: 'row', + alignContent: 'center', + justifyContent: 'flex-end', + }, + }), +); From 9e336a57a585b49cf3028fa366fb3b359e13ef7a Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak <36006588+lewtakm@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:48:56 +0100 Subject: [PATCH 008/259] Feat: update GitHub Pull Request plugin version (#3265) --- packages/app/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index 5cf74b0fbb..107b8c1cda 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -35,7 +35,7 @@ "@material-ui/icons": "^4.9.1", "@octokit/rest": "^18.0.0", "@roadiehq/backstage-plugin-github-insights": "^0.2.12", - "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.1", + "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.2", "@roadiehq/backstage-plugin-travis-ci": "^0.2.7", "history": "^5.0.0", "prop-types": "^15.7.2", diff --git a/yarn.lock b/yarn.lock index c265aecd6a..83b96cebdb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3672,10 +3672,10 @@ react-use "^15.3.3" remark-gfm "^1.0.0" -"@roadiehq/backstage-plugin-github-pull-requests@^0.6.1": - version "0.6.1" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.6.1.tgz#bce3ce93af7b2f6c38d4665c8a0d8a6b2eaa4212" - integrity sha512-3r0BBV6kRlbrlfVA/G7Tc0xbw9A+9roihspCaV/yhhqXgk/ik5v6ftEautlx45+gKaFMp+jgmfKHqFgOdSyhZw== +"@roadiehq/backstage-plugin-github-pull-requests@^0.6.2": + version "0.6.2" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.6.2.tgz#02f4a7a03e1a7dc24342ec695130017ce7d3ad8d" + integrity sha512-OdBWO6NdiBKlol1WlbFQGeHxV2C8wl8EiI2NgA15lozsqFA22kWis6Z16cAHe4ZdRCjihDmqINtAIJzeZZ1gOg== dependencies: "@backstage/catalog-model" "^0.2.0" "@backstage/core" "^0.2.0" From f12b26274c875659fd7ffc57226c1cebcfc05918 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 9 Nov 2020 10:08:24 -0500 Subject: [PATCH 009/259] remove unused import --- .../src/components/ProductInsightsCard/ProductInsightsChart.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx index 489c852f26..1392b05b34 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx @@ -22,7 +22,6 @@ import { } from 'recharts'; import { Box, Typography } from '@material-ui/core'; import { default as FullScreenIcon } from '@material-ui/icons/Fullscreen'; -import { useLastCompleteBillingDate } from '../../hooks'; import { LegendItem } from '../LegendItem'; import { ProductEntityDialog } from './ProductEntityDialog'; import { CostGrowth, CostGrowthIndicator } from '../CostGrowth'; From f059eaf7550e642c8ca9e5ab1d7e1a5617ec9608 Mon Sep 17 00:00:00 2001 From: Sebastian Qvarfordt Date: Mon, 9 Nov 2020 16:28:51 +0100 Subject: [PATCH 010/259] readTree method on UrlReader (#2925) * Basic implementation of readTree on github and use it in techdocs * Added a UrlPreparer and cleaned up some minor things * Download entire archive and filter out the wanted files in readTree * Added test for dir output of tree reader * Fixed formatting * Fixed formatting * Some cleanups and fixes * Fix typing issues * Fix prettier * Fix prettier * Removed unused dependency * Fixed comments on PR * Prettier * Moved @types/fs-extra to devDeps * Fixed another PR comment * Added a test for getDocFilesFromRepository * Prettier --- packages/backend-common/package.json | 10 ++ .../src/reading/GithubUrlReader.test.ts | 80 ++++++++++++ .../src/reading/GithubUrlReader.ts | 93 +++++++++++++- .../src/reading/UrlReaderPredicateMux.ts | 29 ++++- .../src/reading/__fixtures__/repo.tar.gz | Bin 0 -> 232 bytes packages/backend-common/src/reading/index.ts | 2 +- packages/backend-common/src/reading/types.ts | 16 +++ packages/backend/src/plugins/techdocs.ts | 8 +- plugins/techdocs-backend/src/helpers.test.ts | 117 ++++++++++++++++++ plugins/techdocs-backend/src/helpers.ts | 34 ++++- .../techdocs-backend/src/service/helpers.ts | 2 +- .../src/techdocs/stages/prepare/index.ts | 1 + .../src/techdocs/stages/prepare/types.ts | 8 +- .../src/techdocs/stages/prepare/url.ts | 42 +++++++ yarn.lock | 16 ++- 15 files changed, 450 insertions(+), 8 deletions(-) create mode 100644 packages/backend-common/src/reading/__fixtures__/repo.tar.gz create mode 100644 plugins/techdocs-backend/src/helpers.test.ts create mode 100644 plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c004eaac28..f76cb1757a 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -36,11 +36,13 @@ "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "compression": "^1.7.4", + "concat-stream": "^2.0.0", "cors": "^2.8.5", "cross-fetch": "^3.0.6", "express": "^4.17.1", "express-prom-bundle": "^6.1.0", "express-promise-router": "^3.0.3", + "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "helmet": "^4.0.0", "knex": "^0.21.6", @@ -51,6 +53,7 @@ "prom-client": "^12.0.0", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", + "tar": "^6.0.5", "winston": "^3.2.1" }, "peerDependencies": { @@ -64,17 +67,24 @@ "devDependencies": { "@backstage/cli": "^0.2.0", "@types/compression": "^1.7.0", + "@types/concat-stream": "^1.6.0", + "@types/fs-extra": "^9.0.3", "@types/http-errors": "^1.6.3", "@types/minimist": "^1.2.0", + "@types/mock-fs": "^4.13.0", "@types/morgan": "^1.9.0", + "@types/recursive-readdir": "^2.2.0", "@types/stoppable": "^1.1.0", "@types/supertest": "^2.0.8", + "@types/tar": "^4.0.3", "@types/webpack-env": "^1.15.2", "@types/yaml": "^1.9.7", "get-port": "^5.1.1", "http-errors": "^1.7.3", "jest": "^26.0.1", + "mock-fs": "^4.13.0", "msw": "^0.21.2", + "recursive-readdir": "^2.2.2", "supertest": "^4.0.2" }, "files": [ diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 8df464db9f..1e7326af2b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -15,6 +15,9 @@ */ import { ConfigReader } from '@backstage/config'; +import { setupServer } from 'msw/node'; +import { msw } from '@backstage/test-utils'; +import { rest } from 'msw'; import { getApiRequestOptions, getApiUrl, @@ -24,6 +27,10 @@ import { ProviderConfig, readConfig, } from './GithubUrlReader'; +import fs from 'fs'; +import path from 'path'; +import mockfs from 'mock-fs'; +import recursive from 'recursive-readdir'; describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { @@ -230,4 +237,77 @@ describe('GithubUrlReader', () => { ); }); }); + + describe('readTree', () => { + const worker = setupServer(); + + msw.setupDefaultHandlers(worker); + + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://github.com/spotify/mock/archive/repo.tar.gz', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.body(repoBuffer), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const processor = new GithubUrlReader({ + host: 'github.com', + }); + + const response = await processor.readTree( + 'https://github.com/spotify/mock', + 'repo', + ['mkdocs.yml', 'docs'], + ); + + const files = response.files(); + + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('returns a folder path from an archive', async () => { + const processor = new GithubUrlReader({ + host: 'github.com', + }); + + const response = await processor.readTree( + 'https://github.com/spotify/mock', + 'repo', + ['mkdocs.yml', 'docs'], + ); + + mockfs(); + const directory = await response.dir('/tmp/fs'); + + const writtenToDirectory = fs.existsSync(directory); + const paths = await recursive(directory); + mockfs.restore(); + + expect(writtenToDirectory).toBe(true); + expect(paths.sort()).toEqual( + [ + '/tmp/fs/mock-repo/docs/index.md', + '/tmp/fs/mock-repo/mkdocs.yml', + ].sort(), + ); + + worker.resetHandlers(); + }); + }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index d12344ec91..906076a3c2 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -18,7 +18,12 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { ReaderFactory, UrlReader } from './types'; +import { ReaderFactory, ReadTreeResponse, UrlReader, File } from './types'; +import tar from 'tar'; +import fs from 'fs-extra'; +import concatStream from 'concat-stream'; +import path from 'path'; +import os from 'os'; /** * The configuration parameters for a single GitHub API provider. @@ -229,6 +234,92 @@ export class GithubUrlReader implements UrlReader { throw new Error(message); } + private async getRepositoryArchive( + repoUrl: string, + branchName: string, + ): Promise { + return fetch(new URL(`${repoUrl}/archive/${branchName}.tar.gz`).toString()); + } + + private async writeBufferToFile( + filePath: string, + content: Buffer, + ): Promise { + await fs.outputFile(filePath, content.toString()); + } + + async readTree( + repoUrl: string, + branchName: string, + paths: Array, + ): Promise { + const { name: repoName } = parseGitUri(repoUrl); + + const repoArchive = await this.getRepositoryArchive(repoUrl, branchName); + + const files: File[] = []; + return new Promise(resolve => { + const parser = new (tar.Parse as any)({ + filter: (path: string) => + !!paths.filter(file => { + return path.startsWith(`${repoName}-${branchName}/${file}`); + }).length, + onentry: (entry: tar.ReadEntry) => { + if (entry.type === 'Directory') { + entry.resume(); + return; + } + + const contentPromise: Promise = new Promise(res => { + entry.pipe(concatStream(res)); + }); + + files.push({ + path: entry.path, + content: () => contentPromise, + }); + + entry.resume(); + }, + }); + + // @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why. + repoArchive.body?.pipe(parser).on('finish', () => { + resolve({ + files: () => { + return files; + }, + archive: () => { + return new Promise(resolve => + resolve(Buffer.from('Archive is not yet implemented')), + ); + }, + dir: (outDir: string | undefined) => { + const targetDirectory = + outDir || fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-')); + + return new Promise((res, rej) => { + Promise.all( + files.map(async file => { + return this.writeBufferToFile( + `${targetDirectory}/${file.path}`, + await file.content(), + ); + }), + ) + .then(() => { + res(`${targetDirectory}/${repoName}-${branchName}`); + }) + .catch(err => { + rej(err); + }); + }); + }, + }); + }); + }); + } + toString() { const { host, token } = this.config; return `github{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 7654cc8aac..74f94f0295 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { UrlReader, UrlReaderPredicateTuple } from './types'; +import { ReadTreeResponse, UrlReader, UrlReaderPredicateTuple } from './types'; type Options = { // UrlReader to fall back to if no other reader is matched @@ -53,6 +53,33 @@ export class UrlReaderPredicateMux implements UrlReader { throw new Error(`No reader found that could handle '${url}'`); } + readTree( + repoUrl: string, + branchName: string, + paths: Array, + ): Promise { + const parsed = new URL(repoUrl); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + if (reader.readTree) return reader.readTree(repoUrl, branchName, paths); + throw new Error( + `Trying to call readTree on UrlReader which does not support the feature.`, + ); + } + } + + if (this.fallback) { + if (this.fallback.readTree) + return this.fallback.readTree(repoUrl, branchName, paths); + throw new Error( + `Trying to call readTree on UrlReader which does not support the feature.`, + ); + } + + throw new Error(`No reader found that could handle '${repoUrl}'`); + } + toString() { return `predicateMux{readers=${this.readers .map(t => t.reader) diff --git a/packages/backend-common/src/reading/__fixtures__/repo.tar.gz b/packages/backend-common/src/reading/__fixtures__/repo.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..7a8e9902a24232a5f7130637a05304134abc3fec GIT binary patch literal 232 zcmb2|=3oE==C@Zbay1!XSu&B+beL(LRxe2^Idk!eV446aOJ+o+S-o>~??20b^`}#YbLB3%Cll?@@AuK( zf9m={{U?{EUwv`;^MMVwzTe;Tuda?M|Lgvh`?I~>7XNQAm|j0i{htM|=koGBZ|a}E f`_yk@`Q^XBu@$utzd*@``2}nqTB%DJG#D5FAn; + readTree?( + repoUrl: string, + branchName: string, + paths: Array, + ): Promise; }; export type UrlReaderPredicateTuple = { @@ -37,3 +42,14 @@ export type ReaderFactory = (options: { config: Config; logger: Logger; }) => UrlReaderPredicateTuple[]; + +export type File = { + path: string; + content(): Promise; +}; + +export type ReadTreeResponse = { + files(): File[]; + archive(): Promise; + dir(outDir?: string): Promise; +}; diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index dea31c763b..a9d6293c33 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -22,6 +22,7 @@ import { LocalPublish, TechdocsGenerator, CommonGitPreparer, + UrlPreparer, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -30,20 +31,25 @@ export default async function createPlugin({ logger, config, discovery, + reader, }: PluginEnvironment) { const generators = new Generators(); const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); const preparers = new Preparers(); - const commonGitPreparer = new CommonGitPreparer(logger); const directoryPreparer = new DirectoryPreparer(logger); preparers.register('dir', directoryPreparer); + + const commonGitPreparer = new CommonGitPreparer(logger); preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); preparers.register('azure/api', commonGitPreparer); + const urlPreparer = new UrlPreparer(reader, logger); + preparers.register('url', urlPreparer); + const publisher = new LocalPublish(logger); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/helpers.test.ts b/plugins/techdocs-backend/src/helpers.test.ts new file mode 100644 index 0000000000..8d4d69ba26 --- /dev/null +++ b/plugins/techdocs-backend/src/helpers.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getDocFilesFromRepository } from './helpers'; +import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; + +describe('getDocFilesFromRepository', () => { + it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in root', async () => { + class MockUrlReader implements UrlReader { + async read() { + return Buffer.from('mock'); + } + + async readTree(): Promise { + return { + dir: async () => { + return '/tmp/testfolder'; + }, + files: () => { + return []; + }, + archive: async () => { + return Buffer.from(''); + }, + }; + } + } + + const mockEntity: Entity = { + metadata: { + namespace: 'default', + annotations: { + 'backstage.io/techdocs-ref': + 'url:https://github.com/backstage/backstage/blob/master/mkdocs.yml', + }, + name: 'mytestcomponent', + description: 'A component for testing', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + spec: { + type: 'documentation', + lifecycle: 'experimental', + owner: 'testuser', + }, + }; + + const output = await getDocFilesFromRepository( + new MockUrlReader(), + mockEntity, + ); + + expect(output).toBe('/tmp/testfolder/.'); + }); + + it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in a subfolder', async () => { + class MockUrlReader implements UrlReader { + async read() { + return Buffer.from('mock'); + } + + async readTree(): Promise { + return { + dir: async () => { + return '/tmp/testfolder'; + }, + files: () => { + return []; + }, + archive: async () => { + return Buffer.from(''); + }, + }; + } + } + + const mockEntity: Entity = { + metadata: { + namespace: 'default', + annotations: { + 'backstage.io/techdocs-ref': + 'url:https://github.com/backstage/backstage/blob/master/subfolder/mkdocs.yml', + }, + name: 'mytestcomponent', + description: 'A component for testing', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + spec: { + type: 'documentation', + lifecycle: 'experimental', + owner: 'testuser', + }, + }; + + const output = await getDocFilesFromRepository( + new MockUrlReader(), + mockEntity, + ); + + expect(output).toBe('/tmp/testfolder/subfolder'); + }); +}); diff --git a/plugins/techdocs-backend/src/helpers.ts b/plugins/techdocs-backend/src/helpers.ts index 4cbf46516f..f861d4375e 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/plugins/techdocs-backend/src/helpers.ts @@ -22,7 +22,7 @@ import fs from 'fs-extra'; import { getDefaultBranch } from './default-branch'; import { getGitRepoType, getTokenForGitRepo } from './git-auth'; import { Entity } from '@backstage/catalog-model'; -import { InputError } from '@backstage/backend-common'; +import { InputError, UrlReader } from '@backstage/backend-common'; import { RemoteProtocol } from './techdocs/stages/prepare/types'; import { Logger } from 'winston'; @@ -78,6 +78,7 @@ export const getLocationForEntity = ( case 'github': case 'gitlab': case 'azure/api': + case 'url': return { type, target }; case 'dir': if (path.isAbsolute(target)) return { type, target }; @@ -168,3 +169,34 @@ export const getLastCommitTimestamp = async ( return commit.date().getTime(); }; + +export const getDocFilesFromRepository = async ( + reader: UrlReader, + entity: Entity, +): Promise => { + const { target } = parseReferenceAnnotation( + 'backstage.io/techdocs-ref', + entity, + ); + + const { ref, filepath: mkdocsPath } = parseGitUrl(target); + + const docsRootPath = path.dirname(mkdocsPath); + const docsFolderPath = path.join(docsRootPath, 'docs'); + + if (reader.readTree) { + const readTreeResponse = await reader.readTree( + parseGitUrl(target).toString(), + ref, + [mkdocsPath, docsFolderPath], + ); + + const tmpDir = await readTreeResponse.dir(); + + return `${tmpDir}/${docsRootPath}`; + } + + throw new Error( + `No readTree method available on the UrlReader for ${target}`, + ); +}; diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index b308dc28af..3017a68461 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -103,7 +103,7 @@ export class DocsBuilder { const { type, target } = getLocationForEntity(this.entity); // Unless docs are stored locally - const nonAgeCheckTypes = ['dir', 'file']; + const nonAgeCheckTypes = ['dir', 'file', 'url']; if (!nonAgeCheckTypes.includes(type)) { const lastCommit = await getLastCommitTimestamp(target, this.logger); const storageTimeStamp = buildMetadataStorage.getTimestamp(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts index 46e1d84c69..6c73725a3a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts @@ -15,5 +15,6 @@ */ export { DirectoryPreparer } from './dir'; export { CommonGitPreparer } from './commonGit'; +export { UrlPreparer } from './url'; export { Preparers } from './preparers'; export type { PreparerBuilder, PreparerBase } from './types'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts index 1dd091e107..97e75306c3 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts @@ -30,4 +30,10 @@ export type PreparerBuilder = { get(entity: Entity): PreparerBase; }; -export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api'; +export type RemoteProtocol = + | 'dir' + | 'github' + | 'gitlab' + | 'file' + | 'azure/api' + | 'url'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts new file mode 100644 index 0000000000..330db05aa4 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; +import { PreparerBase } from './types'; +import { getDocFilesFromRepository } from '../../../helpers'; + +import { Logger } from 'winston'; +import { UrlReader } from '@backstage/backend-common'; + +export class UrlPreparer implements PreparerBase { + private readonly logger: Logger; + private readonly reader: UrlReader; + + constructor(reader: UrlReader, logger: Logger) { + this.logger = logger; + this.reader = reader; + } + + async prepare(entity: Entity): Promise { + try { + return getDocFilesFromRepository(this.reader, entity); + } catch (error) { + this.logger.debug( + `Unable to fetch files for building docs ${error.message}`, + ); + throw error; + } + } +} diff --git a/yarn.lock b/yarn.lock index 83b96cebdb..391b71d8c9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4916,6 +4916,13 @@ dependencies: "@types/express" "*" +"@types/concat-stream@^1.6.0": + version "1.6.0" + resolved "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz#394dbe0bb5fee46b38d896735e8b68ef2390d00d" + integrity sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0= + dependencies: + "@types/node" "*" + "@types/connect-history-api-fallback@*": version "1.3.3" resolved "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.3.tgz#4772b79b8b53185f0f4c9deab09236baf76ee3b4" @@ -5109,6 +5116,13 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^9.0.3": + version "9.0.3" + resolved "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.3.tgz#9996e5cce993508c32325380b429f04a1327523e" + integrity sha512-NKdGoXLTFTRED3ENcfCsH8+ekV4gbsysanx2OPbstXVV6fZMgUCqTxubs6I9r7pbOJbFgVq1rpFtLURjKCZWUw== + dependencies: + "@types/node" "*" + "@types/git-url-parse@^9.0.0": version "9.0.0" resolved "https://registry.npmjs.org/@types/git-url-parse/-/git-url-parse-9.0.0.tgz#aac1315a44fa4ed5a52c3820f6c3c2fb79cbd12d" @@ -22456,7 +22470,7 @@ tar@^4, tar@^4.4.10, tar@^4.4.12, tar@^4.4.8: safe-buffer "^5.1.2" yallist "^3.0.3" -tar@^6.0.1, tar@^6.0.2: +tar@^6.0.1, tar@^6.0.2, tar@^6.0.5: version "6.0.5" resolved "https://registry.npmjs.org/tar/-/tar-6.0.5.tgz#bde815086e10b39f1dcd298e89d596e1535e200f" integrity sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg== From 2d0bd1be700bfdb172879c62b03391a6f6c0dc34 Mon Sep 17 00:00:00 2001 From: Chad Wilson Date: Mon, 9 Nov 2020 10:52:00 -0500 Subject: [PATCH 011/259] About card edit improvements (#2950) * Update to Edit link in Card component. * prettier * new line * Added changeset * Actually run prettier. . . * Added functionality to determine source of url for icon choice. Fixed failed tests. * Added test case for new edithref attribute * Prettier * forgot a file * correct url substring sanitation * corrected url substring sanitation * Added handling of Bitbucket Cloud and Gitlab. * Added tests for each GitLab and BitBucket edits. * Update to leverage git-url-parse * update test to reflect owner change --- .changeset/thick-lobsters-swim.md | 5 ++ plugins/catalog/package.json | 1 + .../components/AboutCard/AboutCard.test.tsx | 68 +++++++++++++++- .../src/components/AboutCard/AboutCard.tsx | 33 +++++--- .../components/CatalogTable/CatalogTable.tsx | 11 +-- .../catalog/src/components/createEditLink.ts | 77 +++++++++++++++++++ 6 files changed, 175 insertions(+), 20 deletions(-) create mode 100644 .changeset/thick-lobsters-swim.md create mode 100644 plugins/catalog/src/components/createEditLink.ts diff --git a/.changeset/thick-lobsters-swim.md b/.changeset/thick-lobsters-swim.md new file mode 100644 index 0000000000..3a377b7535 --- /dev/null +++ b/.changeset/thick-lobsters-swim.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Improved the edit link to open the component yaml in edit mode in corresponding SCM. Broke out logic for createEditLink to be reused. diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index 6d52c4722f..045230d3f9 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -32,6 +32,7 @@ "@material-ui/lab": "4.0.0-alpha.45", "@types/react": "^16.9", "classnames": "^2.2.6", + "git-url-parse": "^11.4.0", "moment": "^2.26.0", "react": "^16.13.1", "react-dom": "^16.13.1", diff --git a/plugins/catalog/src/components/AboutCard/AboutCard.test.tsx b/plugins/catalog/src/components/AboutCard/AboutCard.test.tsx index 2854af2f26..3038fc3c5c 100644 --- a/plugins/catalog/src/components/AboutCard/AboutCard.test.tsx +++ b/plugins/catalog/src/components/AboutCard/AboutCard.test.tsx @@ -18,7 +18,7 @@ import React from 'react'; import { render } from '@testing-library/react'; import { AboutCard } from './AboutCard'; -describe('', () => { +describe(' GitHub', () => { it('renders info and "view source" link', () => { const entity = { apiVersion: 'v1', @@ -42,5 +42,71 @@ describe('', () => { 'href', 'https://github.com/backstage/backstage/blob/master/software.yaml', ); + expect(getByText('View Source').closest('a')).toHaveAttribute( + 'edithref', + 'https://github.com/backstage/backstage/edit/master/software.yaml', + ); + }); +}); + +describe(' GitLab', () => { + it('renders info and "view source" link', () => { + const entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'software', + annotations: { + 'backstage.io/managed-by-location': + 'gitlab:https://gitlab.com/backstage/backstage/-/blob/master/software.yaml', + }, + }, + spec: { + owner: 'guest', + type: 'service', + lifecycle: 'production', + }, + }; + const { getByText } = render(); + expect(getByText('service')).toBeInTheDocument(); + expect(getByText('View Source').closest('a')).toHaveAttribute( + 'href', + 'https://gitlab.com/backstage/backstage/-/blob/master/software.yaml', + ); + expect(getByText('View Source').closest('a')).toHaveAttribute( + 'edithref', + 'https://gitlab.com/backstage/backstage/-/edit/master/software.yaml', + ); + }); +}); + +describe(' BitBucket', () => { + it('renders info and "view source" link', () => { + const entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'software', + annotations: { + 'backstage.io/managed-by-location': + 'bitbucket:https://bitbucket.org/backstage/backstage/src/master/software.yaml', + }, + }, + spec: { + owner: 'guest', + type: 'service', + lifecycle: 'production', + }, + }; + const { getByText } = render(); + expect(getByText('service')).toBeInTheDocument(); + expect(getByText('View Source').closest('a')).toHaveAttribute( + 'href', + 'https://bitbucket.org/backstage/backstage/src/master/software.yaml', + ); + expect(getByText('View Source').closest('a')).toHaveAttribute( + 'edithref', + 'https://bitbucket.org/backstage/backstage/src/master/software.yaml?mode=edit&spa=0&at=master', + ); }); }); diff --git a/plugins/catalog/src/components/AboutCard/AboutCard.tsx b/plugins/catalog/src/components/AboutCard/AboutCard.tsx index 74877b5a65..8da67d7493 100644 --- a/plugins/catalog/src/components/AboutCard/AboutCard.tsx +++ b/plugins/catalog/src/components/AboutCard/AboutCard.tsx @@ -37,6 +37,8 @@ import EditIcon from '@material-ui/icons/Edit'; import GitHubIcon from '@material-ui/icons/GitHub'; import React from 'react'; import { IconLinkVertical } from './IconLinkVertical'; +import { findLocationForEntityMeta } from '../../data/utils'; +import { createEditLink, determineUrlType } from '../createEditLink'; const useStyles = makeStyles(theme => ({ links: { @@ -79,18 +81,24 @@ const iconMap: Record = { github: , }; -type CodeLinkInfo = { icon?: React.ReactNode; href?: string }; +type CodeLinkInfo = { + icon?: React.ReactNode; + edithref?: string; + href?: string; +}; function getCodeLinkInfo(entity: Entity): CodeLinkInfo { - const location = - entity?.metadata?.annotations?.['backstage.io/managed-by-location']; - + const location = findLocationForEntityMeta(entity?.metadata); if (location) { - // split by first `:` - // e.g. "github:https://github.com/backstage/backstage/blob/master/software.yaml" - const [type, target] = location.split(/:(.+)/); - - return { icon: iconMap[type], href: target }; + const type = + location.type === 'url' + ? determineUrlType(location.target) + : location.type; + return { + icon: iconMap[type], + edithref: createEditLink(location), + href: location.target, + }; } return {}; } @@ -109,7 +117,12 @@ export function AboutCard({ entity, variant }: AboutCardProps) { + { + window.open(codeLink.edithref || '#', '_blank'); + }} + > } diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index 731a8dedc9..15a220e839 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { Table, TableColumn, TableProps } from '@backstage/core'; import { Chip, Link } from '@material-ui/core'; import Edit from '@material-ui/icons/Edit'; @@ -22,6 +22,7 @@ import { Alert } from '@material-ui/lab'; import React from 'react'; import { generatePath, Link as RouterLink } from 'react-router-dom'; import { findLocationForEntityMeta } from '../../data/utils'; +import { createEditLink } from '../createEditLink'; import { useStarredEntities } from '../../hooks/useStarredEntities'; import { entityRoute, entityRouteParams } from '../../routes'; import { @@ -120,14 +121,6 @@ export const CatalogTable = ({ }; }, (rowData: Entity) => { - const createEditLink = (location: LocationSpec): string => { - switch (location.type) { - case 'github': - return location.target.replace('/blob/', '/edit/'); - default: - return location.target; - } - }; const location = findLocationForEntityMeta(rowData.metadata); return { icon: () => , diff --git a/plugins/catalog/src/components/createEditLink.ts b/plugins/catalog/src/components/createEditLink.ts new file mode 100644 index 0000000000..cc04d605f1 --- /dev/null +++ b/plugins/catalog/src/components/createEditLink.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import gitUrlParse from 'git-url-parse'; + +/** + * Creates the edit link for components yaml file + * @see LocationSpec + * @param location The LocationSpec being used to determine entity SCM location + * @returns string representing the edit location based on SCM path + */ + +export const createEditLink = (location: LocationSpec): string | undefined => { + try { + const urlData = gitUrlParse(location.target); + const url = new URL(location.target); + switch (location.type) { + case 'github': + case 'gitlab': + return location.target.replace('/blob/', '/edit/'); + case 'bitbucket': + url.searchParams.set('mode', 'edit'); + url.searchParams.set('spa', '0'); + url.searchParams.set('at', urlData.ref); + return url.toString(); + case 'url': + if ( + urlData.source === 'github.com' || + urlData.source === 'gitlab.com/' + ) { + return location.target.replace('/blob/', '/edit/'); + } else if (urlData.source === 'bitbucket.org') { + url.searchParams.set('mode', 'edit'); + url.searchParams.set('spa', '0'); + url.searchParams.set('at', urlData.ref); + return url.toString(); + } + return location.target; + default: + return location.target; + } + } catch { + return undefined; + } +}; + +/** + * Determines type based on passed in url. This is used to set the icon associated with the type of entity + * @param url + * @returns string representing type of icon to be used + */ +export const determineUrlType = (url: string): string => { + const urlData = gitUrlParse(url); + + if (urlData.source === 'github.com') { + return 'github'; + } else if (urlData.source === 'bitbucket.org') { + return 'bitbucket'; + } else if (urlData.source === 'gitlab.com') { + return 'gitlab'; + } + return 'url'; +}; From 091765a32ed6d05eed29d6ef396050ed33bd46db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 9 Nov 2020 17:55:32 +0100 Subject: [PATCH 012/259] Update help-im-behind-a-corporate-proxy.md (#3266) --- contrib/docs/tutorials/help-im-behind-a-corporate-proxy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docs/tutorials/help-im-behind-a-corporate-proxy.md b/contrib/docs/tutorials/help-im-behind-a-corporate-proxy.md index 4f8351ac15..f5288cf144 100644 --- a/contrib/docs/tutorials/help-im-behind-a-corporate-proxy.md +++ b/contrib/docs/tutorials/help-im-behind-a-corporate-proxy.md @@ -12,7 +12,7 @@ There are however some ways to get this to work without too much effort. It's mo ### Using `global-agent` -1. Install `global-agent` using `yarn install global-agent` +1. Install `global-agent` using `yarn add global-agent` 2. Go to the entry file for the backend (`src/index.ts`) 3. At the top of the file paste the following: @@ -33,7 +33,7 @@ More information and more options for configuring `global-agent` including just `proxy-agent` is a library that you can use to override the `globalAgents` of `node` land with a tunnel to use for each request. -1. Install `proxy-agent` using `yarn install proxy-agent` +1. Install `proxy-agent` using `yarn add proxy-agent` 2. Go to the entry file for the backend (`src/index.ts`) 3. At the top of the file paste the following: From 0c0798f082ae6f443798196d835d727837046477 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Mon, 9 Nov 2020 18:19:47 +0100 Subject: [PATCH 013/259] Persist Table Filters in the API Explorer (#2936) * fix: allow changing the categories of a checkbox tree * feat: allow setting the current selection in the checkbox tree * feat: allow setting the current selection in the select * feat: add a way to access the tables internal state (filters, search, ...) * feat: add useQueryParams hook * feat: persist the table state of the api explorer in the url * Use react-use instead of writing own hooks * Resolve review comments * Rename selectedChilds to selecetedChildren * Add changesets * Support passing a separate state name to useQueryParamState This allows to use the useQueryParamState hook multiple time per route and have a separate state. * refactor: fix typo... --- .changeset/fluffy-dragons-cry.md | 5 + .changeset/odd-parents-push.md | 5 + .changeset/seven-monkeys-run.md | 5 + packages/core/package.json | 1 + .../CheckboxTree/CheckboxTree.stories.tsx | 34 +++- .../components/CheckboxTree/CheckboxTree.tsx | 90 ++++++++-- .../core/src/components/Select/Select.tsx | 38 ++-- .../core/src/components/Table/Filters.tsx | 36 ++-- packages/core/src/components/Table/Table.tsx | 168 ++++++++++++++---- packages/core/src/components/Table/index.ts | 2 +- packages/core/src/hooks/index.ts | 17 ++ packages/core/src/hooks/useQueryParamState.ts | 84 +++++++++ packages/core/src/index.ts | 1 + .../ApiExplorerTable/ApiExplorerTable.tsx | 15 +- 14 files changed, 427 insertions(+), 74 deletions(-) create mode 100644 .changeset/fluffy-dragons-cry.md create mode 100644 .changeset/odd-parents-push.md create mode 100644 .changeset/seven-monkeys-run.md create mode 100644 packages/core/src/hooks/index.ts create mode 100644 packages/core/src/hooks/useQueryParamState.ts diff --git a/.changeset/fluffy-dragons-cry.md b/.changeset/fluffy-dragons-cry.md new file mode 100644 index 0000000000..32a202b417 --- /dev/null +++ b/.changeset/fluffy-dragons-cry.md @@ -0,0 +1,5 @@ +--- +'@backstage/core': patch +--- + +Extend the table to share its current filter state. The filter state can be used together with the new `useQueryParamState` hook to store the current filter state to the browser history and restore it after navigating to other routes. diff --git a/.changeset/odd-parents-push.md b/.changeset/odd-parents-push.md new file mode 100644 index 0000000000..5d997b7a45 --- /dev/null +++ b/.changeset/odd-parents-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-api-docs': patch +--- + +Persist table state of the API Explorer to the browser history. This allows to navigate between pages and come back to the previous filter state. diff --git a/.changeset/seven-monkeys-run.md b/.changeset/seven-monkeys-run.md new file mode 100644 index 0000000000..7e66daa86f --- /dev/null +++ b/.changeset/seven-monkeys-run.md @@ -0,0 +1,5 @@ +--- +'@backstage/core': patch +--- + +Make the selected state of Select and CheckboxTree controllable from outside. diff --git a/packages/core/package.json b/packages/core/package.json index 0ddc77f1e4..aaedf2ca6f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -44,6 +44,7 @@ "d3-shape": "^2.0.0", "d3-zoom": "^2.0.0", "dagre": "^0.8.5", + "qs": "^6.9.4", "immer": "^7.0.9", "lodash": "^4.17.15", "material-table": "^1.69.1", diff --git a/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx b/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx index d5a33f0f9b..db590e6524 100644 --- a/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx +++ b/packages/core/src/components/CheckboxTree/CheckboxTree.stories.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React from 'react'; +import React, { useState } from 'react'; import { CheckboxTree } from '.'; const CHECKBOX_TREE_ITEMS = [ @@ -71,3 +71,35 @@ export const Default = () => ( subCategories={CHECKBOX_TREE_ITEMS} /> ); + +export const DynamicTree = () => { + function generateTree(showMore: boolean = false) { + const t = [ + { + label: 'Show more', + options: [], + }, + ]; + + if (showMore) { + t.push({ + label: 'More', + options: [], + }); + } + + return t; + } + + const [tree, setTree] = useState(generateTree()); + + return ( + { + setTree(generateTree(state.some(c => c.category === 'Show more'))); + }} + label="default" + subCategories={tree} + /> + ); +}; diff --git a/packages/core/src/components/CheckboxTree/CheckboxTree.tsx b/packages/core/src/components/CheckboxTree/CheckboxTree.tsx index 6825704971..d6d3410913 100644 --- a/packages/core/src/components/CheckboxTree/CheckboxTree.tsx +++ b/packages/core/src/components/CheckboxTree/CheckboxTree.tsx @@ -14,20 +14,22 @@ * limitations under the License. */ /* eslint-disable guard-for-in */ -import React, { useEffect, useReducer } from 'react'; -import { createStyles, makeStyles, Theme } from '@material-ui/core/styles'; import { + Checkbox, + Collapse, List, ListItem, ListItemIcon, - Checkbox, ListItemText, - Collapse, Typography, } from '@material-ui/core'; +import { createStyles, makeStyles, Theme } from '@material-ui/core/styles'; import ExpandLess from '@material-ui/icons/ExpandLess'; import ExpandMore from '@material-ui/icons/ExpandMore'; import produce from 'immer'; +import { isEqual } from 'lodash'; +import React, { useEffect, useReducer } from 'react'; +import { usePrevious } from 'react-use'; type IndexedObject = { [key: string]: T; @@ -96,11 +98,14 @@ type Option = { isChecked?: boolean; }; +type Selection = { category?: string; selectedChildren?: string[] }[]; + export type CheckboxTreeProps = { subCategories: SubCategory[]; label: string; triggerReset?: boolean; - onChange: (arg: any) => any; + selected?: Selection; + onChange: (arg: Selection) => any; }; /* REDUCER */ @@ -114,6 +119,11 @@ type Action = | { type: 'checkOption'; payload: checkOptionPayload } | { type: 'checkCategory'; payload: string } | { type: 'toggleCategory'; payload: string } + | { + type: 'updateCategories'; + payload: IndexedObject; + } + | { type: 'updateSelected'; payload: Selection } | { type: 'triggerReset' }; const reducer = ( @@ -157,6 +167,38 @@ const reducer = ( } }); } + case 'updateCategories': { + return produce(state, newState => { + for (const category in newState) { + delete newState[category]; + } + + for (const category in action.payload) { + newState[category] = action.payload[category]; + + if (state[category]) { + newState[category].isChecked = state[category].isChecked; + newState[category].isOpen = state[category].isOpen; + } + } + }); + } + case 'updateSelected': { + return produce(state, newState => { + for (const category in newState) { + const selection = action.payload.find(s => s.category === category); + + if (selection) { + newState[category].isChecked = true; + + for (const option in newState[category].options) { + newState[category].options[option].isChecked = + selection.selectedChildren?.includes(option) || false; + } + } + } + }); + } default: return state; } @@ -183,23 +225,30 @@ const indexer = ( }; }, {}); -export const CheckboxTree = (props: CheckboxTreeProps) => { - const { onChange } = props; +export const CheckboxTree = ({ + subCategories, + label, + selected, + onChange, + triggerReset, +}: CheckboxTreeProps) => { const classes = useStyles(); - const [state, dispatch] = useReducer(reducer, indexer(props.subCategories)); + const [state, dispatch] = useReducer(reducer, indexer(subCategories)); const handleOpen = (event: any, value: any) => { event.stopPropagation(); dispatch({ type: 'toggleCategory', payload: value }); }; + const previousSubCategories = usePrevious(subCategories); + useEffect(() => { const values = Object.values(state).map(category => ({ - category: category.isChecked ? category.label : null, - selectedChilds: Object.values(category.options) + category: category.isChecked ? category.label : undefined, + selectedChildren: Object.values(category.options) .filter(option => option.isChecked) - .map(option => option.value), + .map(option => option.label), })); onChange(values); // eslint-disable-next-line react-hooks/exhaustive-deps @@ -207,11 +256,26 @@ export const CheckboxTree = (props: CheckboxTreeProps) => { useEffect(() => { dispatch({ type: 'triggerReset' }); - }, [props.triggerReset]); + }, [triggerReset]); + + useEffect(() => { + if (selected) { + dispatch({ type: 'updateSelected', payload: selected }); + } + }, [selected]); + + useEffect(() => { + if (!isEqual(subCategories, previousSubCategories)) { + dispatch({ + type: 'updateCategories', + payload: indexer(subCategories), + }); + } + }, [subCategories, previousSubCategories]); return (
- {props.label} + {label} {Object.values(state).map(item => (
diff --git a/packages/core/src/components/Select/Select.tsx b/packages/core/src/components/Select/Select.tsx index 1c691340b2..1e6dba76e0 100644 --- a/packages/core/src/components/Select/Select.tsx +++ b/packages/core/src/components/Select/Select.tsx @@ -94,30 +94,46 @@ type Item = { value: string | number; }; +type Selection = string | string[] | number | number[]; + export type SelectProps = { multiple?: boolean; items: Item[]; label: string; placeholder?: string; - onChange: (arg: any) => any; + selected?: Selection; + onChange: (arg: Selection) => void; triggerReset?: boolean; }; -export const SelectComponent = (props: SelectProps) => { - const { multiple, items, label, placeholder, onChange } = props; +export const SelectComponent = ({ + multiple, + items, + label, + placeholder, + selected, + onChange, + triggerReset, +}: SelectProps) => { const classes = useStyles(); - const [value, setValue] = useState( - multiple ? [] : '', + const [value, setValue] = useState( + selected || (multiple ? [] : ''), ); const [isOpen, setOpen] = useState(false); useEffect(() => { setValue(multiple ? [] : ''); - }, [props.triggerReset, multiple]); + }, [triggerReset, multiple]); + + useEffect(() => { + if (selected !== undefined) { + setValue(selected); + } + }, [selected]); const handleChange = (event: React.ChangeEvent<{ value: unknown }>) => { - setValue(event.target.value as any); - onChange(event.target.value); + setValue(event.target.value as Selection); + onChange(event.target.value as Selection); }; const handleClick = (event: React.ChangeEvent) => { @@ -153,10 +169,10 @@ export const SelectComponent = (props: SelectProps) => { onClick={handleClick} open={isOpen} input={} - renderValue={selected => + renderValue={s => multiple && (value as any[]).length !== 0 ? (
- {(selected as string[]).map(selectedValue => ( + {(s as string[]).map(selectedValue => ( el.value === selectedValue)?.value} label={ @@ -172,7 +188,7 @@ export const SelectComponent = (props: SelectProps) => { {(value as any[]).length === 0 ? placeholder || '' - : items.find(el => el.value === selected)?.label} + : items.find(el => el.value === s)?.label} ) } diff --git a/packages/core/src/components/Table/Filters.tsx b/packages/core/src/components/Table/Filters.tsx index fdff04e478..138fec4736 100644 --- a/packages/core/src/components/Table/Filters.tsx +++ b/packages/core/src/components/Table/Filters.tsx @@ -65,6 +65,7 @@ export type SelectedFilters = { type Props = { filters: Filter[]; + selectedFilters?: SelectedFilters; onChangeFilters: (arg: any) => any; }; @@ -73,21 +74,20 @@ export const Filters = (props: Props) => { const { onChangeFilters } = props; - const [filters, setFilters] = useState(props.filters); - const [selectedFilters, setSelectedFilters] = useState({}); + const [selectedFilters, setSelectedFilters] = useState({ + ...props.selectedFilters, + }); const [reset, triggerReset] = useState(false); // Trigger re-rendering const handleClick = () => { setSelectedFilters({}); - setFilters([...props.filters]); triggerReset(el => !el); }; useEffect(() => { onChangeFilters(selectedFilters); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedFilters]); + }, [selectedFilters, onChangeFilters]); // As material table doesn't provide a way to add a column filter tab we will make our own filter logic return ( @@ -99,29 +99,38 @@ export const Filters = (props: Props) => {
- {filters?.length && - filters.map(filter => + {props.filters?.length && + props.filters.map(filter => filter.type === 'checkbox-tree' ? ( ({ + category: s, + }), + ) + : undefined + } onChange={el => setSelectedFilters({ ...selectedFilters, [filter.element.label]: el .filter( (checkboxFilter: any) => - checkboxFilter.category !== null || - checkboxFilter.selectedChilds.length, + checkboxFilter.category || + checkboxFilter.selectedChildren.length, ) .map((checkboxFilter: any) => - checkboxFilter.category !== null + checkboxFilter.category ? [ - ...checkboxFilter.selectedChilds, + ...checkboxFilter.selectedChildren, checkboxFilter.category, ] - : checkboxFilter.selectedChilds, + : checkboxFilter.selectedChildren, ) .flat(), }) @@ -132,10 +141,11 @@ export const Filters = (props: Props) => { triggerReset={reset} key={filter.element.label} {...(filter.element as SelectProps)} + selected={selectedFilters[filter.element.label]} onChange={el => setSelectedFilters({ ...selectedFilters, - [filter.element.label]: el, + [filter.element.label]: el as any, }) } /> diff --git a/packages/core/src/components/Table/Table.tsx b/packages/core/src/components/Table/Table.tsx index 87ad50e58f..852f4a1547 100644 --- a/packages/core/src/components/Table/Table.tsx +++ b/packages/core/src/components/Table/Table.tsx @@ -16,10 +16,10 @@ import { BackstageTheme } from '@backstage/theme'; import { + IconButton, makeStyles, Typography, useTheme, - IconButton, } from '@material-ui/core'; // Material-table is not using the standard icons available in in material-ui. https://github.com/mbrn/material-table/issues/51 import AddBox from '@material-ui/icons/AddBox'; @@ -37,6 +37,7 @@ import Remove from '@material-ui/icons/Remove'; import SaveAlt from '@material-ui/icons/SaveAlt'; import Search from '@material-ui/icons/Search'; import ViewColumn from '@material-ui/icons/ViewColumn'; +import { isEqual, transform } from 'lodash'; import MTable, { Column, MaterialTableProps, @@ -44,7 +45,13 @@ import MTable, { MTableToolbar, Options, } from 'material-table'; -import React, { forwardRef, useCallback, useEffect, useState } from 'react'; +import React, { + forwardRef, + useCallback, + useEffect, + useRef, + useState, +} from 'react'; import { CheckboxTreeProps } from '../CheckboxTree/CheckboxTree'; import { SelectProps } from '../Select/Select'; import { Filter, Filters, SelectedFilters, Without } from './Filters'; @@ -188,6 +195,20 @@ function convertColumns( }); } +function removeDefaultValues(state: any, defaultState: any): any { + return transform(state, (result, value, key) => { + if (!isEqual(value, defaultState[key])) { + result[key] = value; + } + }); +} + +const defaultInitialState = { + search: '', + filtersOpen: false, + filters: {}, +}; + export interface TableColumn extends Column { highlight?: boolean; width?: string; @@ -198,11 +219,19 @@ export type TableFilter = { type: 'select' | 'multiple-select' | 'checkbox-tree'; }; +export type TableState = { + search?: string; + filtersOpen?: boolean; + filters?: SelectedFilters; +}; + export interface TableProps extends MaterialTableProps { columns: TableColumn[]; subtitle?: string; filters?: TableFilter[]; + initialState?: TableState; + onStateChange?: (state: TableState) => any; } export function Table({ @@ -211,6 +240,8 @@ export function Table({ title, subtitle, filters, + initialState, + onStateChange, ...props }: TableProps) { const headerClasses = useHeaderStyles(); @@ -222,13 +253,43 @@ export function Table({ const theme = useTheme(); - const [filtersOpen, toggleFilters] = useState(false); + const calculatedInitialState = { ...defaultInitialState, ...initialState }; + + const [filtersOpen, toggleFilters] = useState( + calculatedInitialState.filtersOpen, + ); const [selectedFiltersLength, setSelectedFiltersLength] = useState(0); const [tableData, setTableData] = useState(data as any[]); - const [selectedFilters, setSelectedFilters] = useState(); + const [selectedFilters, setSelectedFilters] = useState( + calculatedInitialState.filters, + ); const MTColumns = convertColumns(columns, theme); + const [search, setSearch] = useState(calculatedInitialState.search); + const toolbarRef = useRef(); + + useEffect(() => { + if (toolbarRef.current) { + toolbarRef.current.onSearchChange(search); + } + }, [search, toolbarRef]); + + useEffect(() => { + if (onStateChange) { + const state = removeDefaultValues( + { + search, + filtersOpen, + filters: selectedFilters, + }, + defaultInitialState, + ); + + onStateChange(state); + } + }, [search, filtersOpen, selectedFilters, onStateChange]); + const defaultOptions: Options = { headerStyle: { textTransform: 'uppercase', @@ -248,7 +309,7 @@ export function Table({ } const selectedFiltersArray = Object.values(selectedFilters); - if (selectedFiltersArray.flat().length) { + if (data && selectedFiltersArray.flat().length) { const newData = (data as any[]).filter( el => !!Object.entries(selectedFilters) @@ -279,7 +340,7 @@ export function Table({ const constructFilters = ( filterConfig: TableFilter[], - dataValue: any[], + dataValue: any[] | undefined, ): Filter[] => { const extractDistinctValues = (field: string | keyof T): Set => { const distinctValues = new Set(); @@ -289,15 +350,20 @@ export function Table({ } }; - dataValue.forEach(el => { - const value = extractValueByField(el, getFieldByTitle(field) as string); + if (dataValue) { + dataValue.forEach(el => { + const value = extractValueByField( + el, + getFieldByTitle(field) as string, + ); - if (Array.isArray(value)) { - (value as []).forEach(addValue); - } else { - addValue(value); - } - }); + if (Array.isArray(value)) { + (value as []).forEach(addValue); + } else { + addValue(value); + } + }); + } return distinctValues; }; @@ -335,11 +401,63 @@ export function Table({ })); }; + const Toolbar = useCallback( + toolbarProps => { + const onSearchChanged = (searchText: string) => { + toolbarProps.onSearchChanged(searchText); + setSearch(searchText); + }; + + if (filters?.length) { + return ( +
+
+ toggleFilters(el => !el)} + aria-label="filter list" + > + + + + Filters ({selectedFiltersLength}) + +
+ +
+ ); + } + + return ( + + ); + }, + [ + filters?.length, + selectedFiltersLength, + toggleFilters, + toolbarClasses, + filtersClasses, + setSearch, + toolbarRef, + ], + ); + return (
- {filtersOpen && filters?.length && ( + {filtersOpen && data && filters?.length && ( )} @@ -348,25 +466,7 @@ export function Table({ Header: headerProps => ( ), - Toolbar: toolbarProps => - filters?.length ? ( -
-
- toggleFilters(el => !el)} - aria-label="filter list" - > - - - - Filters ({selectedFiltersLength}) - -
- -
- ) : ( - - ), + Toolbar, }} options={{ ...defaultOptions, ...options }} columns={MTColumns} diff --git a/packages/core/src/components/Table/index.ts b/packages/core/src/components/Table/index.ts index 9cb8478688..83432ffc70 100644 --- a/packages/core/src/components/Table/index.ts +++ b/packages/core/src/components/Table/index.ts @@ -16,4 +16,4 @@ export { SubvalueCell } from './SubvalueCell'; export { Table } from './Table'; -export type { TableColumn, TableFilter, TableProps } from './Table'; +export type { TableColumn, TableFilter, TableProps, TableState } from './Table'; diff --git a/packages/core/src/hooks/index.ts b/packages/core/src/hooks/index.ts new file mode 100644 index 0000000000..a77a0eb8a4 --- /dev/null +++ b/packages/core/src/hooks/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { useQueryParamState } from './useQueryParamState'; diff --git a/packages/core/src/hooks/useQueryParamState.ts b/packages/core/src/hooks/useQueryParamState.ts new file mode 100644 index 0000000000..9317dcd3e1 --- /dev/null +++ b/packages/core/src/hooks/useQueryParamState.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import qs from 'qs'; +import { useState } from 'react'; +import { useLocation, useNavigate } from 'react-router-dom'; +import { useDebounce } from 'react-use'; + +function stringify(queryParams: any): string { + // Even though these setting don't look nice (e.g. escaped brackets), we should keep + // them this way. The current syntax handles all cases, including variable types with + // arrays or strings. + return qs.stringify(queryParams, { + strictNullHandling: true, + }); +} + +function parse(queryString: string): any { + return qs.parse(queryString, { + ignoreQueryPrefix: true, + strictNullHandling: true, + }); +} + +function extractState(queryString: string, stateName: string): any | undefined { + const queryParams = parse(queryString); + + return queryParams[stateName]; +} + +function joinQueryString( + queryString: string, + stateName: string, + state: any, +): string { + const queryParams = { + ...parse(queryString), + [stateName]: state, + }; + return stringify(queryParams); +} + +type SetQueryParams = (params: T) => void; + +export function useQueryParamState( + stateName: string, +): [T | undefined, SetQueryParams] { + const navigate = useNavigate(); + const location = useLocation(); + const [queryParamState, setQueryParamState] = useState( + extractState(location.search, stateName), + ); + + useDebounce( + () => { + const queryString = joinQueryString( + location.search, + stateName, + queryParamState, + ); + + if (location.search !== queryString) { + navigate({ ...location, search: `?${queryString}` }, { replace: true }); + } + }, + 100, + [queryParamState], + ); + + return [queryParamState, setQueryParamState]; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4c0d42bbb2..43168635cb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -19,3 +19,4 @@ export * from '@backstage/core-api'; export * from './api-wrappers'; export * from './components'; export * from './layout'; +export * from './hooks'; diff --git a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx index f3be9f664b..e12d172e41 100644 --- a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx +++ b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx @@ -15,7 +15,14 @@ */ import { ApiEntityV1alpha1, Entity } from '@backstage/catalog-model'; -import { Table, TableFilter, TableColumn, useApi } from '@backstage/core'; +import { + Table, + TableColumn, + TableFilter, + TableState, + useApi, + useQueryParamState, +} from '@backstage/core'; import { Chip, Link } from '@material-ui/core'; import { Alert } from '@material-ui/lab'; import React from 'react'; @@ -120,6 +127,10 @@ export const ApiExplorerTable = ({ loading, error, }: ExplorerTableProps) => { + const [queryParamState, setQueryParamState] = useQueryParamState( + 'apiTable', + ); + if (error) { return (
@@ -142,6 +153,8 @@ export const ApiExplorerTable = ({ }} data={entities} filters={filters} + initialState={queryParamState} + onStateChange={setQueryParamState} /> ); }; From 82eaf047064523efc40072ab9e0c6c1093442723 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 9 Nov 2020 14:38:02 -0500 Subject: [PATCH 014/259] bugfix: display select for 1 or more metrics --- .../src/components/CostOverviewCard/CostOverviewCard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx index 7d55026be8..8f52a0070b 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx @@ -98,7 +98,7 @@ export const CostOverviewCard = ({ /> - {config.metrics.length > 1 && ( + {config.metrics.length && ( Date: Tue, 10 Nov 2020 10:22:48 +0100 Subject: [PATCH 015/259] Move TechDocs project board: Fix workflow --- .github/workflows/techdocs-project-board.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/techdocs-project-board.yml b/.github/workflows/techdocs-project-board.yml index a0691f2a79..d1b6dc3fd5 100644 --- a/.github/workflows/techdocs-project-board.yml +++ b/.github/workflows/techdocs-project-board.yml @@ -1,6 +1,7 @@ name: Automatically add new TechDocs Issues and PRs to the GitHub project board -# Development of TechDocs in Backstage is managed by this Kanban board - https://github.com/backstage/backstage/projects/5 -# New issues with TechDocs in their title or docs-like-code label will be added to the board. +# Development of TechDocs in Backstage is managed by this Kanban board - https://github.com/orgs/backstage/projects/1 +# New issues and PRs with TechDocs in their title or docs-like-code label will be added to the board. +# Caveat: New PRs created from forks will not be added since GitHub actions don't share credentials with forks. on: issues: @@ -23,7 +24,7 @@ jobs: contains(github.event.issue.title, 'techdocs') || contains(github.event.issue.title, 'Techdocs') with: - project: 'https://github.com/backstage/backstage/projects/5' + project: 'https://github.com/orgs/backstage/projects/1' column_name: 'Incoming' - name: Assign new issue to Incoming based on its label. @@ -31,7 +32,7 @@ jobs: if: | contains(github.event.issue.labels.*.name, 'docs-like-code') with: - project: 'https://github.com/backstage/backstage/projects/5' + project: 'https://github.com/orgs/backstage/projects/1' column_name: 'Incoming' - name: Assign new PR to Incoming based on its title. @@ -41,7 +42,7 @@ jobs: contains(github.event.pull_request.title, 'techdocs') || contains(github.event.pull_request.title, 'Techdocs') with: - project: 'https://github.com/backstage/backstage/projects/5' + project: 'https://github.com/orgs/backstage/projects/1' column_name: 'Incoming' - name: Assign new PR to Incoming based on its label. @@ -49,5 +50,5 @@ jobs: if: | contains(github.event.pull_request.labels.*.name, 'docs-like-code') with: - project: 'https://github.com/backstage/backstage/projects/5' + project: 'https://github.com/orgs/backstage/projects/1' column_name: 'Incoming' From 39eef8134942449d1a926c6fd405a5ebaed06891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres?= Date: Tue, 10 Nov 2020 11:12:17 +0000 Subject: [PATCH 016/259] fix: link to examples in Catalog docs (#3271) Fix reference to artist-lookup-component.yaml path in Catalog docs, the new folder structure is missing --- docs/features/software-catalog/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index 6076068504..08e6260e80 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -62,7 +62,7 @@ Users can register new components by going to `/create` and clicking the Backstage expects the full URL to the YAML in your source control. Example: ```bash -https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml ``` _More examples can be found @@ -91,7 +91,7 @@ above example can be added using the following configuration: catalog: locations: - type: url - target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml + target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml ``` More information about catalog configuration can be found From 8dd60357f128f5793a07a368ad706172ddfad26a Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 10 Nov 2020 15:20:20 +0100 Subject: [PATCH 017/259] Add MY_GITHUB_TOKEN for the project board automation workflow --- .github/workflows/techdocs-project-board.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/techdocs-project-board.yml b/.github/workflows/techdocs-project-board.yml index d1b6dc3fd5..b81be4d79c 100644 --- a/.github/workflows/techdocs-project-board.yml +++ b/.github/workflows/techdocs-project-board.yml @@ -11,6 +11,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MY_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: assign_issue_or_pr_to_project: From 99c35b909a364cfd9afcbd7d4e878f643ff89bba Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 10 Nov 2020 15:29:28 +0100 Subject: [PATCH 018/259] =?UTF-8?q?Emojis=20are=20nice=20-=20Fix=20column?= =?UTF-8?q?=20name=20Incoming=20=F0=9F=93=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/techdocs-project-board.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/techdocs-project-board.yml b/.github/workflows/techdocs-project-board.yml index b81be4d79c..c6beb9567c 100644 --- a/.github/workflows/techdocs-project-board.yml +++ b/.github/workflows/techdocs-project-board.yml @@ -10,7 +10,6 @@ on: types: [opened, reopened, labeled, edited] env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MY_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: @@ -26,7 +25,7 @@ jobs: contains(github.event.issue.title, 'Techdocs') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming' + column_name: 'Incoming 📨' - name: Assign new issue to Incoming based on its label. uses: srggrs/assign-one-project-github-action@1.2.0 @@ -34,7 +33,7 @@ jobs: contains(github.event.issue.labels.*.name, 'docs-like-code') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming' + column_name: 'Incoming 📨' - name: Assign new PR to Incoming based on its title. uses: srggrs/assign-one-project-github-action@1.2.0 @@ -44,7 +43,7 @@ jobs: contains(github.event.pull_request.title, 'Techdocs') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming' + column_name: 'Incoming 📨' - name: Assign new PR to Incoming based on its label. uses: srggrs/assign-one-project-github-action@1.2.0 @@ -52,4 +51,4 @@ jobs: contains(github.event.pull_request.labels.*.name, 'docs-like-code') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming' + column_name: 'Incoming 📨' From b4204066f0352e3fa1e976895cde598693e3de29 Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak <36006588+lewtakm@users.noreply.github.com> Date: Tue, 10 Nov 2020 16:02:13 +0100 Subject: [PATCH 019/259] Feat: Add BuildKite CI/CD Plugin to sample app (#3248) * Feat: Add BuildKite CI/CD Plugin to sample app * Feat: Add EntityPage test, bump plugin version * Feat: update package.json * feat: run prettier * Feat: update test code after CR --- app-config.yaml | 6 ++ packages/app/package.json | 1 + .../components/catalog/EntityPage.test.tsx | 62 +++++++++++++++++++ .../app/src/components/catalog/EntityPage.tsx | 8 ++- packages/app/src/plugins.ts | 1 + yarn.lock | 19 ++++++ 6 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 packages/app/src/components/catalog/EntityPage.test.tsx diff --git a/app-config.yaml b/app-config.yaml index 148627dc5a..3d0d093536 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -46,6 +46,12 @@ proxy: X-Api-Key: $env: NEW_RELIC_REST_API_KEY + '/buildkite/api': + target: https://api.buildkite.com/v2/ + headers: + Authorization: + $env: BUILDKITE_TOKEN + organization: name: My Company diff --git a/packages/app/package.json b/packages/app/package.json index 107b8c1cda..54f0dd2879 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -37,6 +37,7 @@ "@roadiehq/backstage-plugin-github-insights": "^0.2.12", "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.2", "@roadiehq/backstage-plugin-travis-ci": "^0.2.7", + "@roadiehq/backstage-plugin-buildkite": "^0.1.2", "history": "^5.0.0", "prop-types": "^15.7.2", "react": "^16.12.0", diff --git a/packages/app/src/components/catalog/EntityPage.test.tsx b/packages/app/src/components/catalog/EntityPage.test.tsx new file mode 100644 index 0000000000..f4dfafc7d3 --- /dev/null +++ b/packages/app/src/components/catalog/EntityPage.test.tsx @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { CICDSwitcher } from './EntityPage'; +import { UrlPatternDiscovery, ApiProvider, ApiRegistry } from '@backstage/core'; +import { + buildKiteApiRef, + BuildKiteApi, +} from '@roadiehq/backstage-plugin-buildkite'; +import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; + +describe('EntityPage Test', () => { + const entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'ExampleComponent', + annotations: { + 'buildkite.com/project-slug': 'exampleProject/examplePipeline', + }, + }, + spec: { + owner: 'guest', + type: 'service', + lifecycle: 'production', + }, + }; + + const discoveryApi = UrlPatternDiscovery.compile('http://exampleapi.com'); + + const apis = ApiRegistry.from([ + [buildKiteApiRef, new BuildKiteApi({ discoveryApi })], + ]); + + describe('CICDSwitcher Test', () => { + it('Should render BuildKite View', async () => { + const renderedComponent = await renderWithEffects( + wrapInTestApp( + + + , + ), + ); + expect( + renderedComponent.getByText(/exampleProject\/examplePipeline/), + ).toBeInTheDocument(); + }); + }); +}); diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index 0a3007237e..73d84ef1ca 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -66,13 +66,19 @@ import { isPluginApplicableToEntity as isPullRequestsAvailable, PullRequestsStatsCard, } from '@roadiehq/backstage-plugin-github-pull-requests'; +import { + Router as BuildKiteRouter, + isPluginApplicableToEntity as isBuildKiteAvailable, +} from '@roadiehq/backstage-plugin-buildkite'; -const CICDSwitcher = ({ entity }: { entity: Entity }) => { +export const CICDSwitcher = ({ entity }: { entity: Entity }) => { // This component is just an example of how you can implement your company's logic in entity page. // You can for example enforce that all components of type 'service' should use GitHubActions switch (true) { case isJenkinsAvailable(entity): return ; + case isBuildKiteAvailable(entity): + return ; case isGitHubActionsAvailable(entity): return ; case isCircleCIAvailable(entity): diff --git a/packages/app/src/plugins.ts b/packages/app/src/plugins.ts index bdc3851318..c61a16e59c 100644 --- a/packages/app/src/plugins.ts +++ b/packages/app/src/plugins.ts @@ -38,3 +38,4 @@ export { plugin as Cloudbuild } from '@backstage/plugin-cloudbuild'; export { plugin as CostInsights } from '@backstage/plugin-cost-insights'; export { plugin as GitHubInsights } from '@roadiehq/backstage-plugin-github-insights'; export { plugin as UserSettings } from '@backstage/plugin-user-settings'; +export { plugin as BuildKite } from '@roadiehq/backstage-plugin-buildkite'; diff --git a/yarn.lock b/yarn.lock index 391b71d8c9..ceb0234e6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3651,6 +3651,25 @@ resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.4.0.tgz#1b5859298bf3f61137d7b05084f058a775d6fd73" integrity sha512-U8F/suzg4MuV+8mK1/ufs0Y6c3O8hc1wnuD2IKoOVJvegGfz5JCafyoyGAW6iyuT1DZBMPzVWEqfiuYPmoE7pw== +"@roadiehq/backstage-plugin-buildkite@^0.1.2": + version "0.1.2" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-buildkite/-/backstage-plugin-buildkite-0.1.2.tgz#2f2b414acc18ed7820abc6fb0c042f00c31d7c9a" + integrity sha512-NO1ogAK6lfh/YUmftqbYn6pkUmMJXk9cfda0YDnG23XlcKIst1d3xO5FggozVtU5IZiJDxxwxOOo4ylhBWrGeQ== + dependencies: + "@backstage/catalog-model" "^0.2.0" + "@backstage/core" "^0.2.0" + "@backstage/plugin-catalog" "^0.2.0" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + moment "^2.27.0" + react "^16.13.1" + react-dom "^16.13.1" + react-lazylog "^4.5.2" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-use "^15.3.3" + "@roadiehq/backstage-plugin-github-insights@^0.2.12": version "0.2.12" resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.12.tgz#aeda6306769f376cf6b5b9d74268b060a0f4e764" From a906f20e7b8070bc58f7807e26e9f9e68753cf31 Mon Sep 17 00:00:00 2001 From: o-farooq Date: Wed, 11 Nov 2020 04:29:45 +1300 Subject: [PATCH 020/259] feat(techradar-plugin) add techradar blip history backend support (#3191) * add techradar entry history data structure * updated names and added changeset * fix docs Co-authored-by: Omer Farooq <17722640+o-farooq@users.noreply.github.com> --- .changeset/wicked-impalas-tan.md | 5 + plugins/tech-radar/src/api.ts | 12 ++- .../src/components/RadarComponent.tsx | 26 ++++- plugins/tech-radar/src/sampleData.ts | 101 +++++++++++++----- plugins/tech-radar/src/utils/types.ts | 8 ++ 5 files changed, 121 insertions(+), 31 deletions(-) create mode 100644 .changeset/wicked-impalas-tan.md diff --git a/.changeset/wicked-impalas-tan.md b/.changeset/wicked-impalas-tan.md new file mode 100644 index 0000000000..8cfe919f5c --- /dev/null +++ b/.changeset/wicked-impalas-tan.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-radar': minor +--- + +Added tech radar blip history backend support and normalized the data structure diff --git a/plugins/tech-radar/src/api.ts b/plugins/tech-radar/src/api.ts index 7a6e790136..0c30c9374c 100644 --- a/plugins/tech-radar/src/api.ts +++ b/plugins/tech-radar/src/api.ts @@ -34,11 +34,17 @@ export interface RadarQuadrant { export interface RadarEntry { key: string; // react key id: string; - moved: MovedState; - quadrant: RadarQuadrant; - ring: RadarRing; + quadrant: string; title: string; url: string; + timeline: Array; +} + +export interface RadarEntrySnapshot { + date: Date; + ringId: string; + description?: string; + moved?: MovedState; } /** diff --git a/plugins/tech-radar/src/components/RadarComponent.tsx b/plugins/tech-radar/src/components/RadarComponent.tsx index 99190202b9..583a7c91e0 100644 --- a/plugins/tech-radar/src/components/RadarComponent.tsx +++ b/plugins/tech-radar/src/components/RadarComponent.tsx @@ -20,6 +20,7 @@ import { useAsync } from 'react-use'; import Radar from '../components/Radar'; import { TechRadarComponentProps, TechRadarLoaderResponse } from '../api'; import getSampleData from '../sampleData'; +import { Entry } from '../utils/types'; const useTechRadarLoader = (props: TechRadarComponentProps) => { const errorApi = useApi(errorApiRef); @@ -47,6 +48,29 @@ const useTechRadarLoader = (props: TechRadarComponentProps) => { const RadarComponent = (props: TechRadarComponentProps): JSX.Element => { const { loading, error, value: data } = useTechRadarLoader(props); + const mapToEntries = ( + loaderResponse: TechRadarLoaderResponse | undefined, + ): Array => { + return loaderResponse!.entries.map(entry => { + return { + id: entry.key, + quadrant: loaderResponse!.quadrants.find(q => q.id === entry.quadrant)!, + title: entry.title, + ring: loaderResponse!.rings.find( + r => r.id === entry.timeline[0].ringId, + )!, + history: entry.timeline.map(e => { + return { + date: e.date, + ring: loaderResponse!.rings.find(a => a.id === e.ringId)!, + description: e.description, + moved: e.moved, + }; + }), + }; + }); + }; + return ( <> {loading && } @@ -55,7 +79,7 @@ const RadarComponent = (props: TechRadarComponentProps): JSX.Element => { {...props} rings={data!.rings} quadrants={data!.quadrants} - entries={data!.entries} + entries={mapToEntries(data)} /> )} diff --git a/plugins/tech-radar/src/sampleData.ts b/plugins/tech-radar/src/sampleData.ts index a9fa7efd7b..914bc30931 100644 --- a/plugins/tech-radar/src/sampleData.ts +++ b/plugins/tech-radar/src/sampleData.ts @@ -35,85 +35,132 @@ quadrants.push({ id: 'process', name: 'Process' }); const entries = new Array(); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + description: + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua', + }, + ], url: '#', key: 'javascript', id: 'javascript', title: 'JavaScript', - quadrant: { id: 'languages', name: 'Languages' }, + quadrant: 'languages', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + description: + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat', + }, + ], url: '#', key: 'typescript', id: 'typescript', title: 'TypeScript', - quadrant: { id: 'languages', name: 'Languages' }, + quadrant: 'languages', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'webpack', id: 'webpack', title: 'Webpack', - quadrant: { id: 'frameworks', name: 'Frameworks' }, + quadrant: 'frameworks', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'react', id: 'react', title: 'React', - quadrant: { id: 'frameworks', name: 'Frameworks' }, + quadrant: 'frameworks', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'code-reviews', id: 'code-reviews', title: 'Code Reviews', - quadrant: { id: 'process', name: 'Process' }, + quadrant: 'process', }); entries.push({ - moved: 0, + timeline: [ + { + moved: 0, + ringId: 'assess', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'mob-programming', id: 'mob-programming', title: 'Mob Programming', - quadrant: { id: 'process', name: 'Process' }, - ring: { id: 'assess', name: 'ASSESS', color: '#fbdb84' }, + quadrant: 'process', }); entries.push({ - moved: 0, + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'docs-like-code', id: 'docs-like-code', title: 'Docs-like-code', - quadrant: { id: 'process', name: 'Process' }, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + quadrant: 'process', }); entries.push({ - moved: 0, + timeline: [ + { + ringId: 'hold', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'force-push', id: 'force-push', title: 'Force push to master', - quadrant: { id: 'process', name: 'Process' }, - ring: { id: 'hold', name: 'HOLD', color: '#93c47d' }, + quadrant: 'process', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + timeline: [ + { + ringId: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'github-actions', id: 'github-actions', title: 'GitHub Actions', - quadrant: { id: 'infrastructure', name: 'Infrastructure' }, + quadrant: 'infrastructure', }); export default function getSampleData(): Promise { diff --git a/plugins/tech-radar/src/utils/types.ts b/plugins/tech-radar/src/utils/types.ts index f3933ee584..3322a67dcb 100644 --- a/plugins/tech-radar/src/utils/types.ts +++ b/plugins/tech-radar/src/utils/types.ts @@ -69,6 +69,14 @@ export type Entry = { // How this entry has recently moved; -1 for "down", +1 for "up", 0 for not moved moved?: MovedState; active?: boolean; + timeline?: Array; +}; + +export type EntrySnapshot = { + date: Date; + ring: Ring; + description?: string; + moved?: MovedState; }; // The same as Entry except quadrant/ring are declared by their string ID instead of being the actual objects From 2fcc82a95cc179c40d702e5e7cf396b295e5cbfa Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Tue, 10 Nov 2020 15:11:35 -0500 Subject: [PATCH 021/259] emphasize divider background use fixed title for project growth alert map product option to entity id in client remove conditional from test name update tooltip verbiage for unlabeled costs add default tooltip tooltip cleanup remove unused palette property --- plugins/cost-insights/src/client.ts | 4 +- .../src/components/BarChart/BarChart.tsx | 23 +++++++++- .../src/components/BarChart/helpers.tsx | 42 ------------------- .../CostGrowth/CostGrowthIndicator.test.tsx | 4 +- .../ProductInsightsChart.tsx | 6 +-- .../ProjectGrowthAlertChart.tsx | 11 +---- .../UnlabeledDataflowAlertCard.tsx | 7 +--- plugins/cost-insights/src/types/Theme.ts | 1 - plugins/cost-insights/src/utils/styles.ts | 5 +-- 9 files changed, 33 insertions(+), 70 deletions(-) delete mode 100644 plugins/cost-insights/src/components/BarChart/helpers.tsx diff --git a/plugins/cost-insights/src/client.ts b/plugins/cost-insights/src/client.ts index 50e925bc28..740a44fcbd 100644 --- a/plugins/cost-insights/src/client.ts +++ b/plugins/cost-insights/src/client.ts @@ -196,7 +196,7 @@ export class ExampleCostInsightsClient implements CostInsightsApi { productInsightsOptions: ProductInsightsOptions, ): Promise { const projectProductInsights = await this.request(productInsightsOptions, { - id: 'project-product', + id: productInsightsOptions.product, aggregation: [80_000, 110_000], change: { ratio: 0.375, @@ -243,7 +243,7 @@ export class ExampleCostInsightsClient implements CostInsightsApi { }); const productInsights: Entity = await this.request(productInsightsOptions, { - id: 'product', + id: productInsightsOptions.product, aggregation: [200_000, 250_000], change: { ratio: 0.2, diff --git a/plugins/cost-insights/src/components/BarChart/BarChart.tsx b/plugins/cost-insights/src/components/BarChart/BarChart.tsx index 30e8ad71f8..93f738a38b 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChart.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChart.tsx @@ -30,6 +30,8 @@ import { import { Box, useTheme } from '@material-ui/core'; import { BarChartTick } from './BarChartTick'; import { BarChartStepper } from './BarChartStepper'; +import { BarChartTooltip } from './BarChartTooltip'; +import { BarChartTooltipItem } from './BarChartTooltipItem'; import { currencyFormatter } from '../../utils/formatters'; import { BarChartData, @@ -37,8 +39,27 @@ import { DataKey, CostInsightsTheme, } from '../../types'; +import { notEmpty } from '../../utils/assert'; import { useBarChartStyles } from '../../utils/styles'; import { resourceSort } from '../../utils/sort'; +import { isInvalid, titleOf, tooltipItemOf } from '../../utils/graphs'; + +export const defaultTooltip: ContentRenderer = ({ + label, + payload = [], +}) => { + if (isInvalid({ label, payload })) return null; + + const title = titleOf(label); + const items = payload.map(tooltipItemOf).filter(notEmpty); + return ( + + {items.map((item, index) => ( + + ))} + + ); +}; export type BarChartProps = { resources: ResourceData[]; @@ -55,7 +76,7 @@ export const BarChart = ({ responsive = true, displayAmount = 6, options = {}, - tooltip, + tooltip = defaultTooltip, onClick, onMouseMove, }: BarChartProps) => { diff --git a/plugins/cost-insights/src/components/BarChart/helpers.tsx b/plugins/cost-insights/src/components/BarChart/helpers.tsx deleted file mode 100644 index 66389af771..0000000000 --- a/plugins/cost-insights/src/components/BarChart/helpers.tsx +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { - ContentRenderer, - TooltipProps as RechartsTooltipProps, -} from 'recharts'; -import { BarChartTooltip } from './BarChartTooltip'; -import { BarChartTooltipItem } from './BarChartTooltipItem'; -import { notEmpty } from '../../utils/assert'; -import { isInvalid, titleOf, tooltipItemOf } from '../../utils/graphs'; - -export const renderDefaultTooltip: ContentRenderer = ({ - label, - payload = [], -}) => { - if (isInvalid({ label, payload })) return null; - - const title = titleOf(label); - const items = payload.map(tooltipItemOf).filter(notEmpty); - return ( - - {items.map((item, index) => ( - - ))} - - ); -}; diff --git a/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx b/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx index 5aabb09c7d..8be5e238cc 100644 --- a/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx +++ b/plugins/cost-insights/src/components/CostGrowth/CostGrowthIndicator.test.tsx @@ -30,9 +30,7 @@ describe.each` ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold} | ${'excess'} ${ChangeThreshold.upper + 0.01} | ${EngineerThreshold + 0.1} | ${'excess'} `('growthOf', ({ ratio, amount, ariaLabel }) => { - it(`should display the correct indicator for ${ - ariaLabel ? ariaLabel : 'negligible' - }`, async () => { + it(`should display the correct indicator for ${ariaLabel}`, async () => { const { getByLabelText } = await renderInTestApp( , ); diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx index 1392b05b34..6050dae9c7 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductInsightsChart.tsx @@ -144,7 +144,7 @@ export const ProductInsightsChart = ({ const activeEntity = findAlways(entity.entities, e => e.id === id); const ratio = activeEntity.change.ratio; const skus = activeEntity.entities; - const subtitle = `${skus.length} ${pluralOf(skus.length, 'SKU', 'SKUs')}`; + const subtitle = `${skus.length} ${pluralOf(skus.length, 'SKU')}`; if (skus.length) { return ( @@ -179,14 +179,14 @@ export const ProductInsightsChart = ({ topRight={ } content={ id ? null - : `This service is generating costs that can't be attributed to any known entity.` + : "This product has costs that are not labeled and therefore can't be attributed to a specific entity." } > {items.map((item, index) => ( diff --git a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx index dffed24992..53e52f18e8 100644 --- a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx +++ b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertChart.tsx @@ -20,7 +20,6 @@ import { Box } from '@material-ui/core'; import { BarChart, BarChartLegend } from '../BarChart'; import { LegendItem } from '../LegendItem'; import { CostGrowth } from '../CostGrowth'; -import { renderDefaultTooltip } from '../BarChart/helpers'; import { BarChartOptions, Duration, ProjectGrowthData } from '../../types'; import { useBarChartLayoutStyles as useStyles } from '../../utils/styles'; import { resourceOf } from '../../utils/graphs'; @@ -46,17 +45,11 @@ export const ProjectGrowthAlertChart = ({ return ( - + - + ); }; diff --git a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx b/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx index 5384db566c..e56b3b044e 100644 --- a/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx +++ b/plugins/cost-insights/src/components/UnlabeledDataflowAlertCard/UnlabeledDataflowAlertCard.tsx @@ -18,7 +18,6 @@ import React from 'react'; import { InfoCard } from '@backstage/core'; import { Box } from '@material-ui/core'; import { BarChart, BarChartLegend } from '../BarChart'; -import { renderDefaultTooltip } from '../BarChart/helpers'; import { UnlabeledDataflowData, ResourceData } from '../../types'; import { pluralOf } from '../../utils/grammar'; import { useBarChartLayoutStyles as useStyles } from '../../utils/styles'; @@ -54,11 +53,7 @@ export const UnlabeledDataflowAlertCard = ({ costEnd={alert.unlabeledCost} options={options} /> - + ); diff --git a/plugins/cost-insights/src/types/Theme.ts b/plugins/cost-insights/src/types/Theme.ts index fecaa17abb..9053283351 100644 --- a/plugins/cost-insights/src/types/Theme.ts +++ b/plugins/cost-insights/src/types/Theme.ts @@ -30,7 +30,6 @@ type CostInsightsPaletteAdditions = { tooltip: CostInsightsTooltipOptions; navigationText: string; alertBackground: string; - textSecondary: string; }; export type CostInsightsPalette = BackstagePalette & diff --git a/plugins/cost-insights/src/utils/styles.ts b/plugins/cost-insights/src/utils/styles.ts index 4f8ac7fa98..de743d8459 100644 --- a/plugins/cost-insights/src/utils/styles.ts +++ b/plugins/cost-insights/src/utils/styles.ts @@ -15,6 +15,7 @@ */ import { createStyles, + emphasize, darken, getLuminance, lighten, @@ -400,9 +401,7 @@ export const useTooltipStyles = makeStyles( fontSize: `.75rem`, }, divider: { - /* TODO: tooltip divider color should be the inverse of theme. - Dark mode should appear dark, light mode should appear light */ - backgroundColor: darken(theme.palette.divider, 1), + backgroundColor: emphasize(theme.palette.divider, 1), }, subtitle: { fontStyle: 'italic', From 0703edee09888cb484f84c81a9b5ba2150a4c901 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Tue, 10 Nov 2020 16:48:27 -0500 Subject: [PATCH 022/259] changeset --- .changeset/cost-insights-sharp-actors-mix.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/cost-insights-sharp-actors-mix.md diff --git a/.changeset/cost-insights-sharp-actors-mix.md b/.changeset/cost-insights-sharp-actors-mix.md new file mode 100644 index 0000000000..5ae1035612 --- /dev/null +++ b/.changeset/cost-insights-sharp-actors-mix.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-cost-insights': minor +--- + +rename: Tooltip -> BarChartTooltip +rename: TooltipItem -> BarChartTooltipItem +Deprecate BarChartData in favor of BarChartOptions +Export BarChartLegend component +Update BarChart props to accept options prop +Deprecate ProductCost type in favor of Entity. Update CostInsightsApi From 9badb8f73731521bb0b1e9eeec43516d6abd1973 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 11 Nov 2020 09:53:26 +0100 Subject: [PATCH 023/259] TechDocs project: try without having an emoji in the name --- .github/workflows/techdocs-project-board.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/techdocs-project-board.yml b/.github/workflows/techdocs-project-board.yml index c6beb9567c..a2fb980259 100644 --- a/.github/workflows/techdocs-project-board.yml +++ b/.github/workflows/techdocs-project-board.yml @@ -25,7 +25,7 @@ jobs: contains(github.event.issue.title, 'Techdocs') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming 📨' + column_name: 'Incoming' - name: Assign new issue to Incoming based on its label. uses: srggrs/assign-one-project-github-action@1.2.0 @@ -33,7 +33,7 @@ jobs: contains(github.event.issue.labels.*.name, 'docs-like-code') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming 📨' + column_name: 'Incoming' - name: Assign new PR to Incoming based on its title. uses: srggrs/assign-one-project-github-action@1.2.0 @@ -43,7 +43,7 @@ jobs: contains(github.event.pull_request.title, 'Techdocs') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming 📨' + column_name: 'Incoming' - name: Assign new PR to Incoming based on its label. uses: srggrs/assign-one-project-github-action@1.2.0 @@ -51,4 +51,4 @@ jobs: contains(github.event.pull_request.labels.*.name, 'docs-like-code') with: project: 'https://github.com/orgs/backstage/projects/1' - column_name: 'Incoming 📨' + column_name: 'Incoming' From c3e8426a91ef8c413e0116490bd193eb36385176 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 11 Nov 2020 13:03:19 +0100 Subject: [PATCH 024/259] Improve sonarqube readme The wrong name here leads to confusion. --- plugins/sonarqube/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/sonarqube/README.md b/plugins/sonarqube/README.md index af18017359..1c8d5a6a0a 100644 --- a/plugins/sonarqube/README.md +++ b/plugins/sonarqube/README.md @@ -77,7 +77,7 @@ sonarQube: 5. Get and provide `SONARQUBE_AUTH_HEADER` as env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/) -6. Add the `sonarqube.org/project-key` annotation to your component-info.yaml file: +6. Add the `sonarqube.org/project-key` annotation to your catalog-info.yaml file: ```yaml apiVersion: backstage.io/v1alpha1 From 4a9d666a329056eea954455215bea93511b1f565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 11 Nov 2020 13:31:08 +0100 Subject: [PATCH 025/259] catalog: more info about refresh errors (#3272) We sometimes get errors like `Cannot read property 'toLowerCase' of undefined` which really doesn't help debugging. So log the error stack instead of just the message, and also make the rules enforcer (which is the very first code that's hit by newly read data) more defensive. The latter is needed because no validation has been performed on the entity data yet, so if the user mistakenly inserted bad data, the code will crash instead of just rejecting the data as invalid. --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 6 +++--- .../catalog-backend/src/ingestion/HigherOrderOperations.ts | 4 ++-- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index e82699f526..b112e32ef5 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -147,10 +147,10 @@ export class CatalogRulesEnforcer { } for (const matcher of matchers) { - if (matcher.type !== location.type) { + if (matcher.type !== location?.type) { continue; } - if (matcher.target && matcher.target !== location.target) { + if (matcher.target && matcher.target !== location?.target) { continue; } return true; @@ -165,7 +165,7 @@ export class CatalogRulesEnforcer { } for (const matcher of matchers) { - if (entity.kind.toLowerCase() !== matcher.kind.toLowerCase()) { + if (entity?.kind?.toLowerCase() !== matcher.kind.toLowerCase()) { continue; } diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index f352e8d3fd..bc277e1767 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -130,7 +130,7 @@ export class HigherOrderOperations implements HigherOrderOperation { await this.locationsCatalog.logUpdateSuccess(location.id, undefined); } catch (e) { this.logger.warn( - `Failed to refresh location ${location.type}:${location.target}, ${e}`, + `Failed to refresh location ${location.type}:${location.target}, ${e.stack}`, ); await this.locationsCatalog.logUpdateFailure(location.id, e); } @@ -152,7 +152,7 @@ export class HigherOrderOperations implements HigherOrderOperation { for (const item of readerOutput.errors) { this.logger.warn( - `Failed item in location ${item.location.type}:${item.location.target}, ${item.error}`, + `Failed item in location ${item.location.type}:${item.location.target}, ${item.error.stack}`, ); } diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 8031b8ff0d..a9248bb3fb 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -97,7 +97,7 @@ export class LocationReaders implements LocationReader { output.errors.push({ location: item.location, error: new Error( - `Entity of kind ${item.entity.kind} is not allowed from location ${item.location.target}:${item.location.type}`, + `Entity of kind ${item.entity.kind} is not allowed from location ${item.location.type} ${item.location.target}`, ), }); } @@ -117,7 +117,7 @@ export class LocationReaders implements LocationReader { items = newItems; } - const message = `Max recursion depth ${MAX_DEPTH} reached for ${location.type} ${location.target}`; + const message = `Max recursion depth ${MAX_DEPTH} reached for location ${location.type} ${location.target}`; logger.warn(message); output.errors.push({ location, error: new Error(message) }); return output; From 84b654d5de3b15c2c521bea723ed679979012599 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 11 Nov 2020 13:34:40 +0100 Subject: [PATCH 026/259] Use dense table style and outlined chips in the API Explorer (#3276) This matches the styling of the catalog. --- .changeset/perfect-boats-attack.md | 5 +++++ .../src/components/ApiExplorerTable/ApiExplorerTable.tsx | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .changeset/perfect-boats-attack.md diff --git a/.changeset/perfect-boats-attack.md b/.changeset/perfect-boats-attack.md new file mode 100644 index 0000000000..51c327127a --- /dev/null +++ b/.changeset/perfect-boats-attack.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-api-docs': patch +--- + +Use dense table style and outlined chips in the API Explorer. diff --git a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx index e12d172e41..9222623aed 100644 --- a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx +++ b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx @@ -90,7 +90,13 @@ const columns: TableColumn[] = [ <> {entity.metadata.tags && entity.metadata.tags.map(t => ( - + ))} ), @@ -149,6 +155,7 @@ export const ApiExplorerTable = ({ paging: false, actionsColumnIndex: -1, loadingType: 'linear', + padding: 'dense', showEmptyDataSourceMessage: !loading, }} data={entities} From 9090739a53b6e60f3474271b57b46ae2720e0a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 11 Nov 2020 14:42:08 +0100 Subject: [PATCH 027/259] storybook: fix broken stories that lack a router parent (#3273) --- .../ProgressBars/GaugeCard.stories.tsx | 23 ++++-- .../src/layout/InfoCard/InfoCard.stories.tsx | 17 +++-- .../core/src/layout/Page/Page.stories.tsx | 53 +++++++------- .../layout/TabbedCard/TabbedCard.stories.tsx | 71 +++++++++++-------- 4 files changed, 94 insertions(+), 70 deletions(-) diff --git a/packages/core/src/components/ProgressBars/GaugeCard.stories.tsx b/packages/core/src/components/ProgressBars/GaugeCard.stories.tsx index 45de49cd2d..982999f8d6 100644 --- a/packages/core/src/components/ProgressBars/GaugeCard.stories.tsx +++ b/packages/core/src/components/ProgressBars/GaugeCard.stories.tsx @@ -14,19 +14,28 @@ * limitations under the License. */ -import React from 'react'; +import React, { PropsWithChildren } from 'react'; import { GaugeCard } from './GaugeCard'; import { Grid } from '@material-ui/core'; +import { MemoryRouter } from 'react-router'; const linkInfo = { title: 'Go to XYZ Location', link: '#' }; +const Wrapper = ({ children }: PropsWithChildren<{}>) => ( + + + {children} + + +); + export default { title: 'Data Display/Progress Card', component: GaugeCard, }; export const Default = () => ( - + @@ -36,11 +45,11 @@ export const Default = () => ( - + ); export const Subhead = () => ( - + @@ -58,11 +67,11 @@ export const Subhead = () => ( progress={0.89} /> - + ); export const LinkInFooter = () => ( - + @@ -72,5 +81,5 @@ export const LinkInFooter = () => ( - + ); diff --git a/packages/core/src/layout/InfoCard/InfoCard.stories.tsx b/packages/core/src/layout/InfoCard/InfoCard.stories.tsx index 4886a38b0f..0c3c523ba0 100644 --- a/packages/core/src/layout/InfoCard/InfoCard.stories.tsx +++ b/packages/core/src/layout/InfoCard/InfoCard.stories.tsx @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { FC } from 'react'; -import { InfoCard } from '.'; import { Grid, Typography } from '@material-ui/core'; +import React, { PropsWithChildren } from 'react'; +import { MemoryRouter } from 'react-router'; +import { InfoCard } from '.'; const linkInfo = { title: 'Go to XYZ Location', link: '#' }; @@ -35,12 +36,14 @@ const text = ( ); -const Wrapper: FC<{}> = ({ children }) => ( - - - {children} +const Wrapper = ({ children }: PropsWithChildren<{}>) => ( + + + + {children} + - + ); export const Default = () => ( diff --git a/packages/core/src/layout/Page/Page.stories.tsx b/packages/core/src/layout/Page/Page.stories.tsx index 1d3aaea412..1fdaf8eced 100644 --- a/packages/core/src/layout/Page/Page.stories.tsx +++ b/packages/core/src/layout/Page/Page.stories.tsx @@ -14,25 +14,26 @@ * limitations under the License. */ +import { Box, Chip, Grid, Link, Typography } from '@material-ui/core'; import React, { useState } from 'react'; +import { MemoryRouter } from 'react-router'; import { - Header, - Page, - HeaderLabel, - ContentHeader, Content, - InfoCard, + ContentHeader, + Header, + HeaderLabel, HeaderTabs, + InfoCard, + Page, } from '../'; import { + GaugeCard, + StatusOK, SupportButton, Table, - StatusOK, TableColumn, - GaugeCard, TrendLine, } from '../../components'; -import { Box, Typography, Link, Chip, Grid } from '@material-ui/core'; export default { title: 'Plugins/Examples', @@ -194,23 +195,25 @@ const ExampleContentHeader = ({ selectedTab }: { selectedTab?: number }) => ( export const PluginWithData = () => { const [selectedTab, setSelectedTab] = useState(2); return ( -
- - - setSelectedTab(index)} - tabs={tabs.map(({ label }, index) => ({ - id: index.toString(), - label, - }))} - /> - - - - - -
+ +
+ + + setSelectedTab(index)} + tabs={tabs.map(({ label }, index) => ({ + id: index.toString(), + label, + }))} + /> + + + + + +
+ ); }; diff --git a/packages/core/src/layout/TabbedCard/TabbedCard.stories.tsx b/packages/core/src/layout/TabbedCard/TabbedCard.stories.tsx index f4af6dcf9f..65c73b190a 100644 --- a/packages/core/src/layout/TabbedCard/TabbedCard.stories.tsx +++ b/packages/core/src/layout/TabbedCard/TabbedCard.stories.tsx @@ -13,12 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { useState } from 'react'; +import React, { useState, PropsWithChildren } from 'react'; import { TabbedCard, CardTab } from '.'; import { Grid } from '@material-ui/core'; +import { MemoryRouter } from 'react-router'; const cardContentStyle = { height: 200, width: 500 }; +const Wrapper = ({ children }: PropsWithChildren<{}>) => ( + {children} +); + export default { title: 'Layout/Tabbed Card', component: TabbedCard, @@ -33,20 +38,22 @@ export default { export const Default = () => { return ( - - -
Some content
- - -
Some content 2
- - -
Some content 3
- - -
Some content 4
- - + + + +
Some content
+ + +
Some content 2
+ + +
Some content 3
+ + +
Some content 4
+ + + ); }; @@ -54,20 +61,22 @@ const linkInfo = { title: 'Go to XYZ Location', link: '#' }; export const WithFooterLink = () => { return ( - - -
Some content
- - -
Some content 2
- - -
Some content 3
- - -
Some content 4
- - + + + +
Some content
+ + +
Some content 2
+ + +
Some content 3
+ + +
Some content 4
+ + + ); }; @@ -78,7 +87,7 @@ export const WithControlledTabValue = () => { setSelectedTab(newSelectedTab); return ( - <> + Selected tab is {selectedTab} {
Some content 4
 - + ); }; From c250dfe2936f1b2f9b608ff7a7c00c91eb98930a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 11 Nov 2020 18:27:27 +0100 Subject: [PATCH 028/259] Update README.md (#3278) --- plugins/sonarqube/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/sonarqube/README.md b/plugins/sonarqube/README.md index 1c8d5a6a0a..28bf196ea5 100644 --- a/plugins/sonarqube/README.md +++ b/plugins/sonarqube/README.md @@ -9,6 +9,8 @@ The SonarQube Plugin displays code statistics from [SonarCloud](https://sonarclo 1. Install the SonarQube Plugin: ```bash +# packages/app + yarn add @backstage/plugin-sonarqube ``` @@ -43,7 +45,7 @@ const OverviewContent = ({ entity }: { entity: Entity }) => ( **SonarCloud** ```yaml -// app-config.yaml +# app-config.yaml proxy: '/sonarqube': @@ -59,7 +61,7 @@ proxy: **SonarQube** ```yaml -// app-config.yaml +# app-config.yaml proxy: '/sonarqube': From 46287639982a046aeeec3e610de73265540fb5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 12 Nov 2020 11:43:35 +0100 Subject: [PATCH 029/259] auth-backend: Encode the OAuth state param using URL safe chars (#3281) --- .changeset/tidy-bobcats-prove.md | 5 +++++ plugins/auth-backend/src/lib/oauth/helpers.ts | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/tidy-bobcats-prove.md diff --git a/.changeset/tidy-bobcats-prove.md b/.changeset/tidy-bobcats-prove.md new file mode 100644 index 0000000000..ad73d547e0 --- /dev/null +++ b/.changeset/tidy-bobcats-prove.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Encode the OAuth state parameter using URL safe chars only, so that providers have an easier time forming the callback URL. diff --git a/plugins/auth-backend/src/lib/oauth/helpers.ts b/plugins/auth-backend/src/lib/oauth/helpers.ts index 9f250a0285..bc6fcf1004 100644 --- a/plugins/auth-backend/src/lib/oauth/helpers.ts +++ b/plugins/auth-backend/src/lib/oauth/helpers.ts @@ -19,7 +19,7 @@ import { OAuthState } from './types'; export const readState = (stateString: string): OAuthState => { const state = Object.fromEntries( - new URLSearchParams(decodeURIComponent(stateString)), + new URLSearchParams(Buffer.from(stateString, 'hex').toString('utf-8')), ); if ( !state.nonce || @@ -40,7 +40,7 @@ export const encodeState = (state: OAuthState): string => { searchParams.append('nonce', state.nonce); searchParams.append('env', state.env); - return encodeURIComponent(searchParams.toString()); + return Buffer.from(searchParams.toString(), 'utf-8').toString('hex'); }; export const verifyNonce = (req: express.Request, providerId: string) => { From 752808090be59a99ee835463dc69c5e81df5e931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 12 Nov 2020 11:48:46 +0100 Subject: [PATCH 030/259] auth-backend: Handle non-7-bit safe characters in the posted message data (#3280) --- .changeset/modern-islands-joke.md | 5 ++ .../src/lib/flow/authFlowHelpers.test.ts | 58 +++++++++++++++---- .../src/lib/flow/authFlowHelpers.ts | 33 ++++++----- 3 files changed, 72 insertions(+), 24 deletions(-) create mode 100644 .changeset/modern-islands-joke.md diff --git a/.changeset/modern-islands-joke.md b/.changeset/modern-islands-joke.md new file mode 100644 index 0000000000..5737c4f762 --- /dev/null +++ b/.changeset/modern-islands-joke.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Handle non-7-bit safe characters in the posted message data diff --git a/plugins/auth-backend/src/lib/flow/authFlowHelpers.test.ts b/plugins/auth-backend/src/lib/flow/authFlowHelpers.test.ts index f42e4c4270..6c49eaf351 100644 --- a/plugins/auth-backend/src/lib/flow/authFlowHelpers.test.ts +++ b/plugins/auth-backend/src/lib/flow/authFlowHelpers.test.ts @@ -15,10 +15,20 @@ */ import express from 'express'; -import { ensuresXRequestedWith, postMessageResponse } from './authFlowHelpers'; +import { + safelyEncodeURIComponent, + ensuresXRequestedWith, + postMessageResponse, +} from './authFlowHelpers'; import { WebMessageResponse } from './types'; describe('oauth helpers', () => { + describe('safelyEncodeURIComponent', () => { + it('encodes all occurrences of single quotes', () => { + expect(safelyEncodeURIComponent("a'ö'b")).toBe('a%27%C3%B6%27b'); + }); + }); + describe('postMessageResponse', () => { const appOrigin = 'http://localhost:3000'; it('should post a message back with payload success', () => { @@ -45,15 +55,12 @@ describe('oauth helpers', () => { }, }, }; - const jsonData = JSON.stringify(data); - const base64Data = Buffer.from(jsonData, 'utf8').toString('base64'); + const encoded = safelyEncodeURIComponent(JSON.stringify(data)); postMessageResponse(mockResponse, appOrigin, data); expect(mockResponse.setHeader).toBeCalledTimes(3); expect(mockResponse.end).toBeCalledTimes(1); - expect(mockResponse.end).toBeCalledWith( - expect.stringContaining(base64Data), - ); + expect(mockResponse.end).toBeCalledWith(expect.stringContaining(encoded)); }); it('should post a message back with payload error', () => { @@ -64,16 +71,47 @@ describe('oauth helpers', () => { const data: WebMessageResponse = { type: 'authorization_response', - error: new Error('Unknown error occured'), + error: new Error('Unknown error occurred'), + }; + const encoded = safelyEncodeURIComponent(JSON.stringify(data)); + + postMessageResponse(mockResponse, appOrigin, data); + expect(mockResponse.setHeader).toBeCalledTimes(3); + expect(mockResponse.end).toBeCalledTimes(1); + expect(mockResponse.end).toBeCalledWith(expect.stringContaining(encoded)); + }); + + it('handles single quotes and unicode chars safely', () => { + const mockResponse = ({ + end: jest.fn().mockReturnThis(), + setHeader: jest.fn().mockReturnThis(), + } as unknown) as express.Response; + + const data: WebMessageResponse = { + type: 'authorization_response', + response: { + providerInfo: { + accessToken: 'ACCESS_TOKEN', + idToken: 'ID_TOKEN', + expiresInSeconds: 10, + scope: 'email', + }, + profile: { + email: 'foo@bar.com', + displayName: "Adam l'Hôpital", + }, + backstageIdentity: { + id: 'a', + idToken: 'a.b.c', + }, + }, }; - const jsonData = JSON.stringify(data); - const base64Data = Buffer.from(jsonData, 'utf8').toString('base64'); postMessageResponse(mockResponse, appOrigin, data); expect(mockResponse.setHeader).toBeCalledTimes(3); expect(mockResponse.end).toBeCalledTimes(1); expect(mockResponse.end).toBeCalledWith( - expect.stringContaining(base64Data), + expect.stringContaining('Adam%20l%27H%C3%B4pital'), ); }); }); diff --git a/plugins/auth-backend/src/lib/flow/authFlowHelpers.ts b/plugins/auth-backend/src/lib/flow/authFlowHelpers.ts index 63d7c28b50..d2a9557a28 100644 --- a/plugins/auth-backend/src/lib/flow/authFlowHelpers.ts +++ b/plugins/auth-backend/src/lib/flow/authFlowHelpers.ts @@ -18,37 +18,42 @@ import express from 'express'; import crypto from 'crypto'; import { WebMessageResponse } from './types'; +export const safelyEncodeURIComponent = (value: string) => { + // Note the g at the end of the regex; all occurrences of single quotes must + // be replaced, which encodeURIComponent does not do itself by default + return encodeURIComponent(value).replace(/'/g, '%27'); +}; + export const postMessageResponse = ( res: express.Response, appOrigin: string, response: WebMessageResponse, ) => { const jsonData = JSON.stringify(response); - const base64Data = Buffer.from(jsonData, 'utf8').toString('base64'); + const base64Data = safelyEncodeURIComponent(jsonData); + const base64Origin = safelyEncodeURIComponent(appOrigin); - res.setHeader('Content-Type', 'text/html'); - res.setHeader('X-Frame-Options', 'sameorigin'); + // NOTE: It is absolutely imperative that we use the safe encoder above, to + // be sure that the js code below does not allow the injection of malicious + // data. // TODO: Make target app origin configurable globally const script = ` - (window.opener || window.parent).postMessage(JSON.parse(atob('${base64Data}')), '${appOrigin}') - window.close() + var json = decodeURIComponent('${base64Data}'); + var origin = decodeURIComponent('${base64Origin}'); + (window.opener || window.parent).postMessage(JSON.parse(json), origin); + window.close(); `; const hash = crypto.createHash('sha256').update(script).digest('base64'); - res.setHeader('Content-Security-Policy', `script-src 'sha256-${hash}'`); - res.end(` - - - - - - `); + res.setHeader('Content-Type', 'text/html'); + res.setHeader('X-Frame-Options', 'sameorigin'); + res.setHeader('Content-Security-Policy', `script-src 'sha256-${hash}'`); + res.end(``); }; export const ensuresXRequestedWith = (req: express.Request) => { const requiredHeader = req.header('X-Requested-With'); - if (!requiredHeader || requiredHeader !== 'XMLHttpRequest') { return false; } From 76a8099029c79ec43dddff05258ff7289370fc4f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Nov 2020 10:52:23 +0000 Subject: [PATCH 031/259] Version Packages --- .changeset/2863.md | 29 ----------- .changeset/3157.md | 16 ------ .changeset/3194.md | 7 --- .changeset/calm-emus-push.md | 5 -- .changeset/cost-insights-quick-suits-brake.md | 5 -- .changeset/cost-insights-sharp-actors-mix.md | 10 ---- .changeset/dry-pillows-carry.md | 6 --- .changeset/fluffy-dragons-cry.md | 5 -- .changeset/lucky-lemons-return.md | 5 -- .changeset/modern-islands-joke.md | 5 -- .changeset/moody-pugs-smoke.md | 5 -- .changeset/odd-parents-push.md | 5 -- .changeset/perfect-boats-attack.md | 5 -- .changeset/seven-monkeys-run.md | 5 -- .changeset/shaggy-bobcats-warn.md | 5 -- .changeset/silent-dodos-repeat.md | 5 -- .changeset/smooth-hairs-breathe.md | 5 -- .changeset/spicy-bananas-help.md | 5 -- .changeset/strong-moose-cough.md | 5 -- .changeset/thick-lobsters-swim.md | 5 -- .changeset/tidy-bobcats-prove.md | 5 -- .changeset/weak-otters-love.md | 16 ------ .changeset/weak-rice-beam.md | 5 -- .changeset/wicked-impalas-tan.md | 5 -- packages/app/CHANGELOG.md | 48 ++++++++++++++++++ packages/app/package.json | 50 +++++++++---------- packages/backend-common/CHANGELOG.md | 6 +++ packages/backend-common/package.json | 2 +- packages/backend/CHANGELOG.md | 13 +++++ packages/backend/package.json | 10 ++-- packages/cli/package.json | 8 +-- packages/core-api/CHANGELOG.md | 33 ++++++++++++ packages/core-api/package.json | 4 +- packages/core/CHANGELOG.md | 35 +++++++++++++ packages/core/package.json | 6 +-- packages/create-app/CHANGELOG.md | 6 +++ packages/create-app/package.json | 34 ++++++------- packages/dev-utils/CHANGELOG.md | 15 ++++++ packages/dev-utils/package.json | 6 +-- packages/theme/CHANGELOG.md | 6 +++ packages/theme/package.json | 2 +- plugins/api-docs/CHANGELOG.md | 20 ++++++++ plugins/api-docs/package.json | 10 ++-- plugins/auth-backend/CHANGELOG.md | 9 ++++ plugins/auth-backend/package.json | 4 +- plugins/catalog/CHANGELOG.md | 19 +++++++ plugins/catalog/package.json | 12 ++--- plugins/circleci/CHANGELOG.md | 17 +++++++ plugins/circleci/package.json | 10 ++-- plugins/cloudbuild/CHANGELOG.md | 17 +++++++ plugins/cloudbuild/package.json | 10 ++-- plugins/cost-insights/CHANGELOG.md | 25 ++++++++++ plugins/cost-insights/package.json | 8 +-- plugins/explore/CHANGELOG.md | 15 ++++++ plugins/explore/package.json | 8 +-- plugins/gcp-projects/CHANGELOG.md | 15 ++++++ plugins/gcp-projects/package.json | 8 +-- plugins/github-actions/CHANGELOG.md | 19 +++++++ plugins/github-actions/package.json | 12 ++--- plugins/gitops-profiles/CHANGELOG.md | 15 ++++++ plugins/gitops-profiles/package.json | 8 +-- plugins/graphiql/CHANGELOG.md | 15 ++++++ plugins/graphiql/package.json | 8 +-- plugins/jenkins/CHANGELOG.md | 21 ++++++++ plugins/jenkins/package.json | 10 ++-- plugins/kubernetes/CHANGELOG.md | 15 ++++++ plugins/kubernetes/package.json | 8 +-- plugins/lighthouse/CHANGELOG.md | 19 +++++++ plugins/lighthouse/package.json | 12 ++--- plugins/newrelic/CHANGELOG.md | 15 ++++++ plugins/newrelic/package.json | 8 +-- plugins/register-component/CHANGELOG.md | 17 +++++++ plugins/register-component/package.json | 10 ++-- plugins/rollbar/CHANGELOG.md | 17 +++++++ plugins/rollbar/package.json | 10 ++-- plugins/scaffolder-backend/CHANGELOG.md | 21 ++++++++ plugins/scaffolder-backend/package.json | 4 +- plugins/scaffolder/CHANGELOG.md | 31 ++++++++++++ plugins/scaffolder/package.json | 10 ++-- plugins/sentry/CHANGELOG.md | 15 ++++++ plugins/sentry/package.json | 8 +-- plugins/sonarqube/CHANGELOG.md | 15 ++++++ plugins/sonarqube/package.json | 8 +-- plugins/tech-radar/CHANGELOG.md | 20 ++++++++ plugins/tech-radar/package.json | 8 +-- plugins/techdocs/CHANGELOG.md | 19 +++++++ plugins/techdocs/package.json | 12 ++--- plugins/user-settings/CHANGELOG.md | 16 ++++++ plugins/user-settings/package.json | 8 +-- plugins/welcome/CHANGELOG.md | 15 ++++++ plugins/welcome/package.json | 8 +-- 91 files changed, 776 insertions(+), 346 deletions(-) delete mode 100644 .changeset/2863.md delete mode 100644 .changeset/3157.md delete mode 100644 .changeset/3194.md delete mode 100644 .changeset/calm-emus-push.md delete mode 100644 .changeset/cost-insights-quick-suits-brake.md delete mode 100644 .changeset/cost-insights-sharp-actors-mix.md delete mode 100644 .changeset/dry-pillows-carry.md delete mode 100644 .changeset/fluffy-dragons-cry.md delete mode 100644 .changeset/lucky-lemons-return.md delete mode 100644 .changeset/modern-islands-joke.md delete mode 100644 .changeset/moody-pugs-smoke.md delete mode 100644 .changeset/odd-parents-push.md delete mode 100644 .changeset/perfect-boats-attack.md delete mode 100644 .changeset/seven-monkeys-run.md delete mode 100644 .changeset/shaggy-bobcats-warn.md delete mode 100644 .changeset/silent-dodos-repeat.md delete mode 100644 .changeset/smooth-hairs-breathe.md delete mode 100644 .changeset/spicy-bananas-help.md delete mode 100644 .changeset/strong-moose-cough.md delete mode 100644 .changeset/thick-lobsters-swim.md delete mode 100644 .changeset/tidy-bobcats-prove.md delete mode 100644 .changeset/weak-otters-love.md delete mode 100644 .changeset/weak-rice-beam.md delete mode 100644 .changeset/wicked-impalas-tan.md diff --git a/.changeset/2863.md b/.changeset/2863.md deleted file mode 100644 index 729e3aab7b..0000000000 --- a/.changeset/2863.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -'@backstage/core-api': patch ---- - -Updated the AuthApi `.create` methods to configure the default scope of the corresponding Auth Api. As a result the -default scope is configurable when overwriting the Core Api in the app. - -``` -GithubAuth.create({ - discoveryApi, - oauthRequestApi, - defaultScopes: ['read:user', 'repo'], -}), -``` - -Replaced redundant CreateOptions of each Auth Api with the OAuthApiCreateOptions type. - -``` -export type OAuthApiCreateOptions = AuthApiCreateOptions & { - oauthRequestApi: OAuthRequestApi; - defaultScopes?: string[]; -}; - -export type AuthApiCreateOptions = { - discoveryApi: DiscoveryApi; - environment?: string; - provider?: AuthProvider & { id: string }; -}; -``` diff --git a/.changeset/3157.md b/.changeset/3157.md deleted file mode 100644 index f15d05c46a..0000000000 --- a/.changeset/3157.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -'@backstage/core': patch ---- - -Adds the MarkdownContent component to render and display Markdown content with the default -[GFM](https://github.github.com/gfm/) (Github flavored Markdown) dialect. - -``` - -``` - -To render the Markdown content with plain [CommonMark](https://commonmark.org/), set the dialect to `common-mark` - -``` - BarChartTooltip -rename: TooltipItem -> BarChartTooltipItem -Deprecate BarChartData in favor of BarChartOptions -Export BarChartLegend component -Update BarChart props to accept options prop -Deprecate ProductCost type in favor of Entity. Update CostInsightsApi diff --git a/.changeset/dry-pillows-carry.md b/.changeset/dry-pillows-carry.md deleted file mode 100644 index 5352e26d2d..0000000000 --- a/.changeset/dry-pillows-carry.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/core': patch ---- - -- The BottomLink is now able to handle with internal routes. -- @backstage/core Link component detect whether it's an external link or not, and render accordingly diff --git a/.changeset/fluffy-dragons-cry.md b/.changeset/fluffy-dragons-cry.md deleted file mode 100644 index 32a202b417..0000000000 --- a/.changeset/fluffy-dragons-cry.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': patch ---- - -Extend the table to share its current filter state. The filter state can be used together with the new `useQueryParamState` hook to store the current filter state to the browser history and restore it after navigating to other routes. diff --git a/.changeset/lucky-lemons-return.md b/.changeset/lucky-lemons-return.md deleted file mode 100644 index 49125d31d4..0000000000 --- a/.changeset/lucky-lemons-return.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Fix missing api-docs plugin registration in app template diff --git a/.changeset/modern-islands-joke.md b/.changeset/modern-islands-joke.md deleted file mode 100644 index 5737c4f762..0000000000 --- a/.changeset/modern-islands-joke.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-auth-backend': patch ---- - -Handle non-7-bit safe characters in the posted message data diff --git a/.changeset/moody-pugs-smoke.md b/.changeset/moody-pugs-smoke.md deleted file mode 100644 index 3092ec4bfe..0000000000 --- a/.changeset/moody-pugs-smoke.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-user-settings': patch ---- - -Export `AuthProviders`, `DefaultProviderSettings` and `ProviderSettingsItem`. diff --git a/.changeset/odd-parents-push.md b/.changeset/odd-parents-push.md deleted file mode 100644 index 5d997b7a45..0000000000 --- a/.changeset/odd-parents-push.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch ---- - -Persist table state of the API Explorer to the browser history. This allows to navigate between pages and come back to the previous filter state. diff --git a/.changeset/perfect-boats-attack.md b/.changeset/perfect-boats-attack.md deleted file mode 100644 index 51c327127a..0000000000 --- a/.changeset/perfect-boats-attack.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch ---- - -Use dense table style and outlined chips in the API Explorer. diff --git a/.changeset/seven-monkeys-run.md b/.changeset/seven-monkeys-run.md deleted file mode 100644 index 7e66daa86f..0000000000 --- a/.changeset/seven-monkeys-run.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': patch ---- - -Make the selected state of Select and CheckboxTree controllable from outside. diff --git a/.changeset/shaggy-bobcats-warn.md b/.changeset/shaggy-bobcats-warn.md deleted file mode 100644 index 048e1a0ff6..0000000000 --- a/.changeset/shaggy-bobcats-warn.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': minor ---- - -New DependencyGraph component added to core package. diff --git a/.changeset/silent-dodos-repeat.md b/.changeset/silent-dodos-repeat.md deleted file mode 100644 index 04978406f3..0000000000 --- a/.changeset/silent-dodos-repeat.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': patch ---- - -Fix divider prop not respected on InfoCard diff --git a/.changeset/smooth-hairs-breathe.md b/.changeset/smooth-hairs-breathe.md deleted file mode 100644 index 2849a1d4b7..0000000000 --- a/.changeset/smooth-hairs-breathe.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-tech-radar': patch ---- - -Make the footer color of the tech-radar work in both light and dark theme. diff --git a/.changeset/spicy-bananas-help.md b/.changeset/spicy-bananas-help.md deleted file mode 100644 index 6d847f4bc3..0000000000 --- a/.changeset/spicy-bananas-help.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch ---- - -Upgrade @kyma-project/asyncapi-react to 0.14.2 diff --git a/.changeset/strong-moose-cough.md b/.changeset/strong-moose-cough.md deleted file mode 100644 index 41a0da7cca..0000000000 --- a/.changeset/strong-moose-cough.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/theme': patch ---- - -Improve styling of outlined chips in dark mode. diff --git a/.changeset/thick-lobsters-swim.md b/.changeset/thick-lobsters-swim.md deleted file mode 100644 index 3a377b7535..0000000000 --- a/.changeset/thick-lobsters-swim.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog': patch ---- - -Improved the edit link to open the component yaml in edit mode in corresponding SCM. Broke out logic for createEditLink to be reused. diff --git a/.changeset/tidy-bobcats-prove.md b/.changeset/tidy-bobcats-prove.md deleted file mode 100644 index ad73d547e0..0000000000 --- a/.changeset/tidy-bobcats-prove.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-auth-backend': patch ---- - -Encode the OAuth state parameter using URL safe chars only, so that providers have an easier time forming the callback URL. diff --git a/.changeset/weak-otters-love.md b/.changeset/weak-otters-love.md deleted file mode 100644 index 2c273e067b..0000000000 --- a/.changeset/weak-otters-love.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -'@backstage/plugin-scaffolder': minor -'@backstage/plugin-scaffolder-backend': minor ---- - -`createRouter` of scaffolder backend will now require additional option as `entityClient` which could be generated by `CatalogEntityClient` in `plugin-scaffolder-backend` package. Here is example to generate `entityClient`. - -```js -import { CatalogEntityClient } from '@backstage/plugin-scaffolder-backend'; -import { SingleHostDiscovery } from '@backstage/backend-common'; - -const discovery = SingleHostDiscovery.fromConfig(config); -const entityClient = new CatalogEntityClient({ discovery }); -``` - -- Scaffolder's API `/v1/jobs` will accept `templateName` instead of `template` Entity. diff --git a/.changeset/weak-rice-beam.md b/.changeset/weak-rice-beam.md deleted file mode 100644 index c507e350a9..0000000000 --- a/.changeset/weak-rice-beam.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-common': patch ---- - -Capture plugin name under the /api/ prefix for http metrics diff --git a/.changeset/wicked-impalas-tan.md b/.changeset/wicked-impalas-tan.md deleted file mode 100644 index 8cfe919f5c..0000000000 --- a/.changeset/wicked-impalas-tan.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-tech-radar': minor ---- - -Added tech radar blip history backend support and normalized the data structure diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index 458f5a5e5c..e24e7393ec 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,53 @@ # example-app +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [a41730c6e] +- Updated dependencies [9a294574c] +- Updated dependencies [0703edee0] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [5a2705de2] +- Updated dependencies [0c0798f08] +- Updated dependencies [84b654d5d] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [3f05616bf] +- Updated dependencies [803527bd3] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] +- Updated dependencies [59166e5ec] +- Updated dependencies [a906f20e7] + - @backstage/core@0.3.0 + - @backstage/plugin-jenkins@0.3.0 + - @backstage/plugin-cost-insights@0.3.0 + - @backstage/plugin-user-settings@0.2.1 + - @backstage/plugin-api-docs@0.2.1 + - @backstage/plugin-tech-radar@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + - @backstage/plugin-scaffolder@0.3.0 + - @backstage/plugin-circleci@0.2.1 + - @backstage/plugin-cloudbuild@0.2.1 + - @backstage/plugin-explore@0.2.1 + - @backstage/plugin-gcp-projects@0.2.1 + - @backstage/plugin-github-actions@0.2.1 + - @backstage/plugin-gitops-profiles@0.2.1 + - @backstage/plugin-graphiql@0.2.1 + - @backstage/plugin-kubernetes@0.2.1 + - @backstage/plugin-lighthouse@0.2.1 + - @backstage/plugin-newrelic@0.2.1 + - @backstage/plugin-register-component@0.2.1 + - @backstage/plugin-rollbar@0.2.1 + - @backstage/plugin-sentry@0.2.1 + - @backstage/plugin-techdocs@0.2.1 + - @backstage/plugin-welcome@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/packages/app/package.json b/packages/app/package.json index 54f0dd2879..f013200fbb 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,36 +1,36 @@ { "name": "example-app", - "version": "0.2.0", + "version": "0.2.1", "private": true, "bundled": true, "dependencies": { "@backstage/catalog-model": "^0.2.0", "@backstage/cli": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/plugin-api-docs": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/plugin-circleci": "^0.2.0", - "@backstage/plugin-cloudbuild": "^0.2.0", - "@backstage/plugin-cost-insights": "^0.2.0", - "@backstage/plugin-explore": "^0.2.0", - "@backstage/plugin-gcp-projects": "^0.2.0", - "@backstage/plugin-github-actions": "^0.2.0", - "@backstage/plugin-gitops-profiles": "^0.2.0", - "@backstage/plugin-graphiql": "^0.2.0", - "@backstage/plugin-jenkins": "^0.2.0", - "@backstage/plugin-kubernetes": "^0.2.0", - "@backstage/plugin-lighthouse": "^0.2.0", - "@backstage/plugin-newrelic": "^0.2.0", - "@backstage/plugin-register-component": "^0.2.0", - "@backstage/plugin-rollbar": "^0.2.0", - "@backstage/plugin-scaffolder": "^0.2.0", - "@backstage/plugin-sentry": "^0.2.0", - "@backstage/plugin-tech-radar": "^0.2.0", - "@backstage/plugin-techdocs": "^0.2.0", - "@backstage/plugin-user-settings": "^0.2.0", - "@backstage/plugin-welcome": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/plugin-api-docs": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/plugin-circleci": "^0.2.1", + "@backstage/plugin-cloudbuild": "^0.2.1", + "@backstage/plugin-cost-insights": "^0.3.0", + "@backstage/plugin-explore": "^0.2.1", + "@backstage/plugin-gcp-projects": "^0.2.1", + "@backstage/plugin-github-actions": "^0.2.1", + "@backstage/plugin-gitops-profiles": "^0.2.1", + "@backstage/plugin-graphiql": "^0.2.1", + "@backstage/plugin-jenkins": "^0.3.0", + "@backstage/plugin-kubernetes": "^0.2.1", + "@backstage/plugin-lighthouse": "^0.2.1", + "@backstage/plugin-newrelic": "^0.2.1", + "@backstage/plugin-register-component": "^0.2.1", + "@backstage/plugin-rollbar": "^0.2.1", + "@backstage/plugin-scaffolder": "^0.3.0", + "@backstage/plugin-sentry": "^0.2.1", + "@backstage/plugin-tech-radar": "^0.3.0", + "@backstage/plugin-techdocs": "^0.2.1", + "@backstage/plugin-user-settings": "^0.2.1", + "@backstage/plugin-welcome": "^0.2.1", "@backstage/test-utils": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@octokit/rest": "^18.0.0", diff --git a/packages/backend-common/CHANGELOG.md b/packages/backend-common/CHANGELOG.md index 2b8b38b9b0..0f18908932 100644 --- a/packages/backend-common/CHANGELOG.md +++ b/packages/backend-common/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/backend-common +## 0.2.1 + +### Patch Changes + +- 33b7300eb: Capture plugin name under the /api/ prefix for http metrics + ## 0.2.0 ### Minor Changes diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index f76cb1757a..67e4ffe11f 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "private": false, diff --git a/packages/backend/CHANGELOG.md b/packages/backend/CHANGELOG.md index a3e5535e40..2fe72cb101 100644 --- a/packages/backend/CHANGELOG.md +++ b/packages/backend/CHANGELOG.md @@ -1,5 +1,18 @@ # example-backend +## 0.2.1 + +### Patch Changes + +- Updated dependencies [752808090] +- Updated dependencies [462876399] +- Updated dependencies [59166e5ec] +- Updated dependencies [33b7300eb] + - @backstage/plugin-auth-backend@0.2.1 + - @backstage/plugin-scaffolder-backend@0.3.0 + - @backstage/backend-common@0.2.1 + - example-app@0.2.1 + ## 0.2.0 ### Patch Changes diff --git a/packages/backend/package.json b/packages/backend/package.json index 051b83a146..e67cadb44e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "example-backend", - "version": "0.2.0", + "version": "0.2.1", "main": "dist/index.cjs.js", "types": "src/index.ts", "private": true, @@ -18,24 +18,24 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.2.1", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@backstage/plugin-app-backend": "^0.2.0", - "@backstage/plugin-auth-backend": "^0.2.0", + "@backstage/plugin-auth-backend": "^0.2.1", "@backstage/plugin-catalog-backend": "^0.2.0", "@backstage/plugin-graphql-backend": "^0.1.2", "@backstage/plugin-kubernetes-backend": "^0.1.2", "@backstage/plugin-proxy-backend": "^0.2.0", "@backstage/plugin-rollbar-backend": "^0.1.2", - "@backstage/plugin-scaffolder-backend": "^0.2.0", + "@backstage/plugin-scaffolder-backend": "^0.3.0", "@backstage/plugin-sentry-backend": "^0.1.2", "@backstage/plugin-techdocs-backend": "^0.2.0", "@gitbeaker/node": "^25.2.0", "@octokit/rest": "^18.0.0", "azure-devops-node-api": "^10.1.1", "dockerode": "^3.2.0", - "example-app": "^0.2.0", + "example-app": "^0.2.1", "express": "^4.17.1", "express-promise-router": "^3.0.3", "knex": "^0.21.6", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7ae244866d..317e78ca16 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -108,12 +108,12 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.2.1", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/core": "^0.3.0", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@types/diff": "^4.0.2", "@types/fs-extra": "^9.0.1", "@types/html-webpack-plugin": "^3.2.2", diff --git a/packages/core-api/CHANGELOG.md b/packages/core-api/CHANGELOG.md index 0e84d14aea..b936715c70 100644 --- a/packages/core-api/CHANGELOG.md +++ b/packages/core-api/CHANGELOG.md @@ -1,5 +1,38 @@ # @backstage/core-api +## 0.2.1 + +### Patch Changes + +- c5bab94ab: Updated the AuthApi `.create` methods to configure the default scope of the corresponding Auth Api. As a result the + default scope is configurable when overwriting the Core Api in the app. + + ``` + GithubAuth.create({ + discoveryApi, + oauthRequestApi, + defaultScopes: ['read:user', 'repo'], + }), + ``` + + Replaced redundant CreateOptions of each Auth Api with the OAuthApiCreateOptions type. + + ``` + export type OAuthApiCreateOptions = AuthApiCreateOptions & { + oauthRequestApi: OAuthRequestApi; + defaultScopes?: string[]; + }; + + export type AuthApiCreateOptions = { + discoveryApi: DiscoveryApi; + environment?: string; + provider?: AuthProvider & { id: string }; + }; + ``` + +- Updated dependencies [4577e377b] + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/packages/core-api/package.json b/packages/core-api/package.json index 42ba1c2080..98eaa0fa61 100644 --- a/packages/core-api/package.json +++ b/packages/core-api/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core-api", "description": "Internal Core API used by Backstage plugins and apps", - "version": "0.2.0", + "version": "0.2.1", "private": false, "publishConfig": { "access": "public", @@ -31,7 +31,7 @@ "dependencies": { "@backstage/config": "^0.1.1", "@backstage/test-utils": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@types/react": "^16.9", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 10110a7b24..435b344a6a 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,40 @@ # @backstage/core +## 0.3.0 + +### Minor Changes + +- 199237d2f: New DependencyGraph component added to core package. + +### Patch Changes + +- 7b37d65fd: Adds the MarkdownContent component to render and display Markdown content with the default + [GFM](https://github.github.com/gfm/) (Github flavored Markdown) dialect. + + ``` + + ``` + + To render the Markdown content with plain [CommonMark](https://commonmark.org/), set the dialect to `common-mark` + + ``` + BarChartTooltip + rename: TooltipItem -> BarChartTooltipItem + Deprecate BarChartData in favor of BarChartOptions + Export BarChartLegend component + Update BarChart props to accept options prop + Deprecate ProductCost type in favor of Entity. Update CostInsightsApi + +### Patch Changes + +- 9a294574c: Fix styling issue on Cost Insights product panels with no cost +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/cost-insights/package.json b/plugins/cost-insights/package.json index 944dbf587f..39159c03c4 100644 --- a/plugins/cost-insights/package.json +++ b/plugins/cost-insights/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-cost-insights", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,9 +22,9 @@ }, "dependencies": { "@backstage/config": "^0.1.1", - "@backstage/core": "^0.2.0", + "@backstage/core": "^0.3.0", "@backstage/test-utils": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -47,7 +47,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/explore/CHANGELOG.md b/plugins/explore/CHANGELOG.md index dfa428b62a..e4072fd0f2 100644 --- a/plugins/explore/CHANGELOG.md +++ b/plugins/explore/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-explore +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/explore/package.json b/plugins/explore/package.json index a1f71de6a2..7770341ffb 100644 --- a/plugins/explore/package.json +++ b/plugins/explore/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-explore", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -34,7 +34,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/gcp-projects/CHANGELOG.md b/plugins/gcp-projects/CHANGELOG.md index 08e1962f3c..6884ecceee 100644 --- a/plugins/gcp-projects/CHANGELOG.md +++ b/plugins/gcp-projects/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-gcp-projects +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/gcp-projects/package.json b/plugins/gcp-projects/package.json index 812c3b1051..e4af2bd4a3 100644 --- a/plugins/gcp-projects/package.json +++ b/plugins/gcp-projects/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-gcp-projects", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,8 +20,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -32,7 +32,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/github-actions/CHANGELOG.md b/plugins/github-actions/CHANGELOG.md index a4b69a07d4..b679d6844b 100644 --- a/plugins/github-actions/CHANGELOG.md +++ b/plugins/github-actions/CHANGELOG.md @@ -1,5 +1,24 @@ # @backstage/plugin-github-actions +## 0.2.1 + +### Patch Changes + +- Updated dependencies [c5bab94ab] +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core-api@0.2.1 + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index 8d10760843..e0db90453a 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-github-actions", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,10 +22,10 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/core-api": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/core-api": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -41,7 +41,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/gitops-profiles/CHANGELOG.md b/plugins/gitops-profiles/CHANGELOG.md index a5d5249737..7aa4253a2e 100644 --- a/plugins/gitops-profiles/CHANGELOG.md +++ b/plugins/gitops-profiles/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-gitops-profiles +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/gitops-profiles/package.json b/plugins/gitops-profiles/package.json index b61c2f6620..167970f830 100644 --- a/plugins/gitops-profiles/package.json +++ b/plugins/gitops-profiles/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-gitops-profiles", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -33,7 +33,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/graphiql/CHANGELOG.md b/plugins/graphiql/CHANGELOG.md index db59e7d369..19c9e95988 100644 --- a/plugins/graphiql/CHANGELOG.md +++ b/plugins/graphiql/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-graphiql +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/graphiql/package.json b/plugins/graphiql/package.json index e346133e3d..be20cfb635 100644 --- a/plugins/graphiql/package.json +++ b/plugins/graphiql/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-graphiql", "description": "Backstage plugin for browsing GraphQL APIs", - "version": "0.2.0", + "version": "0.2.1", "private": false, "publishConfig": { "access": "public", @@ -31,8 +31,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -44,7 +44,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/jenkins/CHANGELOG.md b/plugins/jenkins/CHANGELOG.md index 010471d9b2..44febbc63d 100644 --- a/plugins/jenkins/CHANGELOG.md +++ b/plugins/jenkins/CHANGELOG.md @@ -1,5 +1,26 @@ # @backstage/plugin-jenkins +## 0.3.0 + +### Minor Changes + +- a41730c6e: Add tooltip for Jenkins rerun button + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/jenkins/package.json b/plugins/jenkins/package.json index d0304376fd..b4f1d25199 100644 --- a/plugins/jenkins/package.json +++ b/plugins/jenkins/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-jenkins", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,9 +22,9 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -37,7 +37,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/kubernetes/CHANGELOG.md b/plugins/kubernetes/CHANGELOG.md index 0ec2c5dbaf..e5611e4953 100644 --- a/plugins/kubernetes/CHANGELOG.md +++ b/plugins/kubernetes/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-kubernetes +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/kubernetes/package.json b/plugins/kubernetes/package.json index 29193e27ed..c1457944de 100644 --- a/plugins/kubernetes/package.json +++ b/plugins/kubernetes/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-kubernetes", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,9 +22,9 @@ "dependencies": { "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.2.0", + "@backstage/core": "^0.3.0", "@backstage/plugin-kubernetes-backend": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@kubernetes/client-node": "^0.12.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -36,7 +36,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/lighthouse/CHANGELOG.md b/plugins/lighthouse/CHANGELOG.md index 9a68b78a92..3c5e94c476 100644 --- a/plugins/lighthouse/CHANGELOG.md +++ b/plugins/lighthouse/CHANGELOG.md @@ -1,5 +1,24 @@ # @backstage/plugin-lighthouse +## 0.2.1 + +### Patch Changes + +- Updated dependencies [c5bab94ab] +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core-api@0.2.1 + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 27b6a21066..9389c84856 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-lighthouse", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -23,10 +23,10 @@ "dependencies": { "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.2.0", - "@backstage/core-api": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/core-api": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -40,7 +40,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/newrelic/CHANGELOG.md b/plugins/newrelic/CHANGELOG.md index d007328180..86b71d497e 100644 --- a/plugins/newrelic/CHANGELOG.md +++ b/plugins/newrelic/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-newrelic +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/newrelic/package.json b/plugins/newrelic/package.json index e7d000abc5..475f19d651 100644 --- a/plugins/newrelic/package.json +++ b/plugins/newrelic/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-newrelic", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -32,7 +32,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/register-component/CHANGELOG.md b/plugins/register-component/CHANGELOG.md index cb1a24d131..3e966ef0d0 100644 --- a/plugins/register-component/CHANGELOG.md +++ b/plugins/register-component/CHANGELOG.md @@ -1,5 +1,22 @@ # @backstage/plugin-register-component +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/register-component/package.json b/plugins/register-component/package.json index 095469f6b4..9c87f33e29 100644 --- a/plugins/register-component/package.json +++ b/plugins/register-component/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-register-component", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,9 +22,9 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -37,7 +37,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/rollbar/CHANGELOG.md b/plugins/rollbar/CHANGELOG.md index ee25fb10d9..3d53c1bf0d 100644 --- a/plugins/rollbar/CHANGELOG.md +++ b/plugins/rollbar/CHANGELOG.md @@ -1,5 +1,22 @@ # @backstage/plugin-rollbar +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/rollbar/package.json b/plugins/rollbar/package.json index 03e393aa1b..722fe35fb6 100644 --- a/plugins/rollbar/package.json +++ b/plugins/rollbar/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-rollbar", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,9 +22,9 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -38,7 +38,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/scaffolder-backend/CHANGELOG.md b/plugins/scaffolder-backend/CHANGELOG.md index 78a6d9527b..96f4ef75f7 100644 --- a/plugins/scaffolder-backend/CHANGELOG.md +++ b/plugins/scaffolder-backend/CHANGELOG.md @@ -1,5 +1,26 @@ # @backstage/plugin-scaffolder-backend +## 0.3.0 + +### Minor Changes + +- 59166e5ec: `createRouter` of scaffolder backend will now require additional option as `entityClient` which could be generated by `CatalogEntityClient` in `plugin-scaffolder-backend` package. Here is example to generate `entityClient`. + + ```js + import { CatalogEntityClient } from '@backstage/plugin-scaffolder-backend'; + import { SingleHostDiscovery } from '@backstage/backend-common'; + + const discovery = SingleHostDiscovery.fromConfig(config); + const entityClient = new CatalogEntityClient({ discovery }); + ``` + + - Scaffolder's API `/v1/jobs` will accept `templateName` instead of `template` Entity. + +### Patch Changes + +- Updated dependencies [33b7300eb] + - @backstage/backend-common@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index ac30e23828..4bed7ea60c 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder-backend", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.2.1", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@gitbeaker/core": "^25.2.0", diff --git a/plugins/scaffolder/CHANGELOG.md b/plugins/scaffolder/CHANGELOG.md index 9c09366174..9eac595997 100644 --- a/plugins/scaffolder/CHANGELOG.md +++ b/plugins/scaffolder/CHANGELOG.md @@ -1,5 +1,36 @@ # @backstage/plugin-scaffolder +## 0.3.0 + +### Minor Changes + +- 59166e5ec: `createRouter` of scaffolder backend will now require additional option as `entityClient` which could be generated by `CatalogEntityClient` in `plugin-scaffolder-backend` package. Here is example to generate `entityClient`. + + ```js + import { CatalogEntityClient } from '@backstage/plugin-scaffolder-backend'; + import { SingleHostDiscovery } from '@backstage/backend-common'; + + const discovery = SingleHostDiscovery.fromConfig(config); + const entityClient = new CatalogEntityClient({ discovery }); + ``` + + - Scaffolder's API `/v1/jobs` will accept `templateName` instead of `template` Entity. + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 738734ceb7..a423c64aa3 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,9 +22,9 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/plugin-catalog": "^0.2.1", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -42,7 +42,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/sentry/CHANGELOG.md b/plugins/sentry/CHANGELOG.md index 06380c39d9..88adfe43a1 100644 --- a/plugins/sentry/CHANGELOG.md +++ b/plugins/sentry/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-sentry +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index 2ede8fd709..7941df5dfd 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sentry", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -37,7 +37,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/sonarqube/CHANGELOG.md b/plugins/sonarqube/CHANGELOG.md index 62f88596e7..0296bd0416 100644 --- a/plugins/sonarqube/CHANGELOG.md +++ b/plugins/sonarqube/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-sonarqube +## 0.1.3 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.1.2 ### Patch Changes diff --git a/plugins/sonarqube/package.json b/plugins/sonarqube/package.json index bc41213ad8..d7240aabde 100644 --- a/plugins/sonarqube/package.json +++ b/plugins/sonarqube/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sonarqube", - "version": "0.1.2", + "version": "0.1.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -36,7 +36,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/tech-radar/CHANGELOG.md b/plugins/tech-radar/CHANGELOG.md index 7f9f11a702..cf6f360fe6 100644 --- a/plugins/tech-radar/CHANGELOG.md +++ b/plugins/tech-radar/CHANGELOG.md @@ -1,5 +1,25 @@ # @backstage/plugin-tech-radar +## 0.3.0 + +### Minor Changes + +- a906f20e7: Added tech radar blip history backend support and normalized the data structure + +### Patch Changes + +- 3f05616bf: Make the footer color of the tech-radar work in both light and dark theme. +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/tech-radar/package.json b/plugins/tech-radar/package.json index 875d6d161b..431fd3d515 100644 --- a/plugins/tech-radar/package.json +++ b/plugins/tech-radar/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-tech-radar", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,9 +21,9 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.2.0", + "@backstage/core": "^0.3.0", "@backstage/test-utils": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -36,7 +36,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/techdocs/CHANGELOG.md b/plugins/techdocs/CHANGELOG.md index 387e4a8d90..1c1dcb5c8a 100644 --- a/plugins/techdocs/CHANGELOG.md +++ b/plugins/techdocs/CHANGELOG.md @@ -1,5 +1,24 @@ # @backstage/plugin-techdocs +## 0.2.1 + +### Patch Changes + +- Updated dependencies [c5bab94ab] +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] +- Updated dependencies [2d0bd1be7] + - @backstage/core-api@0.2.1 + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + - @backstage/plugin-catalog@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index ce0700c1a8..6a664a83fb 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -23,11 +23,11 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.2.0", - "@backstage/core-api": "^0.2.0", - "@backstage/plugin-catalog": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/core-api": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.1", "@backstage/test-utils": "^0.1.2", - "@backstage/theme": "^0.2.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -41,7 +41,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/user-settings/CHANGELOG.md b/plugins/user-settings/CHANGELOG.md index 2522a3af03..729d2a669a 100644 --- a/plugins/user-settings/CHANGELOG.md +++ b/plugins/user-settings/CHANGELOG.md @@ -1,5 +1,21 @@ # @backstage/plugin-user-settings +## 0.2.1 + +### Patch Changes + +- 5a2705de2: Export `AuthProviders`, `DefaultProviderSettings` and `ProviderSettingsItem`. +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index 2f098eae2e..a48c99cbc6 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-user-settings", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -33,7 +33,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/plugins/welcome/CHANGELOG.md b/plugins/welcome/CHANGELOG.md index 2932d41d75..e6165ef4f9 100644 --- a/plugins/welcome/CHANGELOG.md +++ b/plugins/welcome/CHANGELOG.md @@ -1,5 +1,20 @@ # @backstage/plugin-welcome +## 0.2.1 + +### Patch Changes + +- Updated dependencies [7b37d65fd] +- Updated dependencies [4aca74e08] +- Updated dependencies [e8f69ba93] +- Updated dependencies [0c0798f08] +- Updated dependencies [0c0798f08] +- Updated dependencies [199237d2f] +- Updated dependencies [6627b626f] +- Updated dependencies [4577e377b] + - @backstage/core@0.3.0 + - @backstage/theme@0.2.1 + ## 0.2.0 ### Minor Changes diff --git a/plugins/welcome/package.json b/plugins/welcome/package.json index 148b5734ee..fcfbed2af7 100644 --- a/plugins/welcome/package.json +++ b/plugins/welcome/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-welcome", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -21,8 +21,8 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.2.0", - "@backstage/theme": "^0.2.0", + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", @@ -33,7 +33,7 @@ }, "devDependencies": { "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.2", + "@backstage/dev-utils": "^0.1.3", "@backstage/test-utils": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", From 1ef651471d7efa991adf90c68edba2c0d35a60d9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 12:25:39 +0100 Subject: [PATCH 032/259] package: add override of core for roadie plugins --- package.json | 3 +++ yarn.lock | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/package.json b/package.json index 505ee0caad..b97462114c 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,9 @@ "plugins/*" ] }, + "resolutions": { + "**/@roadiehq/backstage-plugin-*/@backstage/core": "0.3.0" + }, "version": "1.0.0", "devDependencies": { "@changesets/cli": "^2.11.0", diff --git a/yarn.lock b/yarn.lock index ceb0234e6b..10e70f52fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1289,6 +1289,42 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" +"@backstage/core@^0.2.0": + version "0.3.0" + dependencies: + "@backstage/config" "^0.1.1" + "@backstage/core-api" "^0.2.1" + "@backstage/theme" "^0.2.1" + "@material-ui/core" "^4.11.0" + "@material-ui/icons" "^4.9.1" + "@material-ui/lab" "4.0.0-alpha.45" + "@types/dagre" "^0.7.44" + "@types/react" "^16.9" + "@types/react-sparklines" "^1.7.0" + classnames "^2.2.6" + clsx "^1.1.0" + d3-selection "^2.0.0" + d3-shape "^2.0.0" + d3-zoom "^2.0.0" + dagre "^0.8.5" + immer "^7.0.9" + lodash "^4.17.15" + material-table "^1.69.1" + prop-types "^15.7.2" + qs "^6.9.4" + rc-progress "^3.0.0" + react "^16.12.0" + react-dom "^16.12.0" + react-helmet "6.1.0" + react-hook-form "^6.6.0" + react-markdown "^5.0.2" + react-router "6.0.0-beta.0" + react-router-dom "6.0.0-beta.0" + react-sparklines "^1.7.0" + react-syntax-highlighter "^13.5.1" + react-use "^15.3.3" + remark-gfm "^1.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" From e74422b4d844b43172b76216fa5e79b7be528dd8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 10:37:04 +0100 Subject: [PATCH 033/259] backend-common: make ReadTreeResponse.files async --- packages/backend-common/src/reading/GithubUrlReader.test.ts | 2 +- packages/backend-common/src/reading/GithubUrlReader.ts | 2 +- packages/backend-common/src/reading/types.ts | 2 +- plugins/techdocs-backend/src/helpers.test.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 1e7326af2b..7552547924 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -272,7 +272,7 @@ describe('GithubUrlReader', () => { ['mkdocs.yml', 'docs'], ); - const files = response.files(); + const files = await response.files(); const mkDocsFile = await files[0].content(); const indexMarkdownFile = await files[1].content(); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 906076a3c2..41db541f7f 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -286,7 +286,7 @@ export class GithubUrlReader implements UrlReader { // @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why. repoArchive.body?.pipe(parser).on('finish', () => { resolve({ - files: () => { + files: async () => { return files; }, archive: () => { diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 26ac4af45e..d9fa5d839e 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -49,7 +49,7 @@ export type File = { }; export type ReadTreeResponse = { - files(): File[]; + files(): Promise; archive(): Promise; dir(outDir?: string): Promise; }; diff --git a/plugins/techdocs-backend/src/helpers.test.ts b/plugins/techdocs-backend/src/helpers.test.ts index 8d4d69ba26..c3f2261637 100644 --- a/plugins/techdocs-backend/src/helpers.test.ts +++ b/plugins/techdocs-backend/src/helpers.test.ts @@ -30,7 +30,7 @@ describe('getDocFilesFromRepository', () => { dir: async () => { return '/tmp/testfolder'; }, - files: () => { + files: async () => { return []; }, archive: async () => { @@ -78,7 +78,7 @@ describe('getDocFilesFromRepository', () => { dir: async () => { return '/tmp/testfolder'; }, - files: () => { + files: async () => { return []; }, archive: async () => { From 1b632b5aa71f695a3d265be6ec6327c4158babb5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 10 Nov 2020 10:47:24 +0100 Subject: [PATCH 034/259] backend-common: add archive tree reading utilities --- .../src/reading/tree/ArchiveResponse.test.ts | 93 +++++++++++++ .../src/reading/tree/ArchiveResponse.ts | 124 ++++++++++++++++++ .../reading/tree/ReadTreeResponseFactory.ts | 47 +++++++ .../backend-common/src/reading/tree/index.ts | 17 +++ 4 files changed, 281 insertions(+) create mode 100644 packages/backend-common/src/reading/tree/ArchiveResponse.test.ts create mode 100644 packages/backend-common/src/reading/tree/ArchiveResponse.ts create mode 100644 packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts create mode 100644 packages/backend-common/src/reading/tree/index.ts diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts new file mode 100644 index 0000000000..57a490650c --- /dev/null +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { Readable } from 'stream'; +import { resolve as resolvePath } from 'path'; +import { ArchiveResponse } from './ArchiveResponse'; + +const archiveData = fs.readFileSync( + resolvePath(__filename, '../../__fixtures__/repo.tar.gz'), +); + +describe('ArchiveResponse', () => { + beforeEach(() => { + mockFs({ + '/test-archive.tar.gz': archiveData, + '/tmp': mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should read as archive and files', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo', '/tmp'); + const buffer = await res.archive(); + + await expect(res.archive()).rejects.toThrow( + 'Response has already been read', + ); + + const res2 = new ArchiveResponse(Readable.from(buffer), '', '/tmp'); + const files = await res2.files(); + + expect(files).toEqual([ + { + path: './mkdocs.yml', + content: expect.any(Function), + }, + { + path: './docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + + it('should extract entire archive into directory', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, '', '/tmp'); + const dir = await res.dir(); + + await expect( + fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); + + it('should extract archive into directory with a subpath', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/docs', '/tmp'); + const dir = await res.dir(); + + await expect( + fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); +}); diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.ts new file mode 100644 index 0000000000..65ad8c18fe --- /dev/null +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import tar, { Parse, ParseStream, ReadEntry } from 'tar'; +import path from 'path'; +import fs from 'fs-extra'; +import { Readable, pipeline as pipelineCb } from 'stream'; +import { promisify } from 'util'; +import concatStream from 'concat-stream'; +import { ReadTreeResponse, File } from '../types'; + +// Tar types for `Parse` is not a proper constructor, but it should be +const TarParseStream = (Parse as unknown) as { new (): ParseStream }; + +const pipeline = promisify(pipelineCb); + +/** + * Wraps a tar archive stream into a tree response reader. + */ +export class ArchiveResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: Readable, + private readonly subPath: string, + private readonly workDir: string = os.tmpdir(), + ) {} + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + async files(): Promise { + this.onlyOnce(); + + const files: File[] = []; + const parser = new TarParseStream(); + + parser.on('entry', (entry: ReadEntry & Readable) => { + if (entry.type === 'Directory') { + entry.resume(); + return; + } + + const contentPromise = new Promise(async resolve => { + await pipeline(entry, concatStream(resolve)); + }); + + files.push({ + path: entry.path, + content: () => contentPromise, + }); + + entry.resume(); + }); + + await pipeline(this.stream, parser); + + return files; + } + + async archive(): Promise { + if (!this.subPath) { + this.onlyOnce(); + + return new Promise(resolve => + pipeline(this.stream, concatStream(resolve)), + ); + } + + // TODO(Rugvip): method for repacking a tar with a subpath is to simply extract into a + // tmp dir and recreate the archive. Would be nicer to stream things instead. + const tmpDir = await this.dir(); + + try { + return await new Promise(async resolve => { + await pipeline( + tar.create({ cwd: tmpDir }, ['.']), + concatStream(resolve), + ); + }); + } finally { + await fs.remove(tmpDir); + } + } + + async dir(outDir?: string): Promise { + this.onlyOnce(); + + const dir = + outDir ?? (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); + + const strip = this.subPath ? this.subPath.split('/').length : 0; + + await pipeline( + this.stream, + tar.extract({ + strip, + cwd: dir, + filter: path => path.startsWith(this.subPath), + }), + ); + + return dir; + } +} diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts new file mode 100644 index 0000000000..0cdf926720 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import { Readable } from 'stream'; +import { Config } from '@backstage/config'; +import { ReadTreeResponse } from '../types'; +import { ArchiveResponse } from './ArchiveResponse'; + +type FromArchiveOptions = { + // A binary stream of a tar archive. + stream: Readable; + // If set, root of the tree will be set to the given path. Should not have a trailing `/`. + path?: string; +}; + +export class ReadTreeResponseFactory { + static create(options: { config: Config }): ReadTreeResponseFactory { + return new ReadTreeResponseFactory( + options.config.getOptionalString('backend.workingDirectory') ?? + os.tmpdir(), + ); + } + + constructor(private readonly workDir: string) {} + + async fromArchive(options: FromArchiveOptions): Promise { + return new ArchiveResponse( + options.stream, + options.path ?? '', + this.workDir, + ); + } +} diff --git a/packages/backend-common/src/reading/tree/index.ts b/packages/backend-common/src/reading/tree/index.ts new file mode 100644 index 0000000000..858cf15877 --- /dev/null +++ b/packages/backend-common/src/reading/tree/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ReadTreeResponseFactory } from './ReadTreeResponseFactory'; From 73231427e138a7aa983a22a5d84e725805500d10 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 12:20:07 +0100 Subject: [PATCH 035/259] backend-common: add top-level archive filtering to ArchiveResponse + fix files() --- .../src/reading/tree/ArchiveResponse.test.ts | 67 +++++++++++++++++-- .../src/reading/tree/ArchiveResponse.ts | 58 ++++++++++++---- .../reading/tree/ReadTreeResponseFactory.ts | 3 + 3 files changed, 112 insertions(+), 16 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts index 57a490650c..6b8f29de8c 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts @@ -36,10 +36,51 @@ describe('ArchiveResponse', () => { mockFs.restore(); }); + it('should read files', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp'); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + { + path: 'docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + + it('should read files with filter', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path => + path.endsWith('.yml'), + ); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + ]); + const content = await files[0].content(); + expect(content.toString('utf8').trim()).toEqual('site_name: Test'); + }); + it('should read as archive and files', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo', '/tmp'); + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( @@ -51,11 +92,11 @@ describe('ArchiveResponse', () => { expect(files).toEqual([ { - path: './mkdocs.yml', + path: 'mkdocs.yml', content: expect.any(Function), }, { - path: './docs/index.md', + path: 'docs/index.md', content: expect.any(Function), }, ]); @@ -83,11 +124,29 @@ describe('ArchiveResponse', () => { it('should extract archive into directory with a subpath', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/docs', '/tmp'); + const res = new ArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); const dir = await res.dir(); + expect(dir).toMatch(/^\/tmp\/.*$/); await expect( fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), ).resolves.toBe('# Test\n'); }); + + it('should extract archive into directory with a subpath and filter', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path => + path.endsWith('.yml'), + ); + const dir = await res.dir({ targetDir: '/tmp' }); + + expect(dir).toBe('/tmp'); + await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( + true, + ); + await expect( + fs.pathExists(resolvePath(dir, 'docs/index.md')), + ).resolves.toBe(false); + }); }); diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.ts index 65ad8c18fe..83f6f6a8e5 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.ts @@ -21,7 +21,7 @@ import fs from 'fs-extra'; import { Readable, pipeline as pipelineCb } from 'stream'; import { promisify } from 'util'; import concatStream from 'concat-stream'; -import { ReadTreeResponse, File } from '../types'; +import { ReadTreeResponse, File, ReadTreeResponseDirOptions } from '../types'; // Tar types for `Parse` is not a proper constructor, but it should be const TarParseStream = (Parse as unknown) as { new (): ParseStream }; @@ -38,7 +38,17 @@ export class ArchiveResponse implements ReadTreeResponse { private readonly stream: Readable, private readonly subPath: string, private readonly workDir: string = os.tmpdir(), - ) {} + private readonly filter?: (path: string) => boolean, + ) { + if (subPath) { + if (!subPath.endsWith('/')) { + throw new TypeError('ArchiveResponse subPath must end with a /'); + } + if (subPath.startsWith('/')) { + throw new TypeError('ArchiveResponse subPath must not start with a /'); + } + } + } // Make sure the input stream is only read once private onlyOnce() { @@ -60,14 +70,28 @@ export class ArchiveResponse implements ReadTreeResponse { return; } - const contentPromise = new Promise(async resolve => { + if (this.subPath) { + if (!entry.path.startsWith(this.subPath)) { + entry.resume(); + return; + } + } + + const path = this.subPath + ? entry.path.replace(this.subPath, '') + : entry.path; + if (this.filter) { + if (!this.filter(path)) { + entry.resume(); + return; + } + } + + const content = new Promise(async resolve => { await pipeline(entry, concatStream(resolve)); }); - files.push({ - path: entry.path, - content: () => contentPromise, - }); + files.push({ path, content: () => content }); entry.resume(); }); @@ -93,7 +117,7 @@ export class ArchiveResponse implements ReadTreeResponse { try { return await new Promise(async resolve => { await pipeline( - tar.create({ cwd: tmpDir }, ['.']), + tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); }); @@ -102,20 +126,30 @@ export class ArchiveResponse implements ReadTreeResponse { } } - async dir(outDir?: string): Promise { + async dir(options?: ReadTreeResponseDirOptions): Promise { this.onlyOnce(); const dir = - outDir ?? (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); + options?.targetDir ?? + (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); - const strip = this.subPath ? this.subPath.split('/').length : 0; + const strip = this.subPath ? this.subPath.split('/').length - 1 : 0; await pipeline( this.stream, tar.extract({ strip, cwd: dir, - filter: path => path.startsWith(this.subPath), + filter: path => { + if (this.subPath && !path.startsWith(this.subPath)) { + return false; + } + if (this.filter) { + const innerPath = path.split('/').slice(strip).join('/'); + return this.filter(innerPath); + } + return true; + }, }), ); diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 0cdf926720..5044cdabdc 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -25,6 +25,8 @@ type FromArchiveOptions = { stream: Readable; // If set, root of the tree will be set to the given path. Should not have a trailing `/`. path?: string; + // Filter passed on from the ReadTreeOptions + filter?: (path: string) => boolean; }; export class ReadTreeResponseFactory { @@ -42,6 +44,7 @@ export class ReadTreeResponseFactory { options.stream, options.path ?? '', this.workDir, + options.filter, ); } } From 5bc30d262f928955776763a1f217da2811858aee Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 12:31:23 +0100 Subject: [PATCH 036/259] backend-common: make ReadTreeResponse.dir() accept an options object instead --- .../src/reading/GithubUrlReader.test.ts | 2 +- .../backend-common/src/reading/GithubUrlReader.ts | 13 ++++++++++--- packages/backend-common/src/reading/types.ts | 12 +++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 7552547924..e165745790 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -293,7 +293,7 @@ describe('GithubUrlReader', () => { ); mockfs(); - const directory = await response.dir('/tmp/fs'); + const directory = await response.dir({ targetDir: '/tmp/fs' }); const writtenToDirectory = fs.existsSync(directory); const paths = await recursive(directory); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 41db541f7f..e3a47ee420 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -18,7 +18,13 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader, File } from './types'; +import { + ReaderFactory, + ReadTreeResponse, + UrlReader, + File, + ReadTreeResponseDirOptions, +} from './types'; import tar from 'tar'; import fs from 'fs-extra'; import concatStream from 'concat-stream'; @@ -294,9 +300,10 @@ export class GithubUrlReader implements UrlReader { resolve(Buffer.from('Archive is not yet implemented')), ); }, - dir: (outDir: string | undefined) => { + dir: (options?: ReadTreeResponseDirOptions) => { const targetDirectory = - outDir || fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-')); + options?.targetDir || + fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-')); return new Promise((res, rej) => { Promise.all( diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index d9fa5d839e..8a78e9d65f 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -17,6 +17,11 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; +export type ReadTreeOptions = { + /** A filter that can be used to select which files should be extracted. By default all files are extracted */ + filter?(path: string): boolean; +}; + /** * A generic interface for fetching plain data from URLs. */ @@ -48,8 +53,13 @@ export type File = { content(): Promise; }; +export type ReadTreeResponseDirOptions = { + /** The directory to write files to. Defaults to the OS tmpdir or `backend.workingDirectory` if set in config */ + targetDir?: string; +}; + export type ReadTreeResponse = { files(): Promise; archive(): Promise; - dir(outDir?: string): Promise; + dir(options?: ReadTreeResponseDirOptions): Promise; }; From e1057d9a506d7763ffdad4d5aaf1c84c4c6414eb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 12:33:58 +0100 Subject: [PATCH 037/259] backend-common: provide a ReadTreeResponseFactory instance to all ReaderFactories --- .../src/reading/AzureUrlReader.test.ts | 17 +++++++++++++++-- .../src/reading/GitlabUrlReader.test.ts | 17 +++++++++++++++-- .../backend-common/src/reading/UrlReaders.ts | 4 +++- packages/backend-common/src/reading/types.ts | 2 ++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 24b435006a..6efc7641b8 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -20,9 +20,14 @@ import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '../logging'; import { AzureUrlReader } from './AzureUrlReader'; import { msw } from '@backstage/test-utils'; +import { ReadTreeResponseFactory } from './tree'; const logger = getVoidLogger(); +const treeResponseFactory = ReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + describe('AzureUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); @@ -87,7 +92,11 @@ describe('AzureUrlReader', () => { }), }, ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = AzureUrlReader.factory({ config, logger }); + const [{ reader }] = AzureUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); const data = await reader.read(url); const res = await JSON.parse(data.toString('utf-8')); @@ -115,7 +124,11 @@ describe('AzureUrlReader', () => { }, ])('should handle error path %#', async ({ url, config, error }) => { await expect(async () => { - const [{ reader }] = AzureUrlReader.factory({ config, logger }); + const [{ reader }] = AzureUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); await reader.read(url); }).rejects.toThrow(error); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 4b3aa471c3..3a76c0631b 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -20,9 +20,14 @@ import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; import { msw } from '@backstage/test-utils'; +import { ReadTreeResponseFactory } from './tree'; const logger = getVoidLogger(); +const treeResponseFactory = ReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + describe('GitlabUrlReader', () => { const worker = setupServer(); @@ -98,7 +103,11 @@ describe('GitlabUrlReader', () => { }), }, ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = GitlabUrlReader.factory({ config, logger }); + const [{ reader }] = GitlabUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); const data = await reader.read(url); const res = await JSON.parse(data.toString('utf-8')); @@ -114,7 +123,11 @@ describe('GitlabUrlReader', () => { }, ])('should handle error path %#', async ({ url, config, error }) => { await expect(async () => { - const [{ reader }] = GitlabUrlReader.factory({ config, logger }); + const [{ reader }] = GitlabUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); await reader.read(url); }).rejects.toThrow(error); }); diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index e1d99a2c49..2bb5617907 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -23,6 +23,7 @@ import { BitbucketUrlReader } from './BitbucketUrlReader'; import { GithubUrlReader } from './GithubUrlReader'; import { GitlabUrlReader } from './GitlabUrlReader'; import { FetchUrlReader } from './FetchUrlReader'; +import { ReadTreeResponseFactory } from './tree'; type CreateOptions = { /** Root config object */ @@ -49,9 +50,10 @@ export class UrlReaders { fallback, }: CreateOptions): UrlReader { const mux = new UrlReaderPredicateMux({ fallback: fallback }); + const treeResponseFactory = ReadTreeResponseFactory.create({ config }); for (const factory of factories ?? []) { - const tuples = factory({ config, logger: logger }); + const tuples = factory({ config, logger: logger, treeResponseFactory }); for (const tuple of tuples) { mux.register(tuple); diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 8a78e9d65f..bf53e98ef2 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -16,6 +16,7 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; +import { ReadTreeResponseFactory } from './tree'; export type ReadTreeOptions = { /** A filter that can be used to select which files should be extracted. By default all files are extracted */ @@ -46,6 +47,7 @@ export type UrlReaderPredicateTuple = { export type ReaderFactory = (options: { config: Config; logger: Logger; + treeResponseFactory: ReadTreeResponseFactory; }) => UrlReaderPredicateTuple[]; export type File = { From 6417d966bb4117093883f6f64d15d7c6f339ec08 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 13:53:22 +0100 Subject: [PATCH 038/259] backend-common,techdocs-backend: refactor .readTree to accept options and implement GitHub reader with response utils --- .../src/reading/GithubUrlReader.test.ts | 82 ++++++----- .../src/reading/GithubUrlReader.ts | 139 ++++++------------ .../src/reading/UrlReaderPredicateMux.ts | 14 +- .../src/reading/tree/ArchiveResponse.ts | 6 +- packages/backend-common/src/reading/types.ts | 6 +- plugins/techdocs-backend/src/helpers.test.ts | 54 +------ plugins/techdocs-backend/src/helpers.ts | 15 +- 7 files changed, 113 insertions(+), 203 deletions(-) diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index e165745790..658d908f84 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -29,8 +29,11 @@ import { } from './GithubUrlReader'; import fs from 'fs'; import path from 'path'; -import mockfs from 'mock-fs'; -import recursive from 'recursive-readdir'; +import { ReadTreeResponseFactory } from './tree'; + +const treeResponseFactory = ReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { @@ -226,10 +229,13 @@ describe('GithubUrlReader', () => { describe('implementation', () => { it('rejects unknown targets', async () => { - const processor = new GithubUrlReader({ - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }); + const processor = new GithubUrlReader( + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }, + { treeResponseFactory }, + ); await expect( processor.read('https://not.github.com/apa'), ).rejects.toThrow( @@ -250,7 +256,7 @@ describe('GithubUrlReader', () => { beforeEach(() => { worker.use( rest.get( - 'https://github.com/spotify/mock/archive/repo.tar.gz', + 'https://github.com/backstage/mock/archive/repo.tar.gz', (_, res, ctx) => res( ctx.status(200), @@ -262,18 +268,20 @@ describe('GithubUrlReader', () => { }); it('returns the wanted files from an archive', async () => { - const processor = new GithubUrlReader({ - host: 'github.com', - }); + const processor = new GithubUrlReader( + { + host: 'github.com', + }, + { treeResponseFactory }, + ); const response = await processor.readTree( - 'https://github.com/spotify/mock', - 'repo', - ['mkdocs.yml', 'docs'], + 'https://github.com/backstage/mock/tree/repo', ); const files = await response.files(); + expect(files.length).toBe(2); const mkDocsFile = await files[0].content(); const indexMarkdownFile = await files[1].content(); @@ -281,33 +289,39 @@ describe('GithubUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); - it('returns a folder path from an archive', async () => { - const processor = new GithubUrlReader({ - host: 'github.com', - }); + it('must specify a branch', async () => { + const processor = new GithubUrlReader( + { + host: 'github.com', + }, + { treeResponseFactory }, + ); + + await expect( + processor.readTree('https://github.com/backstage/mock'), + ).rejects.toThrow( + 'GitHub URL must contain branch to be able to fetch tree', + ); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + const processor = new GithubUrlReader( + { + host: 'github.com', + }, + { treeResponseFactory }, + ); const response = await processor.readTree( - 'https://github.com/spotify/mock', - 'repo', - ['mkdocs.yml', 'docs'], + 'https://github.com/backstage/mock/tree/repo/docs', ); - mockfs(); - const directory = await response.dir({ targetDir: '/tmp/fs' }); + const files = await response.files(); - const writtenToDirectory = fs.existsSync(directory); - const paths = await recursive(directory); - mockfs.restore(); + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); - expect(writtenToDirectory).toBe(true); - expect(paths.sort()).toEqual( - [ - '/tmp/fs/mock-repo/docs/index.md', - '/tmp/fs/mock-repo/mkdocs.yml', - ].sort(), - ); - - worker.resetHandlers(); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index e3a47ee420..ca67a6cfcb 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -17,19 +17,15 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch from 'cross-fetch'; -import { NotFoundError } from '../errors'; +import { Readable } from 'stream'; +import { InputError, NotFoundError } from '../errors'; import { ReaderFactory, ReadTreeResponse, UrlReader, - File, - ReadTreeResponseDirOptions, + ReadTreeOptions, } from './types'; -import tar from 'tar'; -import fs from 'fs-extra'; -import concatStream from 'concat-stream'; -import path from 'path'; -import os from 'os'; +import { ReadTreeResponseFactory } from './tree'; /** * The configuration parameters for a single GitHub API provider. @@ -198,19 +194,18 @@ export function readConfig(config: Config): ProviderConfig[] { * the one exposed by GitHub itself. */ export class GithubUrlReader implements UrlReader { - private config: ProviderConfig; - - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { return readConfig(config).map(provider => { - const reader = new GithubUrlReader(provider); + const reader = new GithubUrlReader(provider, { treeResponseFactory }); const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); }; - constructor(config: ProviderConfig) { - this.config = config; - } + constructor( + private readonly config: ProviderConfig, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) {} async read(url: string): Promise { const useApi = @@ -240,90 +235,48 @@ export class GithubUrlReader implements UrlReader { throw new Error(message); } - private async getRepositoryArchive( - repoUrl: string, - branchName: string, - ): Promise { - return fetch(new URL(`${repoUrl}/archive/${branchName}.tar.gz`).toString()); - } - - private async writeBufferToFile( - filePath: string, - content: Buffer, - ): Promise { - await fs.outputFile(filePath, content.toString()); - } - async readTree( - repoUrl: string, - branchName: string, - paths: Array, + url: string, + options?: ReadTreeOptions, ): Promise { - const { name: repoName } = parseGitUri(repoUrl); + const { + name: repoName, + ref, + protocol, + source, + full_name, + filepath, + } = parseGitUri(url); - const repoArchive = await this.getRepositoryArchive(repoUrl, branchName); + if (!ref) { + // TODO(Rugvip): We should add support for defaulting to the default branch + throw new InputError( + 'GitHub URL must contain branch to be able to fetch tree', + ); + } - const files: File[] = []; - return new Promise(resolve => { - const parser = new (tar.Parse as any)({ - filter: (path: string) => - !!paths.filter(file => { - return path.startsWith(`${repoName}-${branchName}/${file}`); - }).length, - onentry: (entry: tar.ReadEntry) => { - if (entry.type === 'Directory') { - entry.resume(); - return; - } + // TODO(Rugvip): use API to fetch URL instead + const response = await fetch( + new URL( + `${protocol}://${source}/${full_name}/archive/${ref}.tar.gz`, + ).toString(), + ); + if (!response.ok) { + const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } - const contentPromise: Promise = new Promise(res => { - entry.pipe(concatStream(res)); - }); + const path = `${repoName}-${ref}/${filepath}`; - files.push({ - path: entry.path, - content: () => contentPromise, - }); - - entry.resume(); - }, - }); - - // @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why. - repoArchive.body?.pipe(parser).on('finish', () => { - resolve({ - files: async () => { - return files; - }, - archive: () => { - return new Promise(resolve => - resolve(Buffer.from('Archive is not yet implemented')), - ); - }, - dir: (options?: ReadTreeResponseDirOptions) => { - const targetDirectory = - options?.targetDir || - fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-')); - - return new Promise((res, rej) => { - Promise.all( - files.map(async file => { - return this.writeBufferToFile( - `${targetDirectory}/${file.path}`, - await file.content(), - ); - }), - ) - .then(() => { - res(`${targetDirectory}/${repoName}-${branchName}`); - }) - .catch(err => { - rej(err); - }); - }); - }, - }); - }); + return this.deps.treeResponseFactory.fromArchive({ + // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want + // to stick to using that in exclusively backend code. + stream: (response.body as unknown) as Readable, + path, + filter: options?.filter, }); } diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 74f94f0295..9b0566106a 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -14,7 +14,12 @@ * limitations under the License. */ -import { ReadTreeResponse, UrlReader, UrlReaderPredicateTuple } from './types'; +import { + ReadTreeOptions, + ReadTreeResponse, + UrlReader, + UrlReaderPredicateTuple, +} from './types'; type Options = { // UrlReader to fall back to if no other reader is matched @@ -55,14 +60,13 @@ export class UrlReaderPredicateMux implements UrlReader { readTree( repoUrl: string, - branchName: string, - paths: Array, + options?: ReadTreeOptions, ): Promise { const parsed = new URL(repoUrl); for (const { predicate, reader } of this.readers) { if (predicate(parsed)) { - if (reader.readTree) return reader.readTree(repoUrl, branchName, paths); + if (reader.readTree) return reader.readTree(repoUrl, options); throw new Error( `Trying to call readTree on UrlReader which does not support the feature.`, ); @@ -71,7 +75,7 @@ export class UrlReaderPredicateMux implements UrlReader { if (this.fallback) { if (this.fallback.readTree) - return this.fallback.readTree(repoUrl, branchName, paths); + return this.fallback.readTree(repoUrl, options); throw new Error( `Trying to call readTree on UrlReader which does not support the feature.`, ); diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.ts index 83f6f6a8e5..0e06ff3804 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.ts @@ -42,10 +42,12 @@ export class ArchiveResponse implements ReadTreeResponse { ) { if (subPath) { if (!subPath.endsWith('/')) { - throw new TypeError('ArchiveResponse subPath must end with a /'); + this.subPath += '/'; } if (subPath.startsWith('/')) { - throw new TypeError('ArchiveResponse subPath must not start with a /'); + throw new TypeError( + `ArchiveResponse subPath must not start with a /, got '${subPath}'`, + ); } } } diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index bf53e98ef2..c7e5915401 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -28,11 +28,7 @@ export type ReadTreeOptions = { */ export type UrlReader = { read(url: string): Promise; - readTree?( - repoUrl: string, - branchName: string, - paths: Array, - ): Promise; + readTree?(url: string, options?: ReadTreeOptions): Promise; }; export type UrlReaderPredicateTuple = { diff --git a/plugins/techdocs-backend/src/helpers.test.ts b/plugins/techdocs-backend/src/helpers.test.ts index c3f2261637..5e98ab51c8 100644 --- a/plugins/techdocs-backend/src/helpers.test.ts +++ b/plugins/techdocs-backend/src/helpers.test.ts @@ -19,7 +19,7 @@ import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; describe('getDocFilesFromRepository', () => { - it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in root', async () => { + it('should read a remote directory using UrlReader.readTree', async () => { class MockUrlReader implements UrlReader { async read() { return Buffer.from('mock'); @@ -45,7 +45,7 @@ describe('getDocFilesFromRepository', () => { namespace: 'default', annotations: { 'backstage.io/techdocs-ref': - 'url:https://github.com/backstage/backstage/blob/master/mkdocs.yml', + 'url:https://github.com/backstage/backstage/blob/master/subfolder/', }, name: 'mytestcomponent', description: 'A component for testing', @@ -64,54 +64,6 @@ describe('getDocFilesFromRepository', () => { mockEntity, ); - expect(output).toBe('/tmp/testfolder/.'); - }); - - it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in a subfolder', async () => { - class MockUrlReader implements UrlReader { - async read() { - return Buffer.from('mock'); - } - - async readTree(): Promise { - return { - dir: async () => { - return '/tmp/testfolder'; - }, - files: async () => { - return []; - }, - archive: async () => { - return Buffer.from(''); - }, - }; - } - } - - const mockEntity: Entity = { - metadata: { - namespace: 'default', - annotations: { - 'backstage.io/techdocs-ref': - 'url:https://github.com/backstage/backstage/blob/master/subfolder/mkdocs.yml', - }, - name: 'mytestcomponent', - description: 'A component for testing', - }, - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - spec: { - type: 'documentation', - lifecycle: 'experimental', - owner: 'testuser', - }, - }; - - const output = await getDocFilesFromRepository( - new MockUrlReader(), - mockEntity, - ); - - expect(output).toBe('/tmp/testfolder/subfolder'); + expect(output).toBe('/tmp/testfolder'); }); }); diff --git a/plugins/techdocs-backend/src/helpers.ts b/plugins/techdocs-backend/src/helpers.ts index f861d4375e..97b087558f 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/plugins/techdocs-backend/src/helpers.ts @@ -179,21 +179,10 @@ export const getDocFilesFromRepository = async ( entity, ); - const { ref, filepath: mkdocsPath } = parseGitUrl(target); - - const docsRootPath = path.dirname(mkdocsPath); - const docsFolderPath = path.join(docsRootPath, 'docs'); - if (reader.readTree) { - const readTreeResponse = await reader.readTree( - parseGitUrl(target).toString(), - ref, - [mkdocsPath, docsFolderPath], - ); + const response = await reader.readTree(target); - const tmpDir = await readTreeResponse.dir(); - - return `${tmpDir}/${docsRootPath}`; + return response.dir(); } throw new Error( From b672492e2164a58b0224cfbdbd26f3e29479d7d1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 13:59:10 +0100 Subject: [PATCH 039/259] backend-common: make .readTree required --- .../backend-common/src/reading/AzureUrlReader.ts | 6 +++++- .../backend-common/src/reading/BitbucketUrlReader.ts | 6 +++++- .../backend-common/src/reading/FetchUrlReader.ts | 6 +++++- .../backend-common/src/reading/GitlabUrlReader.ts | 6 +++++- .../src/reading/UrlReaderPredicateMux.ts | 11 ++--------- packages/backend-common/src/reading/types.ts | 2 +- .../ingestion/processors/CodeOwnersProcessor.test.ts | 12 ++++++------ .../processors/PlaceholderProcessor.test.ts | 2 +- .../src/service/CatalogBuilder.test.ts | 5 ++++- plugins/techdocs-backend/src/helpers.ts | 10 ++-------- 10 files changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 8e6e6a75df..79058c79a9 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -17,7 +17,7 @@ import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import { NotFoundError } from '../errors'; -import { ReaderFactory, UrlReader } from './types'; +import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; type Options = { // TODO: added here for future support, but we only allow dev.azure.com for now @@ -86,6 +86,10 @@ export class AzureUrlReader implements UrlReader { throw new Error(message); } + readTree(): Promise { + throw new Error('AzureUrlReader does not implement readTree'); + } + // Converts // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 348426fd34..b2fb9fa309 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -18,7 +18,7 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { ReaderFactory, UrlReader } from './types'; +import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; const DEFAULT_BASE_URL = 'https://api.bitbucket.org/2.0'; @@ -209,6 +209,10 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } + readTree(): Promise { + throw new Error('BitbucketUrlReader does not implement readTree'); + } + toString() { const { host, token, username, appPassword } = this.config; let authed = Boolean(token); diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index d2e5c45620..1d1784590c 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -16,7 +16,7 @@ import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { UrlReader } from './types'; +import { ReadTreeResponse, UrlReader } from './types'; /** * A UrlReader that does a plain fetch of the URL. @@ -41,6 +41,10 @@ export class FetchUrlReader implements UrlReader { throw new Error(message); } + readTree(): Promise { + throw new Error('FetchUrlReader does not implement readTree'); + } + toString() { return 'fetch{}'; } diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 378e76fb61..c621e41338 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -17,7 +17,7 @@ import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import { NotFoundError } from '../errors'; -import { ReaderFactory, UrlReader } from './types'; +import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; type Options = { host: string; @@ -87,6 +87,10 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } + readTree(): Promise { + throw new Error('GitlabUrlReader does not implement readTree'); + } + // Converts // from: https://gitlab.example.com/a/b/blob/master/c.yaml // to: https://gitlab.example.com/a/b/raw/master/c.yaml diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 9b0566106a..9a81fdc9ab 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -66,19 +66,12 @@ export class UrlReaderPredicateMux implements UrlReader { for (const { predicate, reader } of this.readers) { if (predicate(parsed)) { - if (reader.readTree) return reader.readTree(repoUrl, options); - throw new Error( - `Trying to call readTree on UrlReader which does not support the feature.`, - ); + return reader.readTree(repoUrl, options); } } if (this.fallback) { - if (this.fallback.readTree) - return this.fallback.readTree(repoUrl, options); - throw new Error( - `Trying to call readTree on UrlReader which does not support the feature.`, - ); + return this.fallback.readTree(repoUrl, options); } throw new Error(`No reader found that could handle '${repoUrl}'`); diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index c7e5915401..101502fed7 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -28,7 +28,7 @@ export type ReadTreeOptions = { */ export type UrlReader = { read(url: string): Promise; - readTree?(url: string, options?: ReadTreeOptions): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; }; export type UrlReaderPredicateTuple = { diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts index e4ddb249d6..dfb470fa21 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts @@ -157,14 +157,14 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: ownersText })); - const reader = { read }; + const reader = { read, readTree: jest.fn() }; const result = await findRawCodeOwners(mockLocation(), reader); expect(result).toEqual(ownersText); }); it('should raise error when no codeowner', async () => { const read = jest.fn().mockRejectedValue(mockReadResult()); - const reader = { read }; + const reader = { read, readTree: jest.fn() }; await expect( findRawCodeOwners(mockLocation(), reader), @@ -178,7 +178,7 @@ describe('CodeOwnersProcessor', () => { .mockImplementationOnce(() => mockReadResult({ error: 'foo' })) .mockImplementationOnce(() => mockReadResult({ error: 'bar' })) .mockResolvedValue(mockReadResult({ data: ownersText })); - const reader = { read }; + const reader = { read, readTree: jest.fn() }; const result = await findRawCodeOwners(mockLocation(), reader); @@ -197,7 +197,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() })); - const reader = { read }; + const reader = { read, readTree: jest.fn() }; const owner = await resolveCodeOwner(mockLocation(), reader); expect(owner).toBe('backstage-core'); @@ -207,7 +207,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockImplementation(() => mockReadResult({ error: 'error: foo' })); - const reader = { read }; + const reader = { read, readTree: jest.fn() }; await expect( resolveCodeOwner(mockLocation(), reader), @@ -221,7 +221,7 @@ describe('CodeOwnersProcessor', () => { const read = jest .fn() .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() })); - const reader = { read }; + const reader = { read, readTree: jest.fn() }; const processor = new CodeOwnersProcessor({ reader }); return { entity, processor, read }; diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts index 48cd2dae01..5a3511af6d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -27,7 +27,7 @@ import { describe('PlaceholderProcessor', () => { const read: jest.MockedFunction = jest.fn(); - const reader: UrlReader = { read }; + const reader: UrlReader = { read, readTree: jest.fn() }; beforeEach(() => { jest.resetAllMocks(); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 45d92e6555..ee7c6f2246 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -40,7 +40,10 @@ const dummyEntityYaml = yaml.stringify(dummyEntity); describe('CatalogBuilder', () => { let db: Knex; - const reader: jest.Mocked = { read: jest.fn() }; + const reader: jest.Mocked = { + read: jest.fn(), + readTree: jest.fn(), + }; const env: CatalogEnvironment = { logger: getVoidLogger(), database: { getClient: async () => db }, diff --git a/plugins/techdocs-backend/src/helpers.ts b/plugins/techdocs-backend/src/helpers.ts index 97b087558f..901aea533a 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/plugins/techdocs-backend/src/helpers.ts @@ -179,13 +179,7 @@ export const getDocFilesFromRepository = async ( entity, ); - if (reader.readTree) { - const response = await reader.readTree(target); + const response = await reader.readTree(target); - return response.dir(); - } - - throw new Error( - `No readTree method available on the UrlReader for ${target}`, - ); + return await response.dir(); }; From 8e2effb531a862c0415c6adf4dfb352421ccc07c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 14:01:32 +0100 Subject: [PATCH 040/259] changesets: added changeset for UrlReader readTree refactoring --- .changeset/odd-camels-begin.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/odd-camels-begin.md diff --git a/.changeset/odd-camels-begin.md b/.changeset/odd-camels-begin.md new file mode 100644 index 0000000000..d28d729609 --- /dev/null +++ b/.changeset/odd-camels-begin.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': minor +--- + +Refactored UrlReader.readTree to be required and accept (url, options) From bd1cef1847bb2f2c75d7eda147a68a868f06b31e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Nov 2020 00:43:20 +0100 Subject: [PATCH 041/259] backend-common: review comments for readTree + switch archive return type to a stream --- .../src/reading/UrlReaderPredicateMux.ts | 13 ++++------ .../src/reading/tree/ArchiveResponse.test.ts | 3 +-- .../src/reading/tree/ArchiveResponse.ts | 26 +++++++++---------- .../reading/tree/ReadTreeResponseFactory.ts | 2 +- packages/backend-common/src/reading/types.ts | 19 +++++++++++--- plugins/techdocs-backend/src/helpers.test.ts | 3 ++- 6 files changed, 37 insertions(+), 29 deletions(-) diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 9a81fdc9ab..465c125fda 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -58,23 +58,20 @@ export class UrlReaderPredicateMux implements UrlReader { throw new Error(`No reader found that could handle '${url}'`); } - readTree( - repoUrl: string, - options?: ReadTreeOptions, - ): Promise { - const parsed = new URL(repoUrl); + readTree(url: string, options?: ReadTreeOptions): Promise { + const parsed = new URL(url); for (const { predicate, reader } of this.readers) { if (predicate(parsed)) { - return reader.readTree(repoUrl, options); + return reader.readTree(url, options); } } if (this.fallback) { - return this.fallback.readTree(repoUrl, options); + return this.fallback.readTree(url, options); } - throw new Error(`No reader found that could handle '${repoUrl}'`); + throw new Error(`No reader found that could handle '${url}'`); } toString() { diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts index 6b8f29de8c..fd351863b2 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts @@ -16,7 +16,6 @@ import fs from 'fs-extra'; import mockFs from 'mock-fs'; -import { Readable } from 'stream'; import { resolve as resolvePath } from 'path'; import { ArchiveResponse } from './ArchiveResponse'; @@ -87,7 +86,7 @@ describe('ArchiveResponse', () => { 'Response has already been read', ); - const res2 = new ArchiveResponse(Readable.from(buffer), '', '/tmp'); + const res2 = new ArchiveResponse(buffer, '', '/tmp'); const files = await res2.files(); expect(files).toEqual([ diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.ts index 0e06ff3804..e16be63127 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.ts @@ -14,14 +14,17 @@ * limitations under the License. */ -import os from 'os'; import tar, { Parse, ParseStream, ReadEntry } from 'tar'; import path from 'path'; import fs from 'fs-extra'; import { Readable, pipeline as pipelineCb } from 'stream'; import { promisify } from 'util'; import concatStream from 'concat-stream'; -import { ReadTreeResponse, File, ReadTreeResponseDirOptions } from '../types'; +import { + ReadTreeResponse, + ReadTreeResponseFile, + ReadTreeResponseDirOptions, +} from '../types'; // Tar types for `Parse` is not a proper constructor, but it should be const TarParseStream = (Parse as unknown) as { new (): ParseStream }; @@ -37,7 +40,7 @@ export class ArchiveResponse implements ReadTreeResponse { constructor( private readonly stream: Readable, private readonly subPath: string, - private readonly workDir: string = os.tmpdir(), + private readonly workDir: string, private readonly filter?: (path: string) => boolean, ) { if (subPath) { @@ -60,10 +63,10 @@ export class ArchiveResponse implements ReadTreeResponse { this.read = true; } - async files(): Promise { + async files(): Promise { this.onlyOnce(); - const files: File[] = []; + const files = Array(); const parser = new TarParseStream(); parser.on('entry', (entry: ReadEntry & Readable) => { @@ -79,9 +82,7 @@ export class ArchiveResponse implements ReadTreeResponse { } } - const path = this.subPath - ? entry.path.replace(this.subPath, '') - : entry.path; + const path = entry.path.slice(this.subPath.length); if (this.filter) { if (!this.filter(path)) { entry.resume(); @@ -103,13 +104,11 @@ export class ArchiveResponse implements ReadTreeResponse { return files; } - async archive(): Promise { + async archive(): Promise { if (!this.subPath) { this.onlyOnce(); - return new Promise(resolve => - pipeline(this.stream, concatStream(resolve)), - ); + return this.stream; } // TODO(Rugvip): method for repacking a tar with a subpath is to simply extract into a @@ -117,12 +116,13 @@ export class ArchiveResponse implements ReadTreeResponse { const tmpDir = await this.dir(); try { - return await new Promise(async resolve => { + const data = await new Promise(async resolve => { await pipeline( tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); }); + return Readable.from(data); } finally { await fs.remove(tmpDir); } diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 5044cdabdc..a134d185f3 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -23,7 +23,7 @@ import { ArchiveResponse } from './ArchiveResponse'; type FromArchiveOptions = { // A binary stream of a tar archive. stream: Readable; - // If set, root of the tree will be set to the given path. Should not have a trailing `/`. + // If set, the root of the tree will be set to the given directory path. path?: string; // Filter passed on from the ReadTreeOptions filter?: (path: string) => boolean; diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 101502fed7..f9dca3e1d5 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -19,7 +19,18 @@ import { Config } from '@backstage/config'; import { ReadTreeResponseFactory } from './tree'; export type ReadTreeOptions = { - /** A filter that can be used to select which files should be extracted. By default all files are extracted */ + /** + * A filter that can be used to select which files should be included. + * + * The path passed to the filter function is the relative path from the URL + * that the file tree is fetched from, without any leading '/'. + * + * For example, given the URL https://github.com/my/repo/tree/master/my-dir, a file + * at https://github.com/my/repo/blob/master/my-dir/my-subdir/my-file.txt will + * be represented as my-subdir/my-file.txt + * + * If no filter is provided all files are extracted. + */ filter?(path: string): boolean; }; @@ -46,7 +57,7 @@ export type ReaderFactory = (options: { treeResponseFactory: ReadTreeResponseFactory; }) => UrlReaderPredicateTuple[]; -export type File = { +export type ReadTreeResponseFile = { path: string; content(): Promise; }; @@ -57,7 +68,7 @@ export type ReadTreeResponseDirOptions = { }; export type ReadTreeResponse = { - files(): Promise; - archive(): Promise; + files(): Promise; + archive(): Promise; dir(options?: ReadTreeResponseDirOptions): Promise; }; diff --git a/plugins/techdocs-backend/src/helpers.test.ts b/plugins/techdocs-backend/src/helpers.test.ts index 5e98ab51c8..10518df74f 100644 --- a/plugins/techdocs-backend/src/helpers.test.ts +++ b/plugins/techdocs-backend/src/helpers.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { Readable } from 'stream'; import { getDocFilesFromRepository } from './helpers'; import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; @@ -34,7 +35,7 @@ describe('getDocFilesFromRepository', () => { return []; }, archive: async () => { - return Buffer.from(''); + return Readable.from(''); }, }; } From a51d475899f17083d3f266d084dbcbc67ce040ba Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 13 Nov 2020 02:51:11 -0500 Subject: [PATCH 042/259] Standarize code fence and wording tweaks (#3285) --- contrib/chart/backstage/README.md | 30 ++++++++++++++---------------- docs/api/utility-apis.md | 8 ++++---- docs/conf/index.md | 4 ++-- docs/conf/writing.md | 11 +++++++---- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/contrib/chart/backstage/README.md b/contrib/chart/backstage/README.md index 7b60062265..96bfdb4351 100644 --- a/contrib/chart/backstage/README.md +++ b/contrib/chart/backstage/README.md @@ -7,7 +7,7 @@ This folder contains Helm charts that can easily create a Kubernetes deployment These charts depend on the `nginx-ingress` controller being present in the cluster. If it's not already installed you can run: -``` +```shell helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm install nginx-ingress ingress-nginx/ingress-nginx ``` @@ -16,7 +16,7 @@ helm install nginx-ingress ingress-nginx/ingress-nginx After choosing a DNS name where backstage will be hosted create a yaml file for your custom configuration. -``` +```yaml appConfig: app: baseUrl: https://backstage.mydomain.com @@ -30,12 +30,11 @@ appConfig: techdocs: storageUrl: https://backstage.mydomain.com/api/techdocs/static/docs requestUrl: https://backstage.mydomain.com/api/techdocs - ``` Then use it to run: -``` +```shell git clone https://github.com/backstage/backstage.git cd contrib/chart/backstage helm dependency update @@ -54,7 +53,7 @@ After a few minutes Backstage should be up and running in your cluster under the Make sure to create the appropriate DNS entry in your infrastructure. To find the public IP address run: -```bash +```shell $ kubectl get ingress NAME HOSTS ADDRESS PORTS AGE backstage-ingress * 123.1.2.3 80 17m @@ -74,10 +73,10 @@ These charts can install or reuse a `clusterIssuer` to generate certificates for To enable it you need to provide a valid email address in the chart's values: -``` +```yaml issuer: email: me@example.com - clusterIssuer: "letsencrypt-prod" + clusterIssuer: 'letsencrypt-prod' ``` By default, the charts use `letsencrypt-staging` so in the above example we instruct helm to use the production issuer @@ -92,7 +91,7 @@ Configuring a connection to an existing PostgreSQL instance is possible through First create a yaml file with the configuration you want to override, for example `backstage-prod.yaml`: -```bash +```yaml postgresql: enabled: false @@ -122,12 +121,11 @@ lighthouse: user: password: database: lighthouse_audit_service - ``` For the CA, create a `configMap` named `--postgres-ca` with a file called `ca.crt`: -``` +```shell kubectl create configmap my-company-backstage-postgres-ca --from-file=ca.crt" ``` @@ -135,7 +133,7 @@ kubectl create configmap my-company-backstage-postgres-ca --from-file=ca.crt" Now install the helm chart: -``` +```shell cd contrib/chart/backstage helm install -f backstage-prod.yaml my-backstage . ``` @@ -144,7 +142,7 @@ helm install -f backstage-prod.yaml my-backstage . The docker images used for the deployment can be configured through the charts values: -``` +```yaml frontend: image: repository: @@ -165,7 +163,7 @@ lighthouse: Create a docker-registry secret -``` +```shell kubectl create secret docker-registry # args ``` @@ -181,7 +179,7 @@ dockerRegistrySecretName: To install the charts a specific namespace use `--namespace `: -``` +```shell helm install -f my_values.yaml --namespace demos backstage . ``` @@ -189,7 +187,7 @@ helm install -f my_values.yaml --namespace demos backstage . To deploy backstage with the pre-loaded demo data disable `backend.demoData`: -``` +```shell helm install -f my_values.yaml --set backend.demoData=false backstage . ``` @@ -244,7 +242,7 @@ This error happens in the backend when it tries to connect to the configured Pos To uninstall Backstage simply run: -``` +```shell RELEASE_NAME= # use `helm list` to find out the name helm uninstall ${RELEASE_NAME} kubectl delete pvc data-${RELEASE_NAME}-postgresql-0 diff --git a/docs/api/utility-apis.md b/docs/api/utility-apis.md index a4d154b44b..2547db5c5c 100644 --- a/docs/api/utility-apis.md +++ b/docs/api/utility-apis.md @@ -11,7 +11,7 @@ possible residing within the plugin itself and its backend APIs. There will however always be a need for plugins to communicate outside of its boundaries, both with other plugins and the app itself. -Backstage provides two primary methods for plugins to communication across their +Backstage provides two primary methods for plugins to communicate across their boundaries in client-side code. The first one being the `createPlugin` API and the registration hooks passed to the `register` method, and the second one being Utility APIs. While the `createPlugin` API is focused on the initialization @@ -195,8 +195,8 @@ interface for the API, and create an `ApiRef` using `createApiRef` exported from `@backstage/core`. Also be sure to provide at least one implementation of the API, and to declare a default factory for the API in `createPlugin`. -Custom Utility APIs can be either public or private, which it is up to the -plugin to choose. Private APIs do not expose an external API surface, and it's +Custom Utility APIs can be either public or private, which is up to the plugin +to choose. Private APIs do not expose an external API surface, and it's therefore possible to make breaking changes to the API without affecting other users of the plugin. If an API is made public however, it opens up for other plugins to make use of the API, and it also makes it possible for users for your @@ -243,7 +243,7 @@ The figure below shows the relationship between The current method for connecting Utility API providers and consumers is via the React tree using an `ApiProvider`, which is added to the `AppProvider` of the `App`. In the future there may potentially be more ways to do this, in ways that -are not tied to react. A design goal of the Utility APIs was to not have them +are not tied to React. A design goal of the Utility APIs was to not have them directly tied to React. The indirection provided by Utility APIs also makes it straightforward to test diff --git a/docs/conf/index.md b/docs/conf/index.md index 2c3ee06ba6..5e4639cc99 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -28,8 +28,8 @@ development or small tweaks to be able to reuse deployment artifacts in different environments. The configuration is shared between the frontend and backend, meaning that -values that are common between the two only needs to be defined once. Such as -the `backend.baseUrl`. +values that are common between the two only need to be defined once. Such as the +`backend.baseUrl`. For more details, see [Writing Configuration](./writing.md). diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 4eb92bb2ff..6b60c015ae 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -74,7 +74,7 @@ Note that if any config flags are provided, the default `app-config.yaml` files are NOT loaded. To include them you need to explicitly include them with a flag, for example: -``` +```shell yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml ``` @@ -99,8 +99,8 @@ order: ## Secrets -Secrets are supported via a special secret keys that are prefixed with `$`, -which in turn provides a number of different ways to read in secrets. To load a +Secrets are supported via special secret keys that are prefixed with `$`, which +in turn provide a number of different ways to read in secrets. To load a configuration value as a secret, supply an object with one of the special secret keys, for example `$env` or `$file`. A full list of supported secret keys can be found below. For example, the following will read the config key @@ -155,8 +155,11 @@ used to point to a specific value inside the file. Supported file extensions are ```yaml $data: ./my-secrets.json#deployment.key +``` -# my-secrets.json +Example `my-secrets.json` file: + +```json { "deployment": { "key": "my-secret-key" From 1b11e7b70807114b9e01a3ec692891a744643760 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 13 Nov 2020 02:52:27 -0500 Subject: [PATCH 043/259] Fix broken link from Docs site to Code of Conduct (#3284) * Fix link * Add clarifications --- docs/support/support.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/support/support.md b/docs/support/support.md index 797aca041f..49075366af 100644 --- a/docs/support/support.md +++ b/docs/support/support.md @@ -5,15 +5,16 @@ description: Support and Community Details and Links --- - [Discord chatroom](https://discord.gg/MUpMjP2) - Get support or discuss the - project + project. - [Good First Issues](https://github.com/backstage/backstage/contribute) - Start - here if you want to contribute + here if you want to contribute. - [RFCs](https://github.com/backstage/backstage/labels/rfc) - Help shape the - technical direction -- [FAQ](../FAQ.md) - Frequently Asked Questions -- [Code of Conduct](../../CODE_OF_CONDUCT.md) - This is how we roll -- [Blog](https://backstage.io/blog/) - Announcements and updates + technical direction by reviewing _Request for Comments_ issues. +- [FAQ](../FAQ.md) - Frequently Asked Questions. +- [Code of Conduct](https://github.com/backstage/backstage/blob/master/CODE_OF_CONDUCT.md) - + This is how we roll. +- [Blog](https://backstage.io/blog/) - Announcements and updates. - [Newsletter](https://mailchi.mp/spotify/backstage-community) - Subscribe to - our email newsletter + our email newsletter. - Give us a star ⭐️ - If you are using Backstage or think it is an interesting - project, we would love a star ❤️ + project, we would love a star! ❤️ From d21e4172b5d4a269244bae30ffd16bff9d2a3c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 13 Nov 2020 10:04:33 +0100 Subject: [PATCH 044/259] auth-backend: change messaging around missing email (#3287) --- plugins/auth-backend/src/providers/auth0/provider.ts | 2 +- plugins/auth-backend/src/providers/oauth2/provider.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/auth-backend/src/providers/auth0/provider.ts b/plugins/auth-backend/src/providers/auth0/provider.ts index 668ab17ee1..f851fb1eb5 100644 --- a/plugins/auth-backend/src/providers/auth0/provider.ts +++ b/plugins/auth-backend/src/providers/auth0/provider.ts @@ -139,7 +139,7 @@ export class Auth0AuthProvider implements OAuthHandlers { const { profile } = response; if (!profile.email) { - throw new Error('Profile does not contain a profile'); + throw new Error('Profile does not contain an email'); } const id = profile.email.split('@')[0]; diff --git a/plugins/auth-backend/src/providers/oauth2/provider.ts b/plugins/auth-backend/src/providers/oauth2/provider.ts index a657ea2195..6342d556e1 100644 --- a/plugins/auth-backend/src/providers/oauth2/provider.ts +++ b/plugins/auth-backend/src/providers/oauth2/provider.ts @@ -148,7 +148,7 @@ export class OAuth2AuthProvider implements OAuthHandlers { const { profile } = response; if (!profile.email) { - throw new Error('Profile does not contain a profile'); + throw new Error('Profile does not contain an email'); } const id = profile.email.split('@')[0]; From f531d307c4b9c8fb98f0d4b1b3006dd6023a8961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 13 Nov 2020 11:16:33 +0100 Subject: [PATCH 045/259] catalog-backend: reject overwrite of entities from foreign locations (#3286) --- .changeset/hip-experts-dance.md | 5 +++++ .../catalog/DatabaseEntitiesCatalog.test.ts | 9 ++++++++- .../src/catalog/DatabaseEntitiesCatalog.ts | 20 +++++++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 .changeset/hip-experts-dance.md diff --git a/.changeset/hip-experts-dance.md b/.changeset/hip-experts-dance.md new file mode 100644 index 0000000000..f88fe62cd4 --- /dev/null +++ b/.changeset/hip-experts-dance.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +An entity A, that exists in the catalog, can no longer be overwritten by registering a different location that also tries to supply an entity with the same kind+namespace+name. Writes of that new entity will instead be rejected with a log message similar to `Rejecting write of entity Component:default/artist-lookup from file:/Users/freben/dev/github/backstage/packages/catalog-model/examples/components/artist-lookup-component.yaml because entity existed from github:https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml` diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts index 00898b33a6..8d2138b3e5 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts @@ -15,7 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import type { Entity } from '@backstage/catalog-model'; +import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model'; import { Database, DatabaseManager, Transaction } from '../database'; import { EntityFilters } from '../service/EntityFilters'; import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; @@ -127,6 +127,9 @@ describe('DatabaseEntitiesCatalog', () => { metadata: { name: 'c', namespace: 'd', + annotations: { + [LOCATION_ANNOTATION]: 'mock', + }, }, }; const dbEntity: Entity = { @@ -135,7 +138,11 @@ describe('DatabaseEntitiesCatalog', () => { metadata: { name: 'c', namespace: 'd', + description: 'changes', uid: 'u', + annotations: { + [LOCATION_ANNOTATION]: 'mock', + }, }, }; db.entities.mockResolvedValue([ diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 5c55a30a55..f40e327eaa 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -205,9 +205,11 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { // TODO(Rugvip): We currently always update relations, but we // likely want to figure out a way to avoid that for (const { entity, relations } of toIgnore) { - const entityId = entity.metadata.uid!; - await this.setRelations(entityId, relations, tx); - modifiedEntityIds.push({ entityId }); + const entityId = entity.metadata.uid; + if (entityId) { + await this.setRelations(entityId, relations, tx); + modifiedEntityIds.push({ entityId }); + } } break; @@ -300,12 +302,22 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { for (const request of requests) { const newEntity = request.entity; const oldEntity = oldEntitiesByName.get(newEntity.metadata.name); + const newLocation = newEntity.metadata.annotations?.[LOCATION_ANNOTATION]; + const oldLocation = + oldEntity?.metadata.annotations?.[LOCATION_ANNOTATION]; if (!oldEntity) { toAdd.push(request); + } else if (oldLocation !== newLocation) { + this.logger.warn( + `Rejecting write of entity ${serializeEntityRef( + newEntity, + )} from ${newLocation} because entity existed from ${oldLocation}`, + ); + toIgnore.push(request); } else if (entityHasChanges(oldEntity, newEntity)) { // TODO(freben): This currently uses addOrUpdateEntity under the hood, // but should probably calculate the end result entity right here - // instead and call a dedicated batch update database method instead + // instead and call a dedicated batch update database method toUpdate.push(request); } else { toIgnore.push(request); From 8c1662fbf9914b831cfd316f436f6dc878b6efbc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Nov 2020 00:14:22 +0100 Subject: [PATCH 046/259] docs: add stability index Co-authored-by: blam --- docs/overview/stability-index.md | 470 +++++++++++++++++++++++++++++++ microsite/sidebars.json | 1 + 2 files changed, 471 insertions(+) create mode 100644 docs/overview/stability-index.md diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md new file mode 100644 index 0000000000..1ca54f150a --- /dev/null +++ b/docs/overview/stability-index.md @@ -0,0 +1,470 @@ +--- +id: stability-index +title: Stability Index +description: + An overview of the commitment to stability for different parts of the + Backstage codebase. +--- + +## Overview + +The purpose of the Backstage Stability Index is to communicate the stability of +various parts of the project. It is tracked using a scoring system where a +higher score indicates a higher level of stability and is a commitment to +smoother transitions between breaking changes. Importantly, the Stability Index +does not supersede [semver](https://semver.org/), meaning we will still adhere +to semver and only do breaking changes in minor releases as long as we are on +`0.x`. + +Each package or section is assigned a stability score between 0 and 3, with each +point building on top of the previous one: + +- **0** - Breaking changes are noted in the changelog, and documentation is + updated. +- **1** - The changelog entry includes a clearly documented upgrade path, + providing guidance for how to migrate previous usage patterns to the new + version. +- **2** - Breaking changes always include a deprecation phase where both the old + and the new APIs can be used in parallel. This deprecation must have been + released for at least two weeks before the deprecated API is removed in a + minor version bump. +- **3** - The time limit for the deprecation is 3 months instead of two weeks. + +TL;DR: + +- **0** - There's a changelog entry. +- **1** - There's a migration guide. +- **2** - 2 weeks of deprecation. +- **3** - 3 months of deprecation. + +## Packages + +### [example-app](https://github.com/backstage/backstage/tree/master/packages/app/) + +This is the `packages/app` package, and it's serves as an example as well as +utility for local development in the main Backstage repo. + +Stability: `N/A` + +### [example-backend](https://github.com/backstage/backstage/tree/master/packages/backend/) + +This is the `packages/backend` package, and it's serves as an example as well as +utility for local development in the main Backstage repo. + +Stability: `N/A` + +### [backend-common](https://github.com/backstage/backstage/tree/master/packages/backend-common/) + +A collection of common helpers to be used by both backend plugins, and for +constructing backend packages. + +Stability: `1` + +### [catalog-client](https://github.com/backstage/backstage/tree/master/packages/catalog-client/) + +An HTTP client for interacting with the catalog backend. Usable both in frontend +and Backend. + +Stability: `0`. This is a very new addition and we have some immediate changes +planned. + +### [catalog-model](https://github.com/backstage/backstage/tree/master/packages/catalog-model/) + +Contains the core catalog model, and utilities for working with entities. Usable +both in frontend and Backend. + +Stability: `2`. The catalog model is evolving, but because of the broad usage we + +want to ensure some stability. + +### [cli](https://github.com/backstage/backstage/tree/master/packages/cli/) + +The main toolchain used for Backstage development. The interface that is +considered for stability are the various commands and options passed to those +commands, as well as the environment variables read by the CLI. The build output +may change over time and is not considered a breaking change unless it is likely +to affect external tooling. + +Stability: `2` + +### [cli-common](https://github.com/backstage/backstage/tree/master/packages/cli-common/) + +Lightweight utilities used by the various Backstage CLIs, not intended for +external use. + +Stability: `N/A` + +### [config](https://github.com/backstage/backstage/tree/master/packages/config/) + +Provides the logic and interfaces for reading static configuration. + +Stability: `2` + +### [config-loader](https://github.com/backstage/backstage/tree/master/packages/config-loader/) + +Used to load in static configuration, mainly for use by the CLI and +@backstage/backend-common. + +Stability: `1`. Mainly intended for internal use. + +### [core](https://github.com/backstage/backstage/tree/master/packages/core/) + +#### Section: React Components + +All of the React components exported from `src/components/` and `src/layout/` + +Stability: `1`. These components have not received a proper review of the API, +but we also want to ensure stability. + +#### Section: Plugin API + +The parts of the core API that are used by plugins, and the way plugins expose +functionality to apps and other plugins. Includes for example `createPlugin`, +`createRouteRef`, `createApiRef`. + +Stability: `2`. There are planned breaking changes around the way that plugins +expose features and do routing. We still commit to keeping a short deprecation +period so that plugins outside of the main repo have time to migrate. + +#### Section: App API + +The APIs used exclusively in the app, such as `createApp` and the system icons. + +Stability: `2` + +#### Section: Utility API Definitions + +The type declarations of the core Utility APIs. + +Stability: `2`. Changes to the Utility API type declarations need time to +propagate. + +#### Section: Utility API Implementations + +The interfaces and default implementations for various Utility APIs, such as +ErrorApi, IdentityApi, the auth APIs, etc. + +Stability: `1`. Most changes to the core utility APIs will not lead to +widespread breaking changes since most apps rely on the default implementations. + +### [core-api](https://github.com/backstage/backstage/tree/master/packages/core-api/) + +The non-visual parts of @backstage/core. Everything in this packages is +re-exported from @backstage/core, and this package should not be used directly. + +Stability: See @backstage/core + +### [create-app](https://github.com/backstage/backstage/tree/master/packages/create-app/) + +The CLI used to scaffold new Backstage projects. + +Stability: `2` + +### [dev-utils](https://github.com/backstage/backstage/tree/master/packages/dev-utils/) + +Provides utilities for developing plugins in isolation. + +Stability: `0`. This package is largely broken and needs updates. + +### [docgen](https://github.com/backstage/backstage/tree/master/packages/docgen/) + +Internal CLI utility for generating API Documentation. + +Stability: `N/A` + +### [e2e-test](https://github.com/backstage/backstage/tree/master/packages/e2e-test/) + +Internal CLI utility for running e2e tests. + +Stability: `N/A` + +### [storybook](https://github.com/backstage/backstage/tree/master/packages/storybook/) + +Internal storybook build for publishing stories to +https://backstage.io/storybook + +Stability: `N/A` + +### [test-utils](https://github.com/backstage/backstage/tree/master/packages/test-utils/) + +Utilities for writing tests for Backstage plugins and apps. + +Stability: `2` + +### [test-utils-core](https://github.com/backstage/backstage/tree/master/packages/test-utils-core/) + +Internal testing utilities that are separated out for usage in +@backstage/core-api. All exports are re-exported by @backstage/test-utils. This +package should not be depended on directly. + +Stability: See @backstage/test-utils + +### [theme](https://github.com/backstage/backstage/tree/master/packages/theme/) + +The core Backstage MUI theme along with customization utilities. + +#### Section: TypeScript + +This is the TypeScript API exported by the theme package. + +Stability: `2` + +#### Section: Visual Theme + +The visual theme exported by the theme packages, where for example changing a +color could be considered a breaking change. + +Stability: `1` + +## Plugins + +Plugins are rarely marked as stable as the `@backstage/core` plugin API is under +heavy development. + +Many backend plugins are split into "REST API" and "TypeScript Interface" +sections. The "TypeScript Interface" refers to the API used to integrate the +plugin into the backend. + +### [api-docs](https://github.com/backstage/backstage/tree/master/plugins/api-docs/) + +Components to discover and display API entities as an extension to the catalog +plugin. + +Stability: `0` + +### [app-backend](https://github.com/backstage/backstage/tree/master/plugins/app-backend/) + +A backend plugin that can be used to serve the frontend app and inject +configuration. + +Stability: `2` + +### [auth-backend](https://github.com/backstage/backstage/tree/master/plugins/auth-backend/) + +A backend plugin that implements the backend portion of the various +authentication flows used in Backstage. + +#### Section: REST API + +Stability: `2` + +#### Section: TypeScript Interface + +Stability: `1` + +### [catalog](https://github.com/backstage/backstage/tree/master/plugins/catalog/) + +The frontend plugin for the catalog, with the table and building blocks for the +entity pages. + +Stability: `1`. We're planning some work to overhaul how entity pages are +constructed. + +### [catalog-backend](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend/) + +The backend API for the catalog, also exposes the processing subsystem for +customization of the catalog. Powers the @backstage/plugin-catalog frontend +plugin. + +#### Section: REST API + +Stability: `1`. There are plans to remove and rework some endpoints. + +#### Section: TypeScript Interface + +Stability: `1`. There are plans to rework parts of the Processor interface. + +### [catalog-graphql](https://github.com/backstage/backstage/tree/master/plugins/catalog-graphql/) + +Provides the catalog schema and resolvers for the graphql backend. + +Stability: `0`. Under heavy development and subject to change. + +### [circleci](https://github.com/backstage/backstage/tree/master/plugins/circleci/) + +Automate your development process with CI hosted in the cloud or on a private +server. + +Stability: `0` + +### [cloudbuild](https://github.com/backstage/backstage/tree/master/plugins/cloudbuild/) + +Visualize Google Cloud Build flows. + +Stability: `0` + +### [cost-insights](https://github.com/backstage/backstage/tree/master/plugins/cost-insights/) + +Visualize, understand and optimize your team's cloud costs. + +Stability: `0` + +### [explore](https://github.com/backstage/backstage/tree/master/plugins/explore/) + +A frontend plugin that introduces the concept of exploring internal and external +tooling in an organization. + +Stability: `0`. Only an example at the moment and not customizable. + +### [gcp-projects](https://github.com/backstage/backstage/tree/master/plugins/gcp-projects/) + +Create, list and manage your Google Cloud Projects. + +Stability: `0` + +### [github-actions](https://github.com/backstage/backstage/tree/master/plugins/github-actions/) + +GitHub Actions makes it easy to automate all your software workflows, now with +world-class CI/CD. Build, test, and deploy your code right from GitHub. + +Stability: `0` + +### [gitops-profiles](https://github.com/backstage/backstage/tree/master/plugins/gitops-profiles/) + +A frontend plugin with a separate backend that can be used to provision EKS +clusters. + +Stability: `0`. This is an early plugin that now has quite a lot of overlap with +the scaffolder plugin. + +### [graphiql](https://github.com/backstage/backstage/tree/master/plugins/graphiql/) + +Integrates GraphiQL as a tool to browse GraphQL API endpoints inside Backstage. + +Stability: `1` + +### [graphql](https://github.com/backstage/backstage/tree/master/plugins/graphql/) + +A backend plugin that provides + +Stability: `0`. Under heavy development and subject to change. + +### [jenkins](https://github.com/backstage/backstage/tree/master/plugins/jenkins/) + +A plugin that visualizes Jenkins workflows for entities. Jenkins offers a simple +way to set up a continuous integration and continuous delivery environment. + +Stability: `0` + +### [kubernetes](https://github.com/backstage/backstage/tree/master/plugins/kubernetes/) + +The frontend component of the Kubernetes plugin, used to browse and visualize +Kubernetes resources. Stability: `1`. + +### [kubernetes-backend](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend/) + +The backend component of the Kubernetes plugin, used to fetch Kubernetes +resources from clusters and associate them with entities in the Catalog. +Stability: `1`. + +### [lighthouse](https://github.com/backstage/backstage/tree/master/plugins/lighthouse/) + +Google's Lighthouse tool is a great resource for benchmarking and improving the +accessibility, performance, SEO, and best practices of your website. + +Stability: `0` + +### [newrelic](https://github.com/backstage/backstage/tree/master/plugins/newrelic/) + +Observability platform built to help engineers create and monitor their +software. Stability: `0` + +### [proxy-backend](https://github.com/backstage/backstage/tree/master/plugins/proxy-backend/) + +A backend plugin used to set up proxying to other endpoints based on static +configuration. + +Stability: `1` + +### [register-component](https://github.com/backstage/backstage/tree/master/plugins/register-component/) + +A frontend plugin that allows the user to register entity locations in the +catalog. + +Stability: `0`. This plugin is likely to be replaced by a generic entity import +plugin instead. + +### [rollbar](https://github.com/backstage/backstage/tree/master/plugins/rollbar/) + +The frontend component of the rollbar plugin, which can be used to view Rollbar +errors for your services in Backstage. + +Stability: `0` + +### [rollbar-backend](https://github.com/backstage/backstage/tree/master/plugins/rollbar-backend/) + +The backend component of the rollbar plugin, which can be used to view Rollbar +errors for your services in Backstage. + +Stability: `0` + +### [scaffolder](https://github.com/backstage/backstage/tree/master/plugins/scaffolder/) + +The frontend scaffolder plugin where one can browse templates and initiate +scaffolding jobs. + +Stability: `1` + +### [scaffolder-backend](https://github.com/backstage/backstage/tree/master/plugins/scaffolder-backend/) + +The backend scaffolder plugin that provides an implementation for templates in +the catalog. + +Stability: `1`. There is planned work to rework the scaffolder in +https://github.com/backstage/backstage/issues/2771. + +### [sentry](https://github.com/backstage/backstage/tree/master/plugins/sentry/) + +The frontend component of the sentry plugin, which can be used to view Sentry +issues in Backstage. + +Stability: `0` + +### [sentry-backend](https://github.com/backstage/backstage/tree/master/plugins/sentry-backend/) + +The backend component of the sentry plugin, which can be used to view Sentry +issues in Backstage. + +Stability: `0` + +### [sonarqube](https://github.com/backstage/backstage/tree/master/plugins/sonarqube/) + +Components to display code quality metrics from SonarCloud and SonarQube. + +Stability: `0` + +### [tech-radar](https://github.com/backstage/backstage/tree/master/plugins/tech-radar/) + +Visualize the your company's official guidelines of different areas of software +development. + +Stability: `0` + +### [techdocs](https://github.com/backstage/backstage/tree/master/plugins/techdocs/) + +The frontend component of the TechDocs plugin, used to browser technical +documentation of entities. + +Stability: `1` + +### [techdocs-backend](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend/) + +The backend component of the TechDocs plugin, used to transform and serve +TechDocs. + +Stability: `1` + +### [user-settings](https://github.com/backstage/backstage/tree/master/plugins/user-settings/) + +A frontend plugin that provides a page where the user can tweak various +settings. + +Stability: `1` + +### [welcome](https://github.com/backstage/backstage/tree/master/plugins/welcome/) + +A plugin that can be used to welcome the user to Backstage. + +Stability: `0`. This used to be the start page for the example app, but has been +replaced by the catalog plugin. It is still viewable at `/welcome` but may be +removed. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 64fdd2f420..f3cf642de1 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -7,6 +7,7 @@ "overview/vision", "overview/background", "overview/adopting", + "overview/stability-index", "overview/logos" ], "Getting Started": [ From 148ecbfcd655193593b5cd8fe3e82f61b8f3f656 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Nov 2020 13:21:42 +0100 Subject: [PATCH 047/259] github/vocab: added words --- .github/styles/vocab.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 3c5fa34adc..efda3a185d 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -20,6 +20,7 @@ Changesets changset chanwit Chanwit +circleci cisphobia cissexist classname @@ -64,6 +65,7 @@ github Github gitlab Gitlab +graphiql graphql graphviz Hackathons @@ -80,6 +82,7 @@ inlinehilite interop javascript Javascript +jenkins jq js json @@ -164,9 +167,11 @@ scaffolded scaffolder Scaffolder semlas +semver Serverless Sinon smartsymobls +sonarqube sparklines Spotifiers spotify @@ -190,6 +195,7 @@ theres toc tolerations Tolerations +toolchain toolsets tooltip touchpoints From a969371bf61d50c620587c176e87520e11c9574e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Nov 2020 13:51:49 +0100 Subject: [PATCH 048/259] docs/stability-index: fix a grammar --- docs/overview/stability-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index 1ca54f150a..e410015662 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -442,7 +442,7 @@ Stability: `0` ### [techdocs](https://github.com/backstage/backstage/tree/master/plugins/techdocs/) -The frontend component of the TechDocs plugin, used to browser technical +The frontend component of the TechDocs plugin, used to browse technical documentation of entities. Stability: `1` From b4c9e09b87aa29b6c9be9284682f98e09aa66d37 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Nov 2020 13:54:49 +0100 Subject: [PATCH 049/259] docs/stability-index: wrap package titles in code blocks --- .github/styles/vocab.txt | 4 -- docs/overview/stability-index.md | 106 +++++++++++++++---------------- 2 files changed, 53 insertions(+), 57 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index efda3a185d..bd89163f1f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -20,7 +20,6 @@ Changesets changset chanwit Chanwit -circleci cisphobia cissexist classname @@ -65,7 +64,6 @@ github Github gitlab Gitlab -graphiql graphql graphviz Hackathons @@ -82,7 +80,6 @@ inlinehilite interop javascript Javascript -jenkins jq js json @@ -171,7 +168,6 @@ semver Serverless Sinon smartsymobls -sonarqube sparklines Spotifiers spotify diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index e410015662..96ff33542f 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -39,28 +39,28 @@ TL;DR: ## Packages -### [example-app](https://github.com/backstage/backstage/tree/master/packages/app/) +### [`example-app`](https://github.com/backstage/backstage/tree/master/packages/app/) This is the `packages/app` package, and it's serves as an example as well as utility for local development in the main Backstage repo. Stability: `N/A` -### [example-backend](https://github.com/backstage/backstage/tree/master/packages/backend/) +### [`example-backend`](https://github.com/backstage/backstage/tree/master/packages/backend/) This is the `packages/backend` package, and it's serves as an example as well as utility for local development in the main Backstage repo. Stability: `N/A` -### [backend-common](https://github.com/backstage/backstage/tree/master/packages/backend-common/) +### [`backend-common`](https://github.com/backstage/backstage/tree/master/packages/backend-common/) A collection of common helpers to be used by both backend plugins, and for constructing backend packages. Stability: `1` -### [catalog-client](https://github.com/backstage/backstage/tree/master/packages/catalog-client/) +### [`catalog-client`](https://github.com/backstage/backstage/tree/master/packages/catalog-client/) An HTTP client for interacting with the catalog backend. Usable both in frontend and Backend. @@ -68,7 +68,7 @@ and Backend. Stability: `0`. This is a very new addition and we have some immediate changes planned. -### [catalog-model](https://github.com/backstage/backstage/tree/master/packages/catalog-model/) +### [`catalog-model`](https://github.com/backstage/backstage/tree/master/packages/catalog-model/) Contains the core catalog model, and utilities for working with entities. Usable both in frontend and Backend. @@ -77,7 +77,7 @@ Stability: `2`. The catalog model is evolving, but because of the broad usage we want to ensure some stability. -### [cli](https://github.com/backstage/backstage/tree/master/packages/cli/) +### [`cli`](https://github.com/backstage/backstage/tree/master/packages/cli/) The main toolchain used for Backstage development. The interface that is considered for stability are the various commands and options passed to those @@ -87,27 +87,27 @@ to affect external tooling. Stability: `2` -### [cli-common](https://github.com/backstage/backstage/tree/master/packages/cli-common/) +### [`cli-common`](https://github.com/backstage/backstage/tree/master/packages/cli-common/) Lightweight utilities used by the various Backstage CLIs, not intended for external use. Stability: `N/A` -### [config](https://github.com/backstage/backstage/tree/master/packages/config/) +### [`config`](https://github.com/backstage/backstage/tree/master/packages/config/) Provides the logic and interfaces for reading static configuration. Stability: `2` -### [config-loader](https://github.com/backstage/backstage/tree/master/packages/config-loader/) +### [`config-loader`](https://github.com/backstage/backstage/tree/master/packages/config-loader/) Used to load in static configuration, mainly for use by the CLI and @backstage/backend-common. Stability: `1`. Mainly intended for internal use. -### [core](https://github.com/backstage/backstage/tree/master/packages/core/) +### [`core`](https://github.com/backstage/backstage/tree/master/packages/core/) #### Section: React Components @@ -147,51 +147,51 @@ ErrorApi, IdentityApi, the auth APIs, etc. Stability: `1`. Most changes to the core utility APIs will not lead to widespread breaking changes since most apps rely on the default implementations. -### [core-api](https://github.com/backstage/backstage/tree/master/packages/core-api/) +### [`core-api`](https://github.com/backstage/backstage/tree/master/packages/core-api/) The non-visual parts of @backstage/core. Everything in this packages is re-exported from @backstage/core, and this package should not be used directly. Stability: See @backstage/core -### [create-app](https://github.com/backstage/backstage/tree/master/packages/create-app/) +### [`create-app`](https://github.com/backstage/backstage/tree/master/packages/create-app/) The CLI used to scaffold new Backstage projects. Stability: `2` -### [dev-utils](https://github.com/backstage/backstage/tree/master/packages/dev-utils/) +### [`dev-utils`](https://github.com/backstage/backstage/tree/master/packages/dev-utils/) Provides utilities for developing plugins in isolation. Stability: `0`. This package is largely broken and needs updates. -### [docgen](https://github.com/backstage/backstage/tree/master/packages/docgen/) +### [`docgen`](https://github.com/backstage/backstage/tree/master/packages/docgen/) Internal CLI utility for generating API Documentation. Stability: `N/A` -### [e2e-test](https://github.com/backstage/backstage/tree/master/packages/e2e-test/) +### [`e2e-test`](https://github.com/backstage/backstage/tree/master/packages/e2e-test/) Internal CLI utility for running e2e tests. Stability: `N/A` -### [storybook](https://github.com/backstage/backstage/tree/master/packages/storybook/) +### [`storybook`](https://github.com/backstage/backstage/tree/master/packages/storybook/) Internal storybook build for publishing stories to https://backstage.io/storybook Stability: `N/A` -### [test-utils](https://github.com/backstage/backstage/tree/master/packages/test-utils/) +### [`test-utils`](https://github.com/backstage/backstage/tree/master/packages/test-utils/) Utilities for writing tests for Backstage plugins and apps. Stability: `2` -### [test-utils-core](https://github.com/backstage/backstage/tree/master/packages/test-utils-core/) +### [`test-utils-core`](https://github.com/backstage/backstage/tree/master/packages/test-utils-core/) Internal testing utilities that are separated out for usage in @backstage/core-api. All exports are re-exported by @backstage/test-utils. This @@ -199,7 +199,7 @@ package should not be depended on directly. Stability: See @backstage/test-utils -### [theme](https://github.com/backstage/backstage/tree/master/packages/theme/) +### [`theme`](https://github.com/backstage/backstage/tree/master/packages/theme/) The core Backstage MUI theme along with customization utilities. @@ -225,21 +225,21 @@ Many backend plugins are split into "REST API" and "TypeScript Interface" sections. The "TypeScript Interface" refers to the API used to integrate the plugin into the backend. -### [api-docs](https://github.com/backstage/backstage/tree/master/plugins/api-docs/) +### [`api-docs`](https://github.com/backstage/backstage/tree/master/plugins/api-docs/) Components to discover and display API entities as an extension to the catalog plugin. Stability: `0` -### [app-backend](https://github.com/backstage/backstage/tree/master/plugins/app-backend/) +### [`app-backend`](https://github.com/backstage/backstage/tree/master/plugins/app-backend/) A backend plugin that can be used to serve the frontend app and inject configuration. Stability: `2` -### [auth-backend](https://github.com/backstage/backstage/tree/master/plugins/auth-backend/) +### [`auth-backend`](https://github.com/backstage/backstage/tree/master/plugins/auth-backend/) A backend plugin that implements the backend portion of the various authentication flows used in Backstage. @@ -252,7 +252,7 @@ Stability: `2` Stability: `1` -### [catalog](https://github.com/backstage/backstage/tree/master/plugins/catalog/) +### [`catalog`](https://github.com/backstage/backstage/tree/master/plugins/catalog/) The frontend plugin for the catalog, with the table and building blocks for the entity pages. @@ -260,7 +260,7 @@ entity pages. Stability: `1`. We're planning some work to overhaul how entity pages are constructed. -### [catalog-backend](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend/) +### [`catalog-backend`](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend/) The backend API for the catalog, also exposes the processing subsystem for customization of the catalog. Powers the @backstage/plugin-catalog frontend @@ -274,52 +274,52 @@ Stability: `1`. There are plans to remove and rework some endpoints. Stability: `1`. There are plans to rework parts of the Processor interface. -### [catalog-graphql](https://github.com/backstage/backstage/tree/master/plugins/catalog-graphql/) +### [`catalog-graphql`](https://github.com/backstage/backstage/tree/master/plugins/catalog-graphql/) Provides the catalog schema and resolvers for the graphql backend. Stability: `0`. Under heavy development and subject to change. -### [circleci](https://github.com/backstage/backstage/tree/master/plugins/circleci/) +### [`circleci`](https://github.com/backstage/backstage/tree/master/plugins/circleci/) Automate your development process with CI hosted in the cloud or on a private server. Stability: `0` -### [cloudbuild](https://github.com/backstage/backstage/tree/master/plugins/cloudbuild/) +### [`cloudbuild`](https://github.com/backstage/backstage/tree/master/plugins/cloudbuild/) Visualize Google Cloud Build flows. Stability: `0` -### [cost-insights](https://github.com/backstage/backstage/tree/master/plugins/cost-insights/) +### [`cost-insights`](https://github.com/backstage/backstage/tree/master/plugins/cost-insights/) Visualize, understand and optimize your team's cloud costs. Stability: `0` -### [explore](https://github.com/backstage/backstage/tree/master/plugins/explore/) +### [`explore`](https://github.com/backstage/backstage/tree/master/plugins/explore/) A frontend plugin that introduces the concept of exploring internal and external tooling in an organization. Stability: `0`. Only an example at the moment and not customizable. -### [gcp-projects](https://github.com/backstage/backstage/tree/master/plugins/gcp-projects/) +### [`gcp-projects`](https://github.com/backstage/backstage/tree/master/plugins/gcp-projects/) Create, list and manage your Google Cloud Projects. Stability: `0` -### [github-actions](https://github.com/backstage/backstage/tree/master/plugins/github-actions/) +### [`github-actions`](https://github.com/backstage/backstage/tree/master/plugins/github-actions/) GitHub Actions makes it easy to automate all your software workflows, now with world-class CI/CD. Build, test, and deploy your code right from GitHub. Stability: `0` -### [gitops-profiles](https://github.com/backstage/backstage/tree/master/plugins/gitops-profiles/) +### [`gitops-profiles`](https://github.com/backstage/backstage/tree/master/plugins/gitops-profiles/) A frontend plugin with a separate backend that can be used to provision EKS clusters. @@ -327,56 +327,56 @@ clusters. Stability: `0`. This is an early plugin that now has quite a lot of overlap with the scaffolder plugin. -### [graphiql](https://github.com/backstage/backstage/tree/master/plugins/graphiql/) +### [`graphiql`](https://github.com/backstage/backstage/tree/master/plugins/graphiql/) Integrates GraphiQL as a tool to browse GraphQL API endpoints inside Backstage. Stability: `1` -### [graphql](https://github.com/backstage/backstage/tree/master/plugins/graphql/) +### [`graphql`](https://github.com/backstage/backstage/tree/master/plugins/graphql/) A backend plugin that provides Stability: `0`. Under heavy development and subject to change. -### [jenkins](https://github.com/backstage/backstage/tree/master/plugins/jenkins/) +### [`jenkins`](https://github.com/backstage/backstage/tree/master/plugins/jenkins/) A plugin that visualizes Jenkins workflows for entities. Jenkins offers a simple way to set up a continuous integration and continuous delivery environment. Stability: `0` -### [kubernetes](https://github.com/backstage/backstage/tree/master/plugins/kubernetes/) +### [`kubernetes`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes/) The frontend component of the Kubernetes plugin, used to browse and visualize Kubernetes resources. Stability: `1`. -### [kubernetes-backend](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend/) +### [`kubernetes-backend`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend/) The backend component of the Kubernetes plugin, used to fetch Kubernetes resources from clusters and associate them with entities in the Catalog. Stability: `1`. -### [lighthouse](https://github.com/backstage/backstage/tree/master/plugins/lighthouse/) +### [`lighthouse`](https://github.com/backstage/backstage/tree/master/plugins/lighthouse/) Google's Lighthouse tool is a great resource for benchmarking and improving the accessibility, performance, SEO, and best practices of your website. Stability: `0` -### [newrelic](https://github.com/backstage/backstage/tree/master/plugins/newrelic/) +### [`newrelic`](https://github.com/backstage/backstage/tree/master/plugins/newrelic/) Observability platform built to help engineers create and monitor their software. Stability: `0` -### [proxy-backend](https://github.com/backstage/backstage/tree/master/plugins/proxy-backend/) +### [`proxy-backend`](https://github.com/backstage/backstage/tree/master/plugins/proxy-backend/) A backend plugin used to set up proxying to other endpoints based on static configuration. Stability: `1` -### [register-component](https://github.com/backstage/backstage/tree/master/plugins/register-component/) +### [`register-component`](https://github.com/backstage/backstage/tree/master/plugins/register-component/) A frontend plugin that allows the user to register entity locations in the catalog. @@ -384,28 +384,28 @@ catalog. Stability: `0`. This plugin is likely to be replaced by a generic entity import plugin instead. -### [rollbar](https://github.com/backstage/backstage/tree/master/plugins/rollbar/) +### [`rollbar`](https://github.com/backstage/backstage/tree/master/plugins/rollbar/) The frontend component of the rollbar plugin, which can be used to view Rollbar errors for your services in Backstage. Stability: `0` -### [rollbar-backend](https://github.com/backstage/backstage/tree/master/plugins/rollbar-backend/) +### [`rollbar-backend`](https://github.com/backstage/backstage/tree/master/plugins/rollbar-backend/) The backend component of the rollbar plugin, which can be used to view Rollbar errors for your services in Backstage. Stability: `0` -### [scaffolder](https://github.com/backstage/backstage/tree/master/plugins/scaffolder/) +### [`scaffolder`](https://github.com/backstage/backstage/tree/master/plugins/scaffolder/) The frontend scaffolder plugin where one can browse templates and initiate scaffolding jobs. Stability: `1` -### [scaffolder-backend](https://github.com/backstage/backstage/tree/master/plugins/scaffolder-backend/) +### [`scaffolder-backend`](https://github.com/backstage/backstage/tree/master/plugins/scaffolder-backend/) The backend scaffolder plugin that provides an implementation for templates in the catalog. @@ -413,55 +413,55 @@ the catalog. Stability: `1`. There is planned work to rework the scaffolder in https://github.com/backstage/backstage/issues/2771. -### [sentry](https://github.com/backstage/backstage/tree/master/plugins/sentry/) +### [`sentry`](https://github.com/backstage/backstage/tree/master/plugins/sentry/) The frontend component of the sentry plugin, which can be used to view Sentry issues in Backstage. Stability: `0` -### [sentry-backend](https://github.com/backstage/backstage/tree/master/plugins/sentry-backend/) +### [`sentry-backend`](https://github.com/backstage/backstage/tree/master/plugins/sentry-backend/) The backend component of the sentry plugin, which can be used to view Sentry issues in Backstage. Stability: `0` -### [sonarqube](https://github.com/backstage/backstage/tree/master/plugins/sonarqube/) +### [`sonarqube`](https://github.com/backstage/backstage/tree/master/plugins/sonarqube/) Components to display code quality metrics from SonarCloud and SonarQube. Stability: `0` -### [tech-radar](https://github.com/backstage/backstage/tree/master/plugins/tech-radar/) +### [`tech-radar`](https://github.com/backstage/backstage/tree/master/plugins/tech-radar/) Visualize the your company's official guidelines of different areas of software development. Stability: `0` -### [techdocs](https://github.com/backstage/backstage/tree/master/plugins/techdocs/) +### [`techdocs`](https://github.com/backstage/backstage/tree/master/plugins/techdocs/) The frontend component of the TechDocs plugin, used to browse technical documentation of entities. Stability: `1` -### [techdocs-backend](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend/) +### [`techdocs-backend`](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend/) The backend component of the TechDocs plugin, used to transform and serve TechDocs. Stability: `1` -### [user-settings](https://github.com/backstage/backstage/tree/master/plugins/user-settings/) +### [`user-settings`](https://github.com/backstage/backstage/tree/master/plugins/user-settings/) A frontend plugin that provides a page where the user can tweak various settings. Stability: `1` -### [welcome](https://github.com/backstage/backstage/tree/master/plugins/welcome/) +### [`welcome`](https://github.com/backstage/backstage/tree/master/plugins/welcome/) A plugin that can be used to welcome the user to Backstage. From 7a46fa07d14decc197cca72806f68ea3571b43b6 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 13 Nov 2020 15:22:24 +0100 Subject: [PATCH 050/259] TechDocs project: Use org token GH_SERVICE_ACCOUNT_TOKEN --- .github/workflows/techdocs-project-board.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/techdocs-project-board.yml b/.github/workflows/techdocs-project-board.yml index a2fb980259..a8d476f713 100644 --- a/.github/workflows/techdocs-project-board.yml +++ b/.github/workflows/techdocs-project-board.yml @@ -10,7 +10,7 @@ on: types: [opened, reopened, labeled, edited] env: - MY_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MY_GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} jobs: assign_issue_or_pr_to_project: From 053fc56745b5cc4e6209222032934729b06da047 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 10:15:32 -0500 Subject: [PATCH 051/259] Move out utility functions for mock data to utils --- plugins/cost-insights/src/client.ts | 39 +++------------------ plugins/cost-insights/src/utils/mockData.ts | 30 ++++++++++++++++ 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/plugins/cost-insights/src/client.ts b/plugins/cost-insights/src/client.ts index 740a44fcbd..74d4954ea4 100644 --- a/plugins/cost-insights/src/client.ts +++ b/plugins/cost-insights/src/client.ts @@ -16,11 +16,9 @@ /* eslint-disable no-restricted-imports */ import dayjs from 'dayjs'; -import regression, { DataPoint } from 'regression'; import { CostInsightsApi, ProductInsightsOptions } from '../src/api'; import { Alert, - ChangeStatistic, Cost, DateAggregation, DEFAULT_DATE_FORMAT, @@ -30,24 +28,21 @@ import { MetricData, Project, ProjectGrowthData, - Trendline, UnlabeledDataflowData, } from '../src/types'; import { ProjectGrowthAlert, UnlabeledDataflowAlert, } from '../src/utils/alerts'; -import { - exclusiveEndDateOf, - inclusiveStartDateOf, -} from '../src/utils/duration'; +import { inclusiveStartDateOf } from '../src/utils/duration'; +import { trendlineOf, changeOf } from './utils/mockData'; type IntervalFields = { duration: Duration; endDate: string; }; -function parseIntervals(intervals: string): IntervalFields { +export function parseIntervals(intervals: string): IntervalFields { const match = intervals.match( /\/(?P\d+[DM])\/(?\d{4}-\d{2}-\d{2})/, ); @@ -66,7 +61,7 @@ function aggregationFor( baseline: number, ): DateAggregation[] { const { duration, endDate } = parseIntervals(intervals); - const days = dayjs(exclusiveEndDateOf(duration, endDate)).diff( + const days = dayjs(endDate).diff( inclusiveStartDateOf(duration, endDate), 'day', ); @@ -86,32 +81,6 @@ function aggregationFor( ); } -function trendlineOf(aggregation: DateAggregation[]): Trendline { - const data: ReadonlyArray = aggregation.map(a => [ - Date.parse(a.date) / 1000, - a.amount, - ]); - const result = regression.linear(data, { precision: 5 }); - return { - slope: result.equation[0], - intercept: result.equation[1], - }; -} - -function changeOf(aggregation: DateAggregation[]): ChangeStatistic { - const half = Math.ceil(aggregation.length / 2); - const before = aggregation - .slice(0, half) - .reduce((sum, a) => sum + a.amount, 0); - const after = aggregation - .slice(half, aggregation.length) - .reduce((sum, a) => sum + a.amount, 0); - return { - ratio: (after - before) / before, - amount: after - before, - }; -} - export class ExampleCostInsightsClient implements CostInsightsApi { private request(_: any, res: any): Promise { return new Promise(resolve => setTimeout(resolve, 0, res)); diff --git a/plugins/cost-insights/src/utils/mockData.ts b/plugins/cost-insights/src/utils/mockData.ts index ddeb4636b9..f5ad21854a 100644 --- a/plugins/cost-insights/src/utils/mockData.ts +++ b/plugins/cost-insights/src/utils/mockData.ts @@ -14,16 +14,20 @@ * limitations under the License. */ +import regression, { DataPoint } from 'regression'; import { Config } from '@backstage/config'; import { ConfigApi } from '@backstage/core'; import { + ChangeStatistic, Duration, Entity, Product, ProductFilters, ProjectGrowthData, + Trendline, UnlabeledDataflowAlertProject, UnlabeledDataflowData, + DateAggregation, } from '../types'; import { DefaultLoadingAction, @@ -185,3 +189,29 @@ export const MockCostInsightsConfig: Partial = { getConfig: () => MockProductsConfig as Config, getOptionalConfig: () => MockMetricsConfig as Config, }; + +export function trendlineOf(aggregation: DateAggregation[]): Trendline { + const data: ReadonlyArray = aggregation.map(a => [ + Date.parse(a.date) / 1000, + a.amount, + ]); + const result = regression.linear(data, { precision: 5 }); + return { + slope: result.equation[0], + intercept: result.equation[1], + }; +} + +export function changeOf(aggregation: DateAggregation[]): ChangeStatistic { + const half = Math.ceil(aggregation.length / 2); + const before = aggregation + .slice(0, half) + .reduce((sum, a) => sum + a.amount, 0); + const after = aggregation + .slice(half, aggregation.length) + .reduce((sum, a) => sum + a.amount, 0); + return { + ratio: (after - before) / before, + amount: after - before, + }; +} From 3ec3fc48a7a7065e454309109a9f50b94c0ca4e2 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 10:15:50 -0500 Subject: [PATCH 052/259] Fix compared change calculation --- plugins/cost-insights/src/utils/change.ts | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/cost-insights/src/utils/change.ts b/plugins/cost-insights/src/utils/change.ts index cc797bb3b0..79f653158a 100644 --- a/plugins/cost-insights/src/utils/change.ts +++ b/plugins/cost-insights/src/utils/change.ts @@ -21,8 +21,10 @@ import { EngineerThreshold, GrowthType, MetricData, + Duration, } from '../types'; import { aggregationSort } from '../utils/sort'; +import moment from 'moment'; // Used for displaying status colors export function growthOf(ratio: number, amount?: number) { @@ -45,11 +47,29 @@ export function growthOf(ratio: number, amount?: number) { export function getComparedChange( dailyCost: Cost, metricData: MetricData, + duration: Duration, ): ChangeStatistic { const ratio = dailyCost.change.ratio - metricData.change.ratio; - const amount = dailyCost.aggregation.slice().sort(aggregationSort)[0].amount; + const previousPeriodTotal = getPreviousPeriodTotalCost(dailyCost, duration); return { ratio: ratio, - amount: amount * ratio, + amount: previousPeriodTotal * ratio, }; } + +export function getPreviousPeriodTotalCost( + dailyCost: Cost, + duration: Duration, +): number { + const costsByDate = dailyCost.aggregation.slice().sort(aggregationSort); + const nextPeriodStart = moment(costsByDate[0].date).add( + moment.duration(duration), + ); + + // Add up costs that incurred before the start of the next period. + return costsByDate.reduce((acc, costByDate) => { + return moment(costByDate.date).isBefore(nextPeriodStart) + ? acc + costByDate.amount + : acc; + }, 0); +} From 83e3df8a579020d8cfe17e4cac6453f557daa2b7 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 10:16:38 -0500 Subject: [PATCH 053/259] Add mock daily cost data for test purposes --- .../CostOverviewCard/CostOverviewCard.tsx | 2 +- .../cost-insights/src/utils/change.test.ts | 25 +- plugins/cost-insights/src/utils/mockData.ts | 247 ++++++++++++++++++ 3 files changed, 271 insertions(+), 3 deletions(-) diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx index 8f52a0070b..358c7a98aa 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx @@ -50,7 +50,7 @@ export const CostOverviewCard = ({ ? findAlways(config.metrics, m => m.kind === filters.metric) : null; const comparedChange = metricData - ? getComparedChange(dailyCostData, metricData) + ? getComparedChange(dailyCostData, metricData, filters.duration) : null; return ( diff --git a/plugins/cost-insights/src/utils/change.test.ts b/plugins/cost-insights/src/utils/change.test.ts index daf4538f2c..709f2dd882 100644 --- a/plugins/cost-insights/src/utils/change.test.ts +++ b/plugins/cost-insights/src/utils/change.test.ts @@ -14,8 +14,15 @@ * limitations under the License. */ -import { growthOf } from './change'; -import { GrowthType, ChangeThreshold, EngineerThreshold } from '../types'; +import { growthOf, getPreviousPeriodTotalCost } from './change'; +import { + GrowthType, + ChangeThreshold, + EngineerThreshold, + Duration, + Cost, +} from '../types'; +import { MockAggregatedDailyCosts, trendlineOf, changeOf } from './mockData'; const GrowthMap = { [GrowthType.Negligible]: 'negligible growth', @@ -60,3 +67,17 @@ describe.each` }); }, ); + +describe('getPreviousPeriodTotalCost', () => { + it('Correctly returns the total cost for the previous period given daily costs', () => { + const mockGroupDailyCost: Cost = { + id: 'test-group', + aggregation: MockAggregatedDailyCosts, + change: changeOf(MockAggregatedDailyCosts), + trendline: trendlineOf(MockAggregatedDailyCosts), + }; + expect( + getPreviousPeriodTotalCost(mockGroupDailyCost, Duration.P1M), + ).toEqual(100_000); + }); +}); diff --git a/plugins/cost-insights/src/utils/mockData.ts b/plugins/cost-insights/src/utils/mockData.ts index f5ad21854a..708ce733dd 100644 --- a/plugins/cost-insights/src/utils/mockData.ts +++ b/plugins/cost-insights/src/utils/mockData.ts @@ -215,3 +215,250 @@ export function changeOf(aggregation: DateAggregation[]): ChangeStatistic { amount: after - before, }; } + +export const MockAggregatedDailyCosts: DateAggregation[] = [ + { + date: '2020-08-07', + amount: 3500, + }, + { + date: '2020-08-06', + amount: 2500, + }, + { + date: '2020-08-05', + amount: 1400, + }, + { + date: '2020-08-04', + amount: 3800, + }, + { + date: '2020-08-09', + amount: 1900, + }, + { + date: '2020-08-08', + amount: 2400, + }, + { + date: '2020-08-03', + amount: 4000, + }, + { + date: '2020-08-02', + amount: 3700, + }, + { + date: '2020-08-01', + amount: 2500, + }, + { + date: '2020-08-18', + amount: 4300, + }, + { + date: '2020-08-17', + amount: 1500, + }, + { + date: '2020-08-16', + amount: 3600, + }, + { + date: '2020-08-15', + amount: 2200, + }, + { + date: '2020-08-19', + amount: 3900, + }, + { + date: '2020-08-10', + amount: 4100, + }, + { + date: '2020-08-14', + amount: 3600, + }, + { + date: '2020-08-13', + amount: 2900, + }, + { + date: '2020-08-12', + amount: 2700, + }, + { + date: '2020-08-11', + amount: 5100, + }, + { + date: '2020-09-19', + amount: 1200, + }, + { + date: '2020-09-18', + amount: 6500, + }, + { + date: '2020-09-17', + amount: 2500, + }, + { + date: '2020-09-16', + amount: 1400, + }, + { + date: '2020-09-11', + amount: 2300, + }, + { + date: '2020-09-10', + amount: 1900, + }, + { + date: '2020-09-15', + amount: 3100, + }, + { + date: '2020-09-14', + amount: 4500, + }, + { + date: '2020-09-13', + amount: 3300, + }, + { + date: '2020-09-12', + amount: 2800, + }, + { + date: '2020-09-29', + amount: 2600, + }, + { + date: '2020-09-28', + amount: 4100, + }, + { + date: '2020-09-27', + amount: 3800, + }, + { + date: '2020-09-22', + amount: 3700, + }, + { + date: '2020-09-21', + amount: 2700, + }, + { + date: '2020-09-20', + amount: 2200, + }, + { + date: '2020-09-26', + amount: 3300, + }, + { + date: '2020-09-25', + amount: 4000, + }, + { + date: '2020-09-24', + amount: 3800, + }, + { + date: '2020-09-23', + amount: 4100, + }, + { + date: '2020-08-29', + amount: 4400, + }, + { + date: '2020-08-28', + amount: 5000, + }, + { + date: '2020-08-27', + amount: 4900, + }, + { + date: '2020-08-26', + amount: 4100, + }, + { + date: '2020-08-21', + amount: 3700, + }, + { + date: '2020-08-20', + amount: 2200, + }, + { + date: '2020-08-25', + amount: 1700, + }, + { + date: '2020-08-24', + amount: 2100, + }, + { + date: '2020-08-23', + amount: 3100, + }, + { + date: '2020-08-22', + amount: 1500, + }, + { + date: '2020-09-08', + amount: 2900, + }, + { + date: '2020-09-07', + amount: 4100, + }, + { + date: '2020-09-06', + amount: 3600, + }, + { + date: '2020-09-05', + amount: 3300, + }, + { + date: '2020-09-09', + amount: 2800, + }, + { + date: '2020-08-31', + amount: 3400, + }, + { + date: '2020-08-30', + amount: 4300, + }, + { + date: '2020-09-04', + amount: 6100, + }, + { + date: '2020-09-03', + amount: 2500, + }, + { + date: '2020-09-02', + amount: 4900, + }, + { + date: '2020-09-01', + amount: 6100, + }, + { + date: '2020-09-30', + amount: 5500, + }, +]; From 43396127893cd1d6162b7090f6b27bec18bddc3c Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 15:15:58 -0500 Subject: [PATCH 054/259] Update compared change calculation to use last billing date --- plugins/cost-insights/src/utils/change.ts | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/plugins/cost-insights/src/utils/change.ts b/plugins/cost-insights/src/utils/change.ts index 79f653158a..a5a0a86f0a 100644 --- a/plugins/cost-insights/src/utils/change.ts +++ b/plugins/cost-insights/src/utils/change.ts @@ -25,6 +25,11 @@ import { } from '../types'; import { aggregationSort } from '../utils/sort'; import moment from 'moment'; +import dayjs, { OpUnitType } from 'dayjs'; +import duration, { DurationInputType } from 'dayjs/plugin/duration'; +import { inclusiveStartDateOf } from './duration'; + +dayjs.extend(duration); // Used for displaying status colors export function growthOf(ratio: number, amount?: number) { @@ -48,9 +53,14 @@ export function getComparedChange( dailyCost: Cost, metricData: MetricData, duration: Duration, + lastCompleteBillingDate: string, // YYYY-MM-DD, ): ChangeStatistic { const ratio = dailyCost.change.ratio - metricData.change.ratio; - const previousPeriodTotal = getPreviousPeriodTotalCost(dailyCost, duration); + const previousPeriodTotal = getPreviousPeriodTotalCost( + dailyCost, + duration, + lastCompleteBillingDate, + ); return { ratio: ratio, amount: previousPeriodTotal * ratio, @@ -60,15 +70,18 @@ export function getComparedChange( export function getPreviousPeriodTotalCost( dailyCost: Cost, duration: Duration, + endDate: string, ): number { - const costsByDate = dailyCost.aggregation.slice().sort(aggregationSort); - const nextPeriodStart = moment(costsByDate[0].date).add( - moment.duration(duration), - ); + const dayjsDuration = dayjs.duration(duration); + const startDate = inclusiveStartDateOf(duration, endDate); + const [amount, type]: [number, OpUnitType] = dayjsDuration.days() + ? [dayjsDuration.days(), 'days' as OpUnitType] + : [dayjsDuration.months(), 'months' as OpUnitType]; + const nextPeriodStart = dayjs(startDate).add(amount, type); // Add up costs that incurred before the start of the next period. - return costsByDate.reduce((acc, costByDate) => { - return moment(costByDate.date).isBefore(nextPeriodStart) + return dailyCost.aggregation.reduce((acc, costByDate) => { + return dayjs(costByDate.date).isBefore(nextPeriodStart) ? acc + costByDate.amount : acc; }, 0); From 3a86e69f8cf773a1306828e9ab41150105a8336d Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 15:16:15 -0500 Subject: [PATCH 055/259] Fix cost chart tooltip date label --- .../src/components/CostOverviewCard/CostOverviewChart.tsx | 7 +++++-- plugins/cost-insights/src/utils/change.test.ts | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx index 6ed72923ff..1910013581 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewChart.tsx @@ -14,7 +14,8 @@ * limitations under the License. */ import React from 'react'; -import moment from 'moment'; +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc'; import { useTheme } from '@material-ui/core'; import { ComposedChart, @@ -50,6 +51,8 @@ import { useCostOverviewStyles as useStyles } from '../../utils/styles'; import { groupByDate, toDataMax, trendFrom } from '../../utils/charts'; import { aggregationSort } from '../../utils/sort'; +dayjs.extend(utc); + type CostOverviewChartProps = { metric: Maybe; metricData: Maybe; @@ -104,7 +107,7 @@ export const CostOverviewChart = ({ if (isInvalid({ label, payload })) return null; const dataKeys = [data.dailyCost.dataKey, data.metric.dataKey]; - const title = moment(label).format(DEFAULT_DATE_FORMAT); + const title = dayjs(label).utc().format(DEFAULT_DATE_FORMAT); const items = payload .filter(p => dataKeys.includes(p.dataKey as string)) .map(p => ({ diff --git a/plugins/cost-insights/src/utils/change.test.ts b/plugins/cost-insights/src/utils/change.test.ts index 709f2dd882..94adc25f64 100644 --- a/plugins/cost-insights/src/utils/change.test.ts +++ b/plugins/cost-insights/src/utils/change.test.ts @@ -76,8 +76,13 @@ describe('getPreviousPeriodTotalCost', () => { change: changeOf(MockAggregatedDailyCosts), trendline: trendlineOf(MockAggregatedDailyCosts), }; + const exclusiveEndDate = '2020-10-01'; expect( - getPreviousPeriodTotalCost(mockGroupDailyCost, Duration.P1M), + getPreviousPeriodTotalCost( + mockGroupDailyCost, + Duration.P1M, + exclusiveEndDate, + ), ).toEqual(100_000); }); }); From da883b1fb11b57ea883985f7452cc507c2202dc1 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 15:16:32 -0500 Subject: [PATCH 056/259] Pass in last complete billing date to compared change calculation --- .../CostOverviewCard/CostOverviewCard.tsx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx index 358c7a98aa..81dc118ea9 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx @@ -22,7 +22,12 @@ import { CostOverviewHeader } from './CostOverviewHeader'; import { LegendItem } from '../LegendItem'; import { MetricSelect } from '../MetricSelect'; import { PeriodSelect } from '../PeriodSelect'; -import { useScroll, useFilters, useConfig } from '../../hooks'; +import { + useScroll, + useFilters, + useConfig, + useLastCompleteBillingDate, +} from '../../hooks'; import { mapFiltersToProps } from './selector'; import { DefaultNavigation } from '../../utils/navigation'; import { formatPercent } from '../../utils/formatters'; @@ -41,6 +46,7 @@ export const CostOverviewCard = ({ }: CostOverviewCardProps) => { const theme = useTheme(); const config = useConfig(); + const lastCompleteBillingDate = useLastCompleteBillingDate(); const { ScrollAnchor } = useScroll(DefaultNavigation.CostOverviewCard); const { setDuration, setProject, setMetric, ...filters } = useFilters( mapFiltersToProps, @@ -50,7 +56,12 @@ export const CostOverviewCard = ({ ? findAlways(config.metrics, m => m.kind === filters.metric) : null; const comparedChange = metricData - ? getComparedChange(dailyCostData, metricData, filters.duration) + ? getComparedChange( + dailyCostData, + metricData, + filters.duration, + lastCompleteBillingDate, + ) : null; return ( From b13324a4de6910dc202e1ae17a55350cfbb8ad27 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 15:18:50 -0500 Subject: [PATCH 057/259] Fix ts errors --- plugins/cost-insights/src/utils/change.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/cost-insights/src/utils/change.ts b/plugins/cost-insights/src/utils/change.ts index a5a0a86f0a..0fca672425 100644 --- a/plugins/cost-insights/src/utils/change.ts +++ b/plugins/cost-insights/src/utils/change.ts @@ -23,10 +23,8 @@ import { MetricData, Duration, } from '../types'; -import { aggregationSort } from '../utils/sort'; -import moment from 'moment'; import dayjs, { OpUnitType } from 'dayjs'; -import duration, { DurationInputType } from 'dayjs/plugin/duration'; +import duration from 'dayjs/plugin/duration'; import { inclusiveStartDateOf } from './duration'; dayjs.extend(duration); From 259d848eee9caf998ac102801f7189854d2f4c8d Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 15:21:24 -0500 Subject: [PATCH 058/259] Add changeset --- .changeset/cost-insights-wild-cars-wait.md | 5 +++++ plugins/cost-insights/src/client.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/cost-insights-wild-cars-wait.md diff --git a/.changeset/cost-insights-wild-cars-wait.md b/.changeset/cost-insights-wild-cars-wait.md new file mode 100644 index 0000000000..24f84dc56b --- /dev/null +++ b/.changeset/cost-insights-wild-cars-wait.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +Fix savings/excess display calculation diff --git a/plugins/cost-insights/src/client.ts b/plugins/cost-insights/src/client.ts index 74d4954ea4..2180385424 100644 --- a/plugins/cost-insights/src/client.ts +++ b/plugins/cost-insights/src/client.ts @@ -42,7 +42,7 @@ type IntervalFields = { endDate: string; }; -export function parseIntervals(intervals: string): IntervalFields { +function parseIntervals(intervals: string): IntervalFields { const match = intervals.match( /\/(?P\d+[DM])\/(?\d{4}-\d{2}-\d{2})/, ); From 319b6b17def15a7b7ad8fd03cc9a36a6e8a3cae6 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Fri, 13 Nov 2020 15:51:25 -0500 Subject: [PATCH 059/259] Make exclusive end dates explicit --- .../cost-insights/src/utils/change.test.ts | 2 +- plugins/cost-insights/src/utils/change.ts | 13 +++++--- plugins/cost-insights/src/utils/duration.ts | 32 +++++++++++-------- plugins/cost-insights/src/utils/formatters.ts | 24 ++++++++------ 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/plugins/cost-insights/src/utils/change.test.ts b/plugins/cost-insights/src/utils/change.test.ts index 94adc25f64..6a0a8e793b 100644 --- a/plugins/cost-insights/src/utils/change.test.ts +++ b/plugins/cost-insights/src/utils/change.test.ts @@ -76,7 +76,7 @@ describe('getPreviousPeriodTotalCost', () => { change: changeOf(MockAggregatedDailyCosts), trendline: trendlineOf(MockAggregatedDailyCosts), }; - const exclusiveEndDate = '2020-10-01'; + const exclusiveEndDate = '2020-09-30'; expect( getPreviousPeriodTotalCost( mockGroupDailyCost, diff --git a/plugins/cost-insights/src/utils/change.ts b/plugins/cost-insights/src/utils/change.ts index 0fca672425..1718462948 100644 --- a/plugins/cost-insights/src/utils/change.ts +++ b/plugins/cost-insights/src/utils/change.ts @@ -22,6 +22,7 @@ import { GrowthType, MetricData, Duration, + DEFAULT_DATE_FORMAT, } from '../types'; import dayjs, { OpUnitType } from 'dayjs'; import duration from 'dayjs/plugin/duration'; @@ -68,13 +69,17 @@ export function getComparedChange( export function getPreviousPeriodTotalCost( dailyCost: Cost, duration: Duration, - endDate: string, + inclusiveEndDate: string, ): number { const dayjsDuration = dayjs.duration(duration); - const startDate = inclusiveStartDateOf(duration, endDate); + const startDate = inclusiveStartDateOf( + duration, + dayjs(inclusiveEndDate).add(1, 'day').format(DEFAULT_DATE_FORMAT), + ); + // dayjs doesn't allow adding an ISO 8601 period to dates. const [amount, type]: [number, OpUnitType] = dayjsDuration.days() - ? [dayjsDuration.days(), 'days' as OpUnitType] - : [dayjsDuration.months(), 'months' as OpUnitType]; + ? [dayjsDuration.days(), 'day'] + : [dayjsDuration.months(), 'month']; const nextPeriodStart = dayjs(startDate).add(amount, type); // Add up costs that incurred before the start of the next period. diff --git a/plugins/cost-insights/src/utils/duration.ts b/plugins/cost-insights/src/utils/duration.ts index 6a880b382c..ae8c255b91 100644 --- a/plugins/cost-insights/src/utils/duration.ts +++ b/plugins/cost-insights/src/utils/duration.ts @@ -22,27 +22,27 @@ import { assertNever } from './assert'; * Derive the start date of a given period, assuming two repeating intervals. * * @param duration see comment on Duration enum - * @param endDate from CostInsightsApi.getLastCompleteBillingDate + * @param endDate from CostInsightsApi.getLastCompleteBillingDate + 1 day */ export function inclusiveStartDateOf( duration: Duration, - endDate: string, + exclusiveEndDate: string, ): string { switch (duration) { case Duration.P30D: case Duration.P90D: - return moment(endDate) + return moment(exclusiveEndDate) .utc() .subtract(moment.duration(duration).add(moment.duration(duration))) .format(DEFAULT_DATE_FORMAT); case Duration.P1M: - return moment(endDate) + return moment(exclusiveEndDate) .utc() .startOf('month') .subtract(moment.duration(duration).add(moment.duration(duration))) .format(DEFAULT_DATE_FORMAT); case Duration.P3M: - return moment(endDate) + return moment(exclusiveEndDate) .utc() .startOf('quarter') .subtract(moment.duration(duration).add(moment.duration(duration))) @@ -54,16 +54,22 @@ export function inclusiveStartDateOf( export function exclusiveEndDateOf( duration: Duration, - endDate: string, + inclusiveEndDate: string, ): string { switch (duration) { case Duration.P30D: case Duration.P90D: - return moment(endDate).utc().add(1, 'day').format(DEFAULT_DATE_FORMAT); + return moment(inclusiveEndDate) + .utc() + .add(1, 'day') + .format(DEFAULT_DATE_FORMAT); case Duration.P1M: - return moment(endDate).utc().startOf('month').format(DEFAULT_DATE_FORMAT); + return moment(inclusiveEndDate) + .utc() + .startOf('month') + .format(DEFAULT_DATE_FORMAT); case Duration.P3M: - return moment(endDate) + return moment(inclusiveEndDate) .utc() .startOf('quarter') .format(DEFAULT_DATE_FORMAT); @@ -74,15 +80,15 @@ export function exclusiveEndDateOf( export function inclusiveEndDateOf( duration: Duration, - endDate: string, + inclusiveEndDate: string, ): string { - return moment(exclusiveEndDateOf(duration, endDate)) + return moment(exclusiveEndDateOf(duration, inclusiveEndDate)) .utc() .subtract(1, 'day') .format(DEFAULT_DATE_FORMAT); } // https://en.wikipedia.org/wiki/ISO_8601#Repeating_intervals -export function intervalsOf(duration: Duration, endDate: string) { - return `R2/${duration}/${exclusiveEndDateOf(duration, endDate)}`; +export function intervalsOf(duration: Duration, inclusiveEndDate: string) { + return `R2/${duration}/${exclusiveEndDateOf(duration, inclusiveEndDate)}`; } diff --git a/plugins/cost-insights/src/utils/formatters.ts b/plugins/cost-insights/src/utils/formatters.ts index bf8e26ef5f..e81fc1f7c9 100644 --- a/plugins/cost-insights/src/utils/formatters.ts +++ b/plugins/cost-insights/src/utils/formatters.ts @@ -15,7 +15,7 @@ */ import moment from 'moment'; -import { Duration } from '../types'; +import { Duration, DEFAULT_DATE_FORMAT } from '../types'; import { inclusiveEndDateOf, inclusiveStartDateOf } from '../utils/duration'; import { pluralOf } from '../utils/grammar'; @@ -84,21 +84,27 @@ export function formatPercent(n: number): string { return `${(n * 100).toFixed(0)}%`; } -export function formatLastTwoLookaheadQuarters(endDate: string) { - const start = moment(inclusiveStartDateOf(Duration.P3M, endDate)).format( - '[Q]Q YYYY', - ); - const end = moment(inclusiveEndDateOf(Duration.P3M, endDate)).format( +export function formatLastTwoLookaheadQuarters(inclusiveEndDate: string) { + const exclusiveEndDate = moment(inclusiveEndDate) + .add(1, 'day') + .format(DEFAULT_DATE_FORMAT); + const start = moment( + inclusiveStartDateOf(Duration.P3M, exclusiveEndDate), + ).format('[Q]Q YYYY'); + const end = moment(inclusiveEndDateOf(Duration.P3M, inclusiveEndDate)).format( '[Q]Q YYYY', ); return `${start} vs ${end}`; } -export function formatLastTwoMonths(endDate: string) { - const start = moment(inclusiveStartDateOf(Duration.P1M, endDate)) +export function formatLastTwoMonths(inclusiveEndDate: string) { + const exclusiveEndDate = moment(inclusiveEndDate) + .add(1, 'day') + .format(DEFAULT_DATE_FORMAT); + const start = moment(inclusiveStartDateOf(Duration.P1M, exclusiveEndDate)) .utc() .format('MMMM'); - const end = moment(inclusiveEndDateOf(Duration.P1M, endDate)) + const end = moment(inclusiveEndDateOf(Duration.P1M, inclusiveEndDate)) .utc() .format('MMMM'); return `${start} vs ${end}`; From e4cf4342c27d931be0b9010f290337ce5e43eb51 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 00:57:26 +0100 Subject: [PATCH 060/259] cli: work around race condition while packing dependencies of the CLI --- packages/cli/src/lib/packager/index.ts | 113 +++++++++++++++---------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 56f1ee453c..8653e66d37 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -24,8 +24,16 @@ import { tmpdir } from 'os'; import tar, { CreateOptions } from 'tar'; import { paths } from '../paths'; import { run } from '../run'; +import { packageVersions } from '../version'; import { ParallelOption } from '../parallel'; +// These packages aren't safe to pack in parallel since the CLI depends on them +const UNSAFE_PACKAGES = [ + ...Object.keys(packageVersions), + '@backstage/cli-common', + '@backstage/config-loader', +]; + type LernaPackage = { name: string; private: boolean; @@ -135,51 +143,68 @@ async function moveToDistWorkspace( workspaceDir: string, localPackages: LernaPackage[], ): Promise { + async function pack(target: LernaPackage, archive: string) { + console.log(`Repacking ${target.name} into dist workspace`); + const archivePath = resolvePath(workspaceDir, archive); + + await run('yarn', ['pack', '--filename', archivePath], { + cwd: target.location, + }); + // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed + if (target.scripts.postpack) { + await run('yarn', ['postpack'], { cwd: target.location }); + } + + const outputDir = relativePath(paths.targetRoot, target.location); + const absoluteOutputPath = resolvePath(workspaceDir, outputDir); + await fs.ensureDir(absoluteOutputPath); + + await tar.extract({ + file: archivePath, + cwd: absoluteOutputPath, + strip: 1, + }); + await fs.remove(archivePath); + + // We remove the dependencies from package.json of packages that are marked + // as bundled, so that yarn doesn't try to install them. + if (target.get('bundled')) { + const pkgJson = await fs.readJson( + resolvePath(absoluteOutputPath, 'package.json'), + ); + delete pkgJson.dependencies; + delete pkgJson.devDependencies; + delete pkgJson.peerDependencies; + delete pkgJson.optionalDependencies; + + await fs.writeJson( + resolvePath(absoluteOutputPath, 'package.json'), + pkgJson, + { + spaces: 2, + }, + ); + } + } + + const unsafePackages = localPackages.filter(p => + UNSAFE_PACKAGES.includes(p.name), + ); + const safePackages = localPackages.filter( + p => !UNSAFE_PACKAGES.includes(p.name), + ); + + // The unsafe package are packed first one by one in order to avoid race conditions + // where the CLI is being executed with broken dependencies. + for (const target of unsafePackages) { + await pack(target, `temp-package.tgz`); + } + + // Repacking in parallel is much faster and safe for all packages outside of the Backstage repo await Promise.all( - localPackages.map(async (target, index) => { - console.log(`Repacking ${target.name} into dist workspace`); - const archive = `temp-package-${index}.tgz`; - const archivePath = resolvePath(workspaceDir, archive); - - await run('yarn', ['pack', '--filename', archivePath], { - cwd: target.location, - }); - // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed - if (target.scripts.postpack) { - await run('yarn', ['postpack'], { cwd: target.location }); - } - - const outputDir = relativePath(paths.targetRoot, target.location); - const absoluteOutputPath = resolvePath(workspaceDir, outputDir); - await fs.ensureDir(absoluteOutputPath); - - await tar.extract({ - file: archivePath, - cwd: absoluteOutputPath, - strip: 1, - }); - await fs.remove(archivePath); - - // We remove the dependencies from package.json of packages that are marked - // as bundled, so that yarn doesn't try to install them. - if (target.get('bundled')) { - const pkgJson = await fs.readJson( - resolvePath(absoluteOutputPath, 'package.json'), - ); - delete pkgJson.dependencies; - delete pkgJson.devDependencies; - delete pkgJson.peerDependencies; - delete pkgJson.optionalDependencies; - - await fs.writeJson( - resolvePath(absoluteOutputPath, 'package.json'), - pkgJson, - { - spaces: 2, - }, - ); - } - }), + safePackages.map(async (target, index) => + pack(target, `temp-package-${index}.tgz`), + ), ); } From 0b2ad2853265a30121e2a35f325701301b01d451 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 10:59:00 +0100 Subject: [PATCH 061/259] microsite: new email --- docs/overview/roadmap.md | 2 +- docs/overview/vision.md | 4 ++-- microsite/blog/2020-03-16-announcing-backstage.md | 2 +- microsite/blog/2020-03-18-what-is-backstage.md | 2 +- microsite/blog/2020-06-22-backstage-service-catalog-alpha.md | 2 +- .../2020-08-05-announcing-backstage-software-templates.md | 2 +- microsite/blog/2020-09-23-backstage-cncf-sandbox.md | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/overview/roadmap.md b/docs/overview/roadmap.md index 5ff4d49a7c..71612cc5e4 100644 --- a/docs/overview/roadmap.md +++ b/docs/overview/roadmap.md @@ -39,7 +39,7 @@ If you have questions about the roadmap or want to provide feedback, we would love to hear from you! Please create an [Issue](https://github.com/backstage/backstage/issues/new/choose), ping us on [Discord](https://discord.gg/EBHEGzX) or reach out directly at -[alund@spotify.com](mailto:alund@spotify.com). +[backstage-interest@spotify.com](mailto:backstage-interest@spotify.com). Want to help out? Awesome ❤️ Head over to [CONTRIBUTING](https://github.com/backstage/backstage/blob/master/CONTRIBUTING.md) diff --git a/docs/overview/vision.md b/docs/overview/vision.md index df75a4f0a6..c17e2b17ba 100644 --- a/docs/overview/vision.md +++ b/docs/overview/vision.md @@ -19,5 +19,5 @@ We are working on making Backstage the trusted standard toolbox (read: UX layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. If this sounds interesting or you'd like to help us shape our product -vision, we'd love to talk. You can email me directly: -[alund@spotify.com](mailto:alund@spotify.com). +vision, we'd love to talk. You can email us directly: +[backstage-interest@spotify.com](mailto:backstage-interest@spotify.com). diff --git a/microsite/blog/2020-03-16-announcing-backstage.md b/microsite/blog/2020-03-16-announcing-backstage.md index 3911debf08..5824b76a57 100644 --- a/microsite/blog/2020-03-16-announcing-backstage.md +++ b/microsite/blog/2020-03-16-announcing-backstage.md @@ -41,4 +41,4 @@ We are envisioning three phases of the project and we have already begun work on - **Phase 3:** Ecosystem (later) - Everyone's infrastructure stack is different. By fostering a vibrant community of contributors we hope to provide an ecosystem of Open Source plugins/integrations that allows you to pick the tools that match your stack. -Our vision for Backstage is for it to become the trusted standard toolbox (read: UI layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. If this sounds interesting or you'd like to help us shape our product vision, we'd love to talk. You can email me directly: [alund@spotify.com](mailto:alund@spotify.com). +Our vision for Backstage is for it to become the trusted standard toolbox (read: UI layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. If this sounds interesting or you'd like to help us shape our product vision, we'd love to talk. You can email us directly: [backstage-interest@spotify.com](mailto:backstage-interest@spotify.com). diff --git a/microsite/blog/2020-03-18-what-is-backstage.md b/microsite/blog/2020-03-18-what-is-backstage.md index 0a0ff9e0b1..f4f62e2cf7 100644 --- a/microsite/blog/2020-03-18-what-is-backstage.md +++ b/microsite/blog/2020-03-18-what-is-backstage.md @@ -85,4 +85,4 @@ We are envisioning [three phases](https://github.com/backstage/backstage/milesto https://github.com/backstage/backstage/milestones -Want to discuss the project or need support? Join us on [Discord](https://discord.gg/MUpMjP2) or reach out on [alund@spotify.com](mailto:alund@spotify.com). +Want to discuss the project or need support? Join us on [Discord](https://discord.gg/MUpMjP2) or reach out on [backstage-interest@spotify.com](mailto:backstage-interest@spotify.com). diff --git a/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md b/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md index eeed43b713..e50d7a6d47 100644 --- a/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md +++ b/microsite/blog/2020-06-22-backstage-service-catalog-alpha.md @@ -49,6 +49,6 @@ Alpha is our shorthand for "we don’t yet think Backstage is ready for producti As with most alpha releases, you should expect things to change quite a lot until we reach the beta stage (we’re targeting the end of summer). There are obviously many things missing as well, but we wanted to start collecting feedback early and make it easier to see the end-to-end flow. -If you have feedback or questions, please open a [GitHub issue](https://github.com/backstage/backstage/issues), ping us on [Discord chat](https://discord.gg/EBHEGzX) or send me an email at [alund@spotify.com](mailto:alund@spotify.com) 🙏 +If you have feedback or questions, please open a [GitHub issue](https://github.com/backstage/backstage/issues), ping us on [Discord chat](https://discord.gg/EBHEGzX) or send us an email at [backstage-interest@spotify.com](mailto:backstage-interest@spotify.com) 🙏 To get regular product updates and news about the Backstage community, sign up for the [Backstage newsletter](https://mailchi.mp/spotify/backstage-community). diff --git a/microsite/blog/2020-08-05-announcing-backstage-software-templates.md b/microsite/blog/2020-08-05-announcing-backstage-software-templates.md index c216d7e398..bd6ae8eeeb 100644 --- a/microsite/blog/2020-08-05-announcing-backstage-software-templates.md +++ b/microsite/blog/2020-08-05-announcing-backstage-software-templates.md @@ -73,6 +73,6 @@ Backstage ships with four example templates, but since these are likely not the We have learned that one of the keys to getting these standards adopted is to keep an open process. Templates are code. By making it clear to your engineers that you are open to pull requests, and that teams with different needs can add their own templates, you are on the path of striking a good balance between autonomy and standardization. -If you have feedback or questions, please open a [GitHub issue](https://github.com/backstage/backstage/issues), ping us on [Discord chat](https://discord.gg/EBHEGzX) or send me an email at [alund@spotify.com](mailto:alund@spotify.com) 🙏 +If you have feedback or questions, please open a [GitHub issue](https://github.com/backstage/backstage/issues), ping us on [Discord chat](https://discord.gg/EBHEGzX) or send us an email at [backstage-interest@spotify.com](mailto:backstage-interest@spotify.com) 🙏 To get regular product updates and news about the Backstage community, sign up for the [Backstage newsletter](https://mailchi.mp/spotify/backstage-community). diff --git a/microsite/blog/2020-09-23-backstage-cncf-sandbox.md b/microsite/blog/2020-09-23-backstage-cncf-sandbox.md index fbf47a39c3..8fc459cb62 100644 --- a/microsite/blog/2020-09-23-backstage-cncf-sandbox.md +++ b/microsite/blog/2020-09-23-backstage-cncf-sandbox.md @@ -18,4 +18,4 @@ The Backstage community is healthy and growing quickly. Over [130 people](https: We’re excited to embark on this journey with the CNCF community. There’s so much great tech being built here, and it’s about time we share it to build even greater products, together. Entering into the CNCF Sandbox is just the first step. We are committed to working with the community to bring Backstage through the Incubation step, and finally all the way to becoming a Graduated, top-level project. -Thanks to everyone for your support so far. We hope you [join us](https://mailchi.mp/spotify/backstage-community) in this next chapter of Backstage's journey. If you have questions or feedback, feel free to [email](mailto:alund@spotify.com) me directly. +Thanks to everyone for your support so far. We hope you [join us](https://mailchi.mp/spotify/backstage-community) in this next chapter of Backstage's journey. If you have questions or feedback, feel free to [email](mailto:backstage-interest@spotify.com) me directly. From 05ae2a4e5238d29eb461dfc713b753341fef3461 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 13:17:35 +0100 Subject: [PATCH 062/259] cli: remove plugin:export --- packages/cli/src/commands/index.ts | 7 ------ packages/cli/src/commands/plugin/export.ts | 27 ---------------------- plugins/techdocs/package.json | 1 - 3 files changed, 35 deletions(-) delete mode 100644 packages/cli/src/commands/plugin/export.ts diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 7f028919f7..5b0dae6244 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -103,13 +103,6 @@ export function registerCommands(program: CommanderStatic) { .option(...configOption) .action(lazy(() => import('./plugin/serve').then(m => m.default))); - program - .command('plugin:export') - .description('Exports the dev/ folder of a plugin') - .option('--stats', 'Write bundle stats to output directory') - .option(...configOption) - .action(lazy(() => import('./plugin/export').then(m => m.default))); - program .command('plugin:diff') .option('--check', 'Fail if changes are required') diff --git a/packages/cli/src/commands/plugin/export.ts b/packages/cli/src/commands/plugin/export.ts deleted file mode 100644 index 69955a2e5f..0000000000 --- a/packages/cli/src/commands/plugin/export.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Command } from 'commander'; -import { buildBundle } from '../../lib/bundler'; -import { loadCliConfig } from '../../lib/config'; - -export default async (cmd: Command) => { - await buildBundle({ - entry: 'dev/index', - statsJsonEnabled: cmd.stats, - ...(await loadCliConfig(cmd.config)), - }); -}; diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 6a664a83fb..88a38549ad 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -12,7 +12,6 @@ }, "scripts": { "build": "backstage-cli plugin:build", - "export": "backstage-cli plugin:export", "start": "backstage-cli plugin:serve", "lint": "backstage-cli lint", "test": "backstage-cli test", From 2d255052f029cb0c628e84ad377d187a97dff096 Mon Sep 17 00:00:00 2001 From: Remi Date: Sun, 15 Nov 2020 16:18:13 +0100 Subject: [PATCH 063/259] feat(core): add missing tests --- packages/core/package.json | 3 +- .../AlertDisplay/AlertDisplay.test.tsx | 65 ++++++++ .../EmptyState/EmptyStateImage.test.tsx | 49 ++++++ .../components/EmptyState/EmptyStateImage.tsx | 9 +- .../FeatureCalloutCircular.test.tsx | 154 ++++++++++++++++++ .../FeatureCalloutCircular.tsx | 2 + yarn.lock | 16 +- 7 files changed, 285 insertions(+), 13 deletions(-) create mode 100644 packages/core/src/components/AlertDisplay/AlertDisplay.test.tsx create mode 100644 packages/core/src/components/EmptyState/EmptyStateImage.test.tsx create mode 100644 packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx diff --git a/packages/core/package.json b/packages/core/package.json index 848f0847bd..f7e8c67fe9 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -60,7 +60,8 @@ "react-sparklines": "^1.7.0", "react-syntax-highlighter": "^13.5.1", "react-use": "^15.3.3", - "remark-gfm": "^1.0.0" + "remark-gfm": "^1.0.0", + "zen-observable": "^0.8.15" }, "devDependencies": { "@backstage/cli": "^0.2.0", diff --git a/packages/core/src/components/AlertDisplay/AlertDisplay.test.tsx b/packages/core/src/components/AlertDisplay/AlertDisplay.test.tsx new file mode 100644 index 0000000000..3f85dab9f3 --- /dev/null +++ b/packages/core/src/components/AlertDisplay/AlertDisplay.test.tsx @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { AlertDisplay } from './AlertDisplay'; +import { + ApiProvider, + ApiRegistry, + alertApiRef, + AlertApiForwarder, +} from '@backstage/core-api'; +import Observable from 'zen-observable'; +import { renderInTestApp } from '@backstage/test-utils'; + +const TEST_MESSAGE = 'TEST_MESSAGE'; + +describe('', () => { + it('renders without exploding', async () => { + const apiRegistry = ApiRegistry.from([ + [alertApiRef, new AlertApiForwarder()], + ]); + + const { queryByText } = await renderInTestApp( + + + , + ); + expect(queryByText(TEST_MESSAGE)).not.toBeInTheDocument(); + }); + + it('renders with message', async () => { + const apiRegistry = ApiRegistry.from([ + [ + alertApiRef, + { + post() {}, + alert$() { + return Observable.of({ message: TEST_MESSAGE }); + }, + }, + ], + ]); + + const { queryByText } = await renderInTestApp( + + + , + ); + + expect(queryByText(TEST_MESSAGE)).toBeInTheDocument(); + }); +}); diff --git a/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx b/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx new file mode 100644 index 0000000000..942f533dac --- /dev/null +++ b/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; +import { EmptyStateImage } from './EmptyStateImage'; + +describe('', () => { + it('render EmptyStateImage component with missing field', async () => { + const rendered = await renderWithEffects( + wrapInTestApp(), + ); + expect(rendered.getByTestId('missingAnnotation')).toBeInTheDocument(); + }); + + it('render EmptyStateImage component with missing info', async () => { + const rendered = await renderWithEffects( + wrapInTestApp(), + ); + expect(rendered.getByTestId('noInformation')).toBeInTheDocument(); + }); + + it('render EmptyStateImage component with missing content', async () => { + const rendered = await renderWithEffects( + wrapInTestApp(), + ); + expect(rendered.getByTestId('createComponent')).toBeInTheDocument(); + }); + + it('render EmptyStateImage component with missing data', async () => { + const rendered = await renderWithEffects( + wrapInTestApp(), + ); + expect(rendered.getByTestId('noBuild')).toBeInTheDocument(); + }); +}); diff --git a/packages/core/src/components/EmptyState/EmptyStateImage.tsx b/packages/core/src/components/EmptyState/EmptyStateImage.tsx index d52a0f7dcf..c7a61f7190 100644 --- a/packages/core/src/components/EmptyState/EmptyStateImage.tsx +++ b/packages/core/src/components/EmptyState/EmptyStateImage.tsx @@ -54,6 +54,7 @@ export const EmptyStateImage = ({ missing }: Props) => { src={noInformation} alt="no Information" className={classes.generalImg} + data-testid="noInformation" /> ); case 'content': @@ -62,11 +63,17 @@ export const EmptyStateImage = ({ missing }: Props) => { src={createComponent} alt="create Component" className={classes.generalImg} + data-testid="createComponent" /> ); case 'data': return ( - no Build + no Build ); default: return null; diff --git a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx new file mode 100644 index 0000000000..5c7f17ba9f --- /dev/null +++ b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { act, fireEvent } from '@testing-library/react'; +import { renderInTestApp } from '@backstage/test-utils'; +import { FeatureCalloutCircular } from './FeatureCalloutCircular'; + +const INITIAL_BOUNDING_RECT: DOMRect = { + width: 100, + height: 100, + x: 0, + y: 0, + bottom: 0, + left: 0, + right: 0, + top: 0, + toJSON: () => {}, +}; + +const UPDATED_BOUNDING_RECT: DOMRect = { + width: 200, + height: 200, + x: 50, + y: 50, + bottom: 0, + left: 0, + right: 0, + top: 0, + toJSON: () => {}, +}; + +beforeEach(() => { + Element.prototype.getBoundingClientRect = jest.fn( + () => INITIAL_BOUNDING_RECT, + ); +}); + +describe('', () => { + it('renders without exploding', async () => { + const rendered = await renderInTestApp( + , + ); + rendered.getByText('description'); + rendered.getByText('title'); + }); + + it('renders with correct style', async () => { + const { getByTestId } = await renderInTestApp( + , + ); + const dot = await getByTestId('dot'); + const text = await getByTestId('text'); + + expect(dot).toBeInTheDocument(); + expect(text).toBeInTheDocument(); + + // Dot style + expect(dot.style.left).toBe('-800px'); + expect(dot.style.top).toBe('-800px'); + expect(dot.style.width).toBe('1700px'); + expect(dot.style.height).toBe('1700px'); + + // Text style + expect(text.style.left).toBe('-400px'); + expect(text.style.top).toBe('120px'); + expect(text.style.width).toBe('450px'); + }); + + it('update when the user scrolls', async () => { + const { getByTestId } = await renderInTestApp( + , + ); + const dot = await getByTestId('dot'); + const text = await getByTestId('text'); + + act(() => { + Element.prototype.getBoundingClientRect = jest.fn( + () => UPDATED_BOUNDING_RECT, + ); + + // Trigger the window resize event. + fireEvent(window, new Event('resize')); + }); + + // Dot style + expect(dot.style.left).toBe('-750px'); + expect(dot.style.top).toBe('-750px'); + expect(dot.style.width).toBe('1800px'); + expect(dot.style.height).toBe('1800px'); + + // Text style + expect(text.style.left).toBe('-300px'); + expect(text.style.top).toBe('270px'); + expect(text.style.width).toBe('450px'); + }); + + it('update when the user resizes the window', async () => { + const { getByTestId } = await renderInTestApp( + , + ); + const dot = await getByTestId('dot'); + const text = await getByTestId('text'); + + act(() => { + Element.prototype.getBoundingClientRect = jest.fn( + () => UPDATED_BOUNDING_RECT, + ); + + // Trigger the window resize event. + fireEvent(window, new Event('scroll')); + }); + + // Dot style + expect(dot.style.left).toBe('-750px'); + expect(dot.style.top).toBe('-750px'); + expect(dot.style.width).toBe('1800px'); + expect(dot.style.height).toBe('1800px'); + + // Text style + expect(text.style.left).toBe('-300px'); + expect(text.style.top).toBe('270px'); + expect(text.style.width).toBe('450px'); + }); +}); diff --git a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx index 38e422dbcc..9dc0d25681 100644 --- a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx +++ b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx @@ -160,6 +160,7 @@ export const FeatureCalloutCircular: FC = ({ <>
= ({
Date: Sun, 15 Nov 2020 16:18:38 +0100 Subject: [PATCH 064/259] fix(core): LinearGauge tooltip + test --- .../ProgressBars/LinearGauge.test.tsx | 37 +++++++++++++++++++ .../components/ProgressBars/LinearGauge.tsx | 14 ++++--- 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 packages/core/src/components/ProgressBars/LinearGauge.test.tsx diff --git a/packages/core/src/components/ProgressBars/LinearGauge.test.tsx b/packages/core/src/components/ProgressBars/LinearGauge.test.tsx new file mode 100644 index 0000000000..fc04b642fb --- /dev/null +++ b/packages/core/src/components/ProgressBars/LinearGauge.test.tsx @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { renderInTestApp } from '@backstage/test-utils'; + +import { LinearGauge } from './LinearGauge'; + +describe('', () => { + it('renders without exploding', async () => { + const { getByTitle } = await renderInTestApp(); + expect(getByTitle('50%')).toBeInTheDocument(); + }); + + it('renders progress and title', async () => { + const { container } = await renderInTestApp(); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders with 100 as max value', async () => { + const { getByTitle } = await renderInTestApp(); + expect(getByTitle('100%')).toBeInTheDocument(); + }); +}); diff --git a/packages/core/src/components/ProgressBars/LinearGauge.tsx b/packages/core/src/components/ProgressBars/LinearGauge.tsx index 2b8e77838d..a6aea59f19 100644 --- a/packages/core/src/components/ProgressBars/LinearGauge.tsx +++ b/packages/core/src/components/ProgressBars/LinearGauge.tsx @@ -40,12 +40,14 @@ export const LinearGauge: FC = ({ value }) => { const strokeColor = getProgressColor(theme.palette, percent, false, 100); return ( - + + + ); }; From 538328e0834d776a79738d816e0a871fd9ddc87f Mon Sep 17 00:00:00 2001 From: Remi Date: Mon, 16 Nov 2020 13:58:37 +0100 Subject: [PATCH 065/259] fix(core): regenerate yarn.lock file --- yarn.lock | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 9d4984e152..10e70f52fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1290,34 +1290,40 @@ to-fast-properties "^2.0.0" "@backstage/core@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@backstage/core/-/core-0.2.0.tgz#543246b2d87563c9aa4d9fb96e40fdfc7e827520" - integrity sha512-75m2u3FoUngBOvt9l65xZcYTzzB+49OXpY1A9VNFUR1+jMs3cL/0HDfByQV2H0xXaHzMngQ8C5u/sWhkQsij1w== + version "0.3.0" dependencies: "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.0" - "@backstage/theme" "^0.2.0" + "@backstage/core-api" "^0.2.1" + "@backstage/theme" "^0.2.1" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" + "@types/dagre" "^0.7.44" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" clsx "^1.1.0" + d3-selection "^2.0.0" + d3-shape "^2.0.0" + d3-zoom "^2.0.0" + dagre "^0.8.5" immer "^7.0.9" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" + qs "^6.9.4" rc-progress "^3.0.0" react "^16.12.0" react-dom "^16.12.0" react-helmet "6.1.0" react-hook-form "^6.6.0" + react-markdown "^5.0.2" react-router "6.0.0-beta.0" react-router-dom "6.0.0-beta.0" react-sparklines "^1.7.0" react-syntax-highlighter "^13.5.1" react-use "^15.3.3" + remark-gfm "^1.0.0" "@bcoe/v8-coverage@^0.2.3": version "0.2.3" From 2f8a074dd2741a8079ae28ccaec8b63424724b61 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Mon, 16 Nov 2020 09:41:01 -0500 Subject: [PATCH 066/259] remove excessive header margin from cost overview banner --- .../src/components/CostInsightsPage/CostInsightsPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx b/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx index 3234d1aa7f..50429e8f13 100644 --- a/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx +++ b/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx @@ -191,7 +191,7 @@ export const CostInsightsPage = () => { const CostOverviewBanner = () => ( Date: Mon, 16 Nov 2020 09:45:59 -0500 Subject: [PATCH 067/259] changeset --- .changeset/cost-insights-strange-rings-smile.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cost-insights-strange-rings-smile.md diff --git a/.changeset/cost-insights-strange-rings-smile.md b/.changeset/cost-insights-strange-rings-smile.md new file mode 100644 index 0000000000..c5f4ff9b73 --- /dev/null +++ b/.changeset/cost-insights-strange-rings-smile.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +remove excessive margin from cost overview banner From 746e861d0bc5b2e47e1a2a6ef3eeeaf83adebd9a Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Mon, 16 Nov 2020 09:56:21 -0500 Subject: [PATCH 068/259] Remove currency feature flag --- .../CostInsightsPage/CostInsightsPage.tsx | 17 ++++++----------- .../CurrencySelect/CurrencySelect.tsx | 1 + 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx b/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx index 3234d1aa7f..80464a56cb 100644 --- a/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx +++ b/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx @@ -16,7 +16,7 @@ import React, { useCallback, useEffect, useState } from 'react'; import { Box, Container, Divider, Grid, Typography } from '@material-ui/core'; -import { featureFlagsApiRef, Progress, useApi } from '@backstage/core'; +import { Progress, useApi } from '@backstage/core'; import { default as MaterialAlert } from '@material-ui/lab/Alert'; import { costInsightsApiRef } from '../../api'; import { AlertActionCardList } from '../AlertActionCardList'; @@ -49,7 +49,6 @@ import { useSubtleTypographyStyles } from '../../utils/styles'; export const CostInsightsPage = () => { const classes = useSubtleTypographyStyles(); - const featureFlags = useApi(featureFlagsApiRef); const client = useApi(costInsightsApiRef); const config = useConfig(); const groups = useGroups(); @@ -203,15 +202,11 @@ export const CostInsightsPage = () => { - {featureFlags.isActive('cost-insights-currencies') && ( - - - - )} + {currencies.map((c: Currency) => ( Date: Mon, 16 Nov 2020 10:07:56 -0500 Subject: [PATCH 069/259] Add changeset --- .changeset/cost-insights-ten-carpets-return.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cost-insights-ten-carpets-return.md diff --git a/.changeset/cost-insights-ten-carpets-return.md b/.changeset/cost-insights-ten-carpets-return.md new file mode 100644 index 0000000000..f378a20eab --- /dev/null +++ b/.changeset/cost-insights-ten-carpets-return.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': minor +--- + +remove cost insights currency feature flag From 7b37e68348d5b389f46bf1d0b26ae10bc8115676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 16 Nov 2020 16:15:09 +0100 Subject: [PATCH 070/259] integrations: move common integration concerns to a separate package (#3295) --- .changeset/ninety-gifts-guess.md | 6 + packages/backend-common/package.json | 1 + .../src/reading/AzureUrlReader.ts | 40 ++---- .../src/reading/BitbucketUrlReader.test.ts | 78 ++-------- .../src/reading/BitbucketUrlReader.ts | 135 +++++------------- .../src/reading/GithubUrlReader.test.ts | 88 +++--------- .../src/reading/GithubUrlReader.ts | 130 +++++------------ .../src/reading/GitlabUrlReader.ts | 43 ++---- packages/integration/.eslintrc.js | 3 + packages/integration/README.md | 9 ++ packages/integration/package.json | 33 +++++ packages/integration/src/azure/config.test.ts | 91 ++++++++++++ packages/integration/src/azure/config.ts | 72 ++++++++++ packages/integration/src/azure/index.ts | 21 +++ .../integration/src/bitbucket/config.test.ts | 133 +++++++++++++++++ packages/integration/src/bitbucket/config.ts | 115 +++++++++++++++ packages/integration/src/bitbucket/index.ts | 21 +++ .../integration/src/github/config.test.ts | 117 +++++++++++++++ packages/integration/src/github/config.ts | 113 +++++++++++++++ packages/integration/src/github/index.ts | 21 +++ .../integration/src/gitlab/config.test.ts | 91 ++++++++++++ packages/integration/src/gitlab/config.ts | 70 +++++++++ packages/integration/src/gitlab/index.ts | 21 +++ packages/integration/src/index.ts | 20 +++ packages/integration/src/setupTests.ts | 17 +++ 25 files changed, 1086 insertions(+), 403 deletions(-) create mode 100644 .changeset/ninety-gifts-guess.md create mode 100644 packages/integration/.eslintrc.js create mode 100644 packages/integration/README.md create mode 100644 packages/integration/package.json create mode 100644 packages/integration/src/azure/config.test.ts create mode 100644 packages/integration/src/azure/config.ts create mode 100644 packages/integration/src/azure/index.ts create mode 100644 packages/integration/src/bitbucket/config.test.ts create mode 100644 packages/integration/src/bitbucket/config.ts create mode 100644 packages/integration/src/bitbucket/index.ts create mode 100644 packages/integration/src/github/config.test.ts create mode 100644 packages/integration/src/github/config.ts create mode 100644 packages/integration/src/github/index.ts create mode 100644 packages/integration/src/gitlab/config.test.ts create mode 100644 packages/integration/src/gitlab/config.ts create mode 100644 packages/integration/src/gitlab/index.ts create mode 100644 packages/integration/src/index.ts create mode 100644 packages/integration/src/setupTests.ts diff --git a/.changeset/ninety-gifts-guess.md b/.changeset/ninety-gifts-guess.md new file mode 100644 index 0000000000..bdd20322ec --- /dev/null +++ b/.changeset/ninety-gifts-guess.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Added the integration package diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 67e4ffe11f..ea84d1a383 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -32,6 +32,7 @@ "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", "@backstage/config-loader": "^0.2.0", + "@backstage/integration": "^0.1.0", "@backstage/test-utils": "^0.1.2", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 79058c79a9..db8b738667 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -14,49 +14,27 @@ * limitations under the License. */ +import { + AzureIntegrationConfig, + readAzureIntegrationConfigs, +} from '@backstage/integration'; import fetch from 'cross-fetch'; -import { Config } from '@backstage/config'; import { NotFoundError } from '../errors'; import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; -type Options = { - // TODO: added here for future support, but we only allow dev.azure.com for now - host: string; - token?: string; -}; - -function readConfig(config: Config): Options[] { - const optionsArr = Array(); - - const providerConfigs = - config.getOptionalConfigArray('integrations.azure') ?? []; - - for (const providerConfig of providerConfigs) { - const host = providerConfig.getOptionalString('host') ?? 'dev.azure.com'; - const token = providerConfig.getOptionalString('token'); - - optionsArr.push({ host, token }); - } - - // As a convenience we always make sure there's at least an unauthenticated - // reader for public azure repos. - if (!optionsArr.some(p => p.host === 'dev.azure.com')) { - optionsArr.push({ host: 'dev.azure.com' }); - } - - return optionsArr; -} - export class AzureUrlReader implements UrlReader { static factory: ReaderFactory = ({ config }) => { - return readConfig(config).map(options => { + const configs = readAzureIntegrationConfigs( + config.getOptionalConfigArray('integrations.azure') ?? [], + ); + return configs.map(options => { const reader = new AzureUrlReader(options); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); }; - constructor(private readonly options: Options) { + constructor(private readonly options: AzureIntegrationConfig) { if (options.host !== 'dev.azure.com') { throw Error( `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`, diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 210ee16873..01744db28a 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,24 +14,22 @@ * limitations under the License. */ -import { ConfigReader } from '@backstage/config'; +import { BitbucketIntegrationConfig } from '@backstage/integration'; import { BitbucketUrlReader, getApiRequestOptions, getApiUrl, - ProviderConfig, - readConfig, } from './BitbucketUrlReader'; describe('BitbucketUrlReader', () => { describe('getApiRequestOptions', () => { it('inserts a token when needed', () => { - const withToken: ProviderConfig = { + const withToken: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '', token: 'A', }; - const withoutToken: ProviderConfig = { + const withoutToken: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '', }; @@ -44,13 +42,13 @@ describe('BitbucketUrlReader', () => { }); it('insert basic auth when needed', () => { - const withUsernameAndPassword: ProviderConfig = { + const withUsernameAndPassword: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '', username: 'some-user', appPassword: 'my-secret', }; - const withoutUsernameAndPassword: ProviderConfig = { + const withoutUsernameAndPassword: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '', }; @@ -67,11 +65,11 @@ describe('BitbucketUrlReader', () => { describe('getApiUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { host: '', apiBaseUrl: '' }; + const config: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '' }; expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for Bitbucket Cloud', () => { - const config: ProviderConfig = { + const config: BitbucketIntegrationConfig = { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0', }; @@ -87,7 +85,7 @@ describe('BitbucketUrlReader', () => { ); }); it('happy path for Bitbucket Server', () => { - const config: ProviderConfig = { + const config: BitbucketIntegrationConfig = { host: 'bitbucket.mycompany.net', apiBaseUrl: 'https://bitbucket.mycompany.net/rest/api/1.0', }; @@ -104,66 +102,6 @@ describe('BitbucketUrlReader', () => { }); }); - describe('readConfig', () => { - function config( - providers: { - host: string; - apiBaseUrl?: string; - token?: string; - username?: string; - password?: string; - }[], - ) { - return ConfigReader.fromConfigs([ - { - context: '', - data: { - integrations: { bitbucket: providers }, - }, - }, - ]); - } - - it('adds a default Bitbucket Cloud entry when missing', () => { - const output = readConfig(config([])); - expect(output).toEqual([ - { - host: 'bitbucket.org', - apiBaseUrl: 'https://api.bitbucket.org/2.0', - }, - ]); - }); - - it('injects the correct Bitbucket Cloud API base URL when missing', () => { - const output = readConfig(config([{ host: 'bitbucket.org' }])); - expect(output).toEqual([ - { - host: 'bitbucket.org', - apiBaseUrl: 'https://api.bitbucket.org/2.0', - }, - ]); - }); - - it('rejects custom targets with no base URLs', () => { - expect(() => - readConfig(config([{ host: 'bitbucket.mycompany.net' }])), - ).toThrow( - "Bitbucket integration for 'bitbucket.mycompany.net' must configure an explicit apiBaseUrl", - ); - }); - - it('rejects funky configs', () => { - expect(() => readConfig(config([{ host: 7 } as any]))).toThrow(/host/); - expect(() => readConfig(config([{ token: 7 } as any]))).toThrow(/token/); - expect(() => - readConfig(config([{ host: 'bitbucket.org', apiBaseUrl: 7 } as any])), - ).toThrow(/apiBaseUrl/); - expect(() => - readConfig(config([{ host: 'bitbucket.org', token: 7 } as any])), - ).toThrow(/token/); - }); - }); - describe('implementation', () => { it('rejects unknown targets', async () => { const processor = new BitbucketUrlReader({ diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index b2fb9fa309..9694c1d987 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -14,57 +14,18 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import parseGitUri from 'git-url-parse'; +import { + BitbucketIntegrationConfig, + readBitbucketIntegrationConfigs, +} from '@backstage/integration'; import fetch from 'cross-fetch'; +import parseGitUri from 'git-url-parse'; import { NotFoundError } from '../errors'; import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; -const DEFAULT_BASE_URL = 'https://api.bitbucket.org/2.0'; - -/** - * The configuration parameters for a single Bitbucket API provider. - */ -export type ProviderConfig = { - /** - * The host of the target that this matches on, e.g. "bitbucket.com" - */ - host: string; - - /** - * The base URL of the API of this provider, e.g. "https://api.bitbucket.org/2.0", - * with no trailing slash. - * - * May be omitted specifically for Bitbucket Cloud; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. - */ - apiBaseUrl?: string; - - /** - * The authorization token to use for requests to a Bitbucket Server provider. - * - * See https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html - * - * If no token is specified, anonymous access is used. - */ - token?: string; - - /** - * The username to use for requests to Bitbucket Cloud (bitbucket.org). - */ - username?: string; - - /** - * Authentication with Bitbucket Cloud (bitbucket.org) is done using app passwords. - * - * See https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/ - */ - appPassword?: string; -}; - -export function getApiRequestOptions(provider: ProviderConfig): RequestInit { +export function getApiRequestOptions( + provider: BitbucketIntegrationConfig, +): RequestInit { const headers: HeadersInit = {}; if (provider.token) { @@ -84,7 +45,10 @@ export function getApiRequestOptions(provider: ProviderConfig): RequestInit { // Converts for example // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml -export function getApiUrl(target: string, provider: ProviderConfig): URL { +export function getApiUrl( + target: string, + provider: BitbucketIntegrationConfig, +): URL { try { const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); if ( @@ -115,74 +79,39 @@ export function getApiUrl(target: string, provider: ProviderConfig): URL { } } -export function readConfig(config: Config): ProviderConfig[] { - const providers: ProviderConfig[] = []; - - const providerConfigs = - config.getOptionalConfigArray('integrations.bitbucket') ?? []; - - // First read all the explicit providers - for (const providerConfig of providerConfigs) { - const host = providerConfig.getOptionalString('host') ?? 'bitbucket.org'; - let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - const token = providerConfig.getOptionalString('token'); - const username = providerConfig.getOptionalString('username'); - const appPassword = providerConfig.getOptionalString('appPassword'); - - if (apiBaseUrl) { - apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (host === 'bitbucket.org') { - apiBaseUrl = DEFAULT_BASE_URL; - } - - if (!apiBaseUrl) { - throw new Error( - `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, - ); - } - if (!token && username && !appPassword) { - throw new Error( - `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, - ); - } - - providers.push({ - host, - apiBaseUrl, - token, - username, - appPassword, - }); - } - - // If no explicit bitbucket.org provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.host === 'bitbucket.org')) { - providers.push({ - host: 'bitbucket.org', - apiBaseUrl: DEFAULT_BASE_URL, - }); - } - - return providers; -} - /** * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as * the one exposed by Bitbucket Cloud itself. */ export class BitbucketUrlReader implements UrlReader { - private config: ProviderConfig; + private readonly config: BitbucketIntegrationConfig; static factory: ReaderFactory = ({ config }) => { - return readConfig(config).map(provider => { + const configs = readBitbucketIntegrationConfigs( + config.getOptionalConfigArray('integrations.bitbucket') ?? [], + ); + return configs.map(provider => { const reader = new BitbucketUrlReader(provider); const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); }; - constructor(config: ProviderConfig) { + constructor(config: BitbucketIntegrationConfig) { + const { host, apiBaseUrl, token, username, appPassword } = config; + + if (!apiBaseUrl) { + throw new Error( + `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, + ); + } + + if (!token && username && !appPassword) { + throw new Error( + `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, + ); + } + this.config = config; } diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 658d908f84..eb87d339ea 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -15,20 +15,19 @@ */ import { ConfigReader } from '@backstage/config'; -import { setupServer } from 'msw/node'; +import { GitHubIntegrationConfig } from '@backstage/integration'; import { msw } from '@backstage/test-utils'; +import fs from 'fs'; import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import path from 'path'; import { getApiRequestOptions, getApiUrl, getRawRequestOptions, getRawUrl, GithubUrlReader, - ProviderConfig, - readConfig, } from './GithubUrlReader'; -import fs from 'fs'; -import path from 'path'; import { ReadTreeResponseFactory } from './tree'; const treeResponseFactory = ReadTreeResponseFactory.create({ @@ -38,19 +37,19 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { it('sets the correct API version', () => { - const config: ProviderConfig = { host: '', apiBaseUrl: '' }; + const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; expect((getApiRequestOptions(config).headers as any).Accept).toEqual( 'application/vnd.github.v3.raw', ); }); it('inserts a token when needed', () => { - const withToken: ProviderConfig = { + const withToken: GitHubIntegrationConfig = { host: '', apiBaseUrl: '', token: 'A', }; - const withoutToken: ProviderConfig = { + const withoutToken: GitHubIntegrationConfig = { host: '', apiBaseUrl: '', }; @@ -65,12 +64,12 @@ describe('GithubUrlReader', () => { describe('getRawRequestOptions', () => { it('inserts a token when needed', () => { - const withToken: ProviderConfig = { + const withToken: GitHubIntegrationConfig = { host: '', rawBaseUrl: '', token: 'A', }; - const withoutToken: ProviderConfig = { + const withoutToken: GitHubIntegrationConfig = { host: '', rawBaseUrl: '', }; @@ -85,12 +84,12 @@ describe('GithubUrlReader', () => { describe('getApiUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { host: '', apiBaseUrl: '' }; + const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for github', () => { - const config: ProviderConfig = { + const config: GitHubIntegrationConfig = { host: 'github.com', apiBaseUrl: 'https://api.github.com', }; @@ -117,7 +116,7 @@ describe('GithubUrlReader', () => { }); it('happy path for ghe', () => { - const config: ProviderConfig = { + const config: GitHubIntegrationConfig = { host: 'ghe.mycompany.net', apiBaseUrl: 'https://ghe.mycompany.net/api/v3', }; @@ -136,12 +135,12 @@ describe('GithubUrlReader', () => { describe('getRawUrl', () => { it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { host: '', apiBaseUrl: '' }; + const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); }); it('happy path for github', () => { - const config: ProviderConfig = { + const config: GitHubIntegrationConfig = { host: 'github.com', rawBaseUrl: 'https://raw.githubusercontent.com', }; @@ -158,7 +157,7 @@ describe('GithubUrlReader', () => { }); it('happy path for ghe', () => { - const config: ProviderConfig = { + const config: GitHubIntegrationConfig = { host: 'ghe.mycompany.net', rawBaseUrl: 'https://ghe.mycompany.net/raw', }; @@ -173,60 +172,6 @@ describe('GithubUrlReader', () => { }); }); - describe('readConfig', () => { - function config( - providers: { host: string; apiBaseUrl?: string; token?: string }[], - ) { - return ConfigReader.fromConfigs([ - { - context: '', - data: { - integrations: { github: providers }, - }, - }, - ]); - } - - it('adds a default GitHub entry when missing', () => { - const output = readConfig(config([])); - expect(output).toEqual([ - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - rawBaseUrl: 'https://raw.githubusercontent.com', - }, - ]); - }); - - it('injects the correct GitHub API base URL when missing', () => { - const output = readConfig(config([{ host: 'github.com' }])); - expect(output).toEqual([ - { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - rawBaseUrl: 'https://raw.githubusercontent.com', - }, - ]); - }); - - it('rejects custom targets with no base URLs', () => { - expect(() => readConfig(config([{ host: 'ghe.company.com' }]))).toThrow( - "GitHub integration for 'ghe.company.com' must configure an explicit apiBaseUrl and rawBaseUrl", - ); - }); - - it('rejects funky configs', () => { - expect(() => readConfig(config([{ host: 7 } as any]))).toThrow(/host/); - expect(() => readConfig(config([{ token: 7 } as any]))).toThrow(/token/); - expect(() => - readConfig(config([{ host: 'github.com', apiBaseUrl: 7 } as any])), - ).toThrow(/apiBaseUrl/); - expect(() => - readConfig(config([{ host: 'github.com', token: 7 } as any])), - ).toThrow(/token/); - }); - }); - describe('implementation', () => { it('rejects unknown targets', async () => { const processor = new GithubUrlReader( @@ -271,6 +216,7 @@ describe('GithubUrlReader', () => { const processor = new GithubUrlReader( { host: 'github.com', + apiBaseUrl: 'https://api.github.com', }, { treeResponseFactory }, ); @@ -293,6 +239,7 @@ describe('GithubUrlReader', () => { const processor = new GithubUrlReader( { host: 'github.com', + apiBaseUrl: 'https://api.github.com', }, { treeResponseFactory }, ); @@ -308,6 +255,7 @@ describe('GithubUrlReader', () => { const processor = new GithubUrlReader( { host: 'github.com', + apiBaseUrl: 'https://api.github.com', }, { treeResponseFactory }, ); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index ca67a6cfcb..2fbaa0b32b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -14,59 +14,25 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import parseGitUri from 'git-url-parse'; +import { + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from '@backstage/integration'; import fetch from 'cross-fetch'; +import parseGitUri from 'git-url-parse'; import { Readable } from 'stream'; import { InputError, NotFoundError } from '../errors'; +import { ReadTreeResponseFactory } from './tree'; import { ReaderFactory, + ReadTreeOptions, ReadTreeResponse, UrlReader, - ReadTreeOptions, } from './types'; -import { ReadTreeResponseFactory } from './tree'; -/** - * The configuration parameters for a single GitHub API provider. - */ -export type ProviderConfig = { - /** - * The host of the target that this matches on, e.g. "github.com" - */ - host: string; - - /** - * The base URL of the API of this provider, e.g. "https://api.github.com", - * with no trailing slash. - * - * May be omitted specifically for GitHub; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. - */ - apiBaseUrl?: string; - - /** - * The base URL of the raw fetch endpoint of this provider, e.g. - * "https://raw.githubusercontent.com", with no trailing slash. - * - * May be omitted specifically for GitHub; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. - */ - rawBaseUrl?: string; - - /** - * The authorization token to use for requests to this provider. - * - * If no token is specified, anonymous access is used. - */ - token?: string; -}; - -export function getApiRequestOptions(provider: ProviderConfig): RequestInit { +export function getApiRequestOptions( + provider: GitHubIntegrationConfig, +): RequestInit { const headers: HeadersInit = { Accept: 'application/vnd.github.v3.raw', }; @@ -80,7 +46,9 @@ export function getApiRequestOptions(provider: ProviderConfig): RequestInit { }; } -export function getRawRequestOptions(provider: ProviderConfig): RequestInit { +export function getRawRequestOptions( + provider: GitHubIntegrationConfig, +): RequestInit { const headers: HeadersInit = {}; if (provider.token) { @@ -95,7 +63,10 @@ export function getRawRequestOptions(provider: ProviderConfig): RequestInit { // Converts for example // from: https://github.com/a/b/blob/branchname/path/to/c.yaml // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname -export function getApiUrl(target: string, provider: ProviderConfig): URL { +export function getApiUrl( + target: string, + provider: GitHubIntegrationConfig, +): URL { try { const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); @@ -120,7 +91,10 @@ export function getApiUrl(target: string, provider: ProviderConfig): URL { // Converts for example // from: https://github.com/a/b/blob/branchname/c.yaml // to: https://raw.githubusercontent.com/a/b/branchname/c.yaml -export function getRawUrl(target: string, provider: ProviderConfig): URL { +export function getRawUrl( + target: string, + provider: GitHubIntegrationConfig, +): URL { try { const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); @@ -142,60 +116,16 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL { } } -export function readConfig(config: Config): ProviderConfig[] { - const providers: ProviderConfig[] = []; - - const providerConfigs = - config.getOptionalConfigArray('integrations.github') ?? []; - - // First read all the explicit providers - for (const providerConfig of providerConfigs) { - const host = providerConfig.getOptionalString('host') ?? 'github.com'; - let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); - const token = providerConfig.getOptionalString('token'); - - if (apiBaseUrl) { - apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); - } else if (host === 'github.com') { - apiBaseUrl = 'https://api.github.com'; - } - - if (rawBaseUrl) { - rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); - } else if (host === 'github.com') { - rawBaseUrl = 'https://raw.githubusercontent.com'; - } - - if (!apiBaseUrl && !rawBaseUrl) { - throw new Error( - `GitHub integration for '${host}' must configure an explicit apiBaseUrl and rawBaseUrl`, - ); - } - - providers.push({ host, apiBaseUrl, rawBaseUrl, token }); - } - - // If no explicit github.com provider was added, put one in the list as - // a convenience - if (!providers.some(p => p.host === 'github.com')) { - providers.push({ - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - rawBaseUrl: 'https://raw.githubusercontent.com', - }); - } - - return providers; -} - /** * A processor that adds the ability to read files from GitHub v3 APIs, such as * the one exposed by GitHub itself. */ export class GithubUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { - return readConfig(config).map(provider => { + const configs = readGitHubIntegrationConfigs( + config.getOptionalConfigArray('integrations.github') ?? [], + ); + return configs.map(provider => { const reader = new GithubUrlReader(provider, { treeResponseFactory }); const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; @@ -203,9 +133,15 @@ export class GithubUrlReader implements UrlReader { }; constructor( - private readonly config: ProviderConfig, + private readonly config: GitHubIntegrationConfig, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, - ) {} + ) { + if (!config.apiBaseUrl && !config.rawBaseUrl) { + throw new Error( + `GitHub integration for '${config.host}' must configure an explicit apiBaseUrl and rawBaseUrl`, + ); + } + } async read(url: string): Promise { const useApi = diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index c621e41338..e2d3edfea2 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -14,48 +14,27 @@ * limitations under the License. */ +import { + GitLabIntegrationConfig, + readGitLabIntegrationConfigs, +} from '@backstage/integration'; import fetch from 'cross-fetch'; -import { Config } from '@backstage/config'; import { NotFoundError } from '../errors'; import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; -type Options = { - host: string; - token?: string; -}; - -function readConfig(config: Config): Options[] { - const optionsArr = Array(); - - const providerConfigs = - config.getOptionalConfigArray('integrations.gitlab') ?? []; - - for (const providerConfig of providerConfigs) { - const host = providerConfig.getOptionalString('host') ?? 'gitlab.com'; - const token = providerConfig.getOptionalString('token'); - - optionsArr.push({ host, token }); - } - - // As a convenience we always make sure there's at least an unauthenticated - // reader for public gitlab repos. - if (!optionsArr.some(p => p.host === 'gitlab.com')) { - optionsArr.push({ host: 'gitlab.com' }); - } - - return optionsArr; -} - export class GitlabUrlReader implements UrlReader { static factory: ReaderFactory = ({ config }) => { - return readConfig(config).map(options => { + const configs = readGitLabIntegrationConfigs( + config.getOptionalConfigArray('integrations.gitlab') ?? [], + ); + return configs.map(options => { const reader = new GitlabUrlReader(options); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); }; - constructor(private readonly options: Options) {} + constructor(private readonly options: GitLabIntegrationConfig) {} async read(url: string): Promise { // TODO(Rugvip): merged the old GitlabReaderProcessor in here and used @@ -133,9 +112,9 @@ export class GitlabUrlReader implements UrlReader { try { const url = new URL(target); - const branchAndfilePath = url.pathname.split('/-/blob/')[1]; + const branchAndFilePath = url.pathname.split('/-/blob/')[1]; - const [branch, ...filePath] = branchAndfilePath.split('/'); + const [branch, ...filePath] = branchAndFilePath.split('/'); url.pathname = [ '/api/v4/projects', diff --git a/packages/integration/.eslintrc.js b/packages/integration/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/packages/integration/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/packages/integration/README.md b/packages/integration/README.md new file mode 100644 index 0000000000..c1b66f0104 --- /dev/null +++ b/packages/integration/README.md @@ -0,0 +1,9 @@ +# Integrations common functionality + +Contains some common functionality of integrations. + +This package will be imported both by the frontend and backend. + +## Links + +- [The Backstage homepage](https://backstage.io) diff --git a/packages/integration/package.json b/packages/integration/package.json new file mode 100644 index 0000000000..cfbdcbe57c --- /dev/null +++ b/packages/integration/package.json @@ -0,0 +1,33 @@ +{ + "name": "@backstage/integration", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "module": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "build": "backstage-cli build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/config": "^0.1.1", + "git-url-parse": "^11.4.0" + }, + "devDependencies": { + "@backstage/cli": "^0.2.0", + "@types/jest": "^26.0.7" + }, + "files": [ + "dist" + ] +} diff --git a/packages/integration/src/azure/config.test.ts b/packages/integration/src/azure/config.test.ts new file mode 100644 index 0000000000..0b943f2081 --- /dev/null +++ b/packages/integration/src/azure/config.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + AzureIntegrationConfig, + readAzureIntegrationConfig, + readAzureIntegrationConfigs, +} from './config'; + +describe('readAzureIntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return ConfigReader.fromConfigs([{ context: '', data }]); + } + + it('reads all values', () => { + const output = readAzureIntegrationConfig( + buildConfig({ + host: 'a.com', + token: 't', + }), + ); + expect(output).toEqual({ + host: 'a.com', + token: 't', + }); + }); + + it('inserts the defaults if missing', () => { + const output = readAzureIntegrationConfig(buildConfig({})); + expect(output).toEqual({ host: 'dev.azure.com' }); + }); + + it('rejects funky configs', () => { + const valid: any = { + host: 'a.com', + token: 't', + }; + expect(() => + readAzureIntegrationConfig(buildConfig({ ...valid, host: 7 })), + ).toThrow(/host/); + expect(() => + readAzureIntegrationConfig(buildConfig({ ...valid, token: 7 })), + ).toThrow(/token/); + }); +}); + +describe('readAzureIntegrationConfigs', () => { + function buildConfig(data: Partial[]): Config[] { + return data.map(item => + ConfigReader.fromConfigs([{ context: '', data: item }]), + ); + } + + it('reads all values', () => { + const output = readAzureIntegrationConfigs( + buildConfig([ + { + host: 'a.com', + token: 't', + }, + ]), + ); + expect(output).toContainEqual({ + host: 'a.com', + token: 't', + }); + }); + + it('adds a default entry when missing', () => { + const output = readAzureIntegrationConfigs(buildConfig([])); + expect(output).toEqual([ + { + host: 'dev.azure.com', + }, + ]); + }); +}); diff --git a/packages/integration/src/azure/config.ts b/packages/integration/src/azure/config.ts new file mode 100644 index 0000000000..27b73ed707 --- /dev/null +++ b/packages/integration/src/azure/config.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const AZURE_HOST = 'dev.azure.com'; + +/** + * The configuration parameters for a single Azure provider. + */ +export type AzureIntegrationConfig = { + /** + * The host of the target that this matches on, e.g. "dev.azure.com". + * + * Currently only "dev.azure.com" is supported. + */ + host: string; + + /** + * The authorization token to use for requests. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +/** + * Reads a single Azure integration config. + * + * @param config The config object of a single integration + */ +export function readAzureIntegrationConfig( + config: Config, +): AzureIntegrationConfig { + const host = config.getOptionalString('host') ?? AZURE_HOST; + const token = config.getOptionalString('token'); + return { host, token }; +} + +/** + * Reads a set of Azure integration configs, and inserts some defaults for + * public Azure if not specified. + * + * @param configs All of the integration config objects + */ +export function readAzureIntegrationConfigs( + configs: Config[], +): AzureIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readAzureIntegrationConfig); + + // If no explicit dev.azure.com integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === AZURE_HOST)) { + result.push({ host: AZURE_HOST }); + } + + return result; +} diff --git a/packages/integration/src/azure/index.ts b/packages/integration/src/azure/index.ts new file mode 100644 index 0000000000..ede0c88a81 --- /dev/null +++ b/packages/integration/src/azure/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + readAzureIntegrationConfig, + readAzureIntegrationConfigs, +} from './config'; +export type { AzureIntegrationConfig } from './config'; diff --git a/packages/integration/src/bitbucket/config.test.ts b/packages/integration/src/bitbucket/config.test.ts new file mode 100644 index 0000000000..775a8b7d2d --- /dev/null +++ b/packages/integration/src/bitbucket/config.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + BitbucketIntegrationConfig, + readBitbucketIntegrationConfig, + readBitbucketIntegrationConfigs, +} from './config'; + +describe('readBitbucketIntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return ConfigReader.fromConfigs([{ context: '', data }]); + } + + it('reads all values', () => { + const output = readBitbucketIntegrationConfig( + buildConfig({ + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + token: 't', + username: 'u', + appPassword: 'p', + }), + ); + expect(output).toEqual({ + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + token: 't', + username: 'u', + appPassword: 'p', + }); + }); + + it('inserts the defaults if missing', () => { + const output = readBitbucketIntegrationConfig(buildConfig({})); + expect(output).toEqual( + expect.objectContaining({ + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }), + ); + }); + + it('rejects funky configs', () => { + const valid: any = { + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + token: 't', + username: 'u', + appPassword: 'p', + }; + expect(() => + readBitbucketIntegrationConfig(buildConfig({ ...valid, host: 7 })), + ).toThrow(/host/); + expect(() => + readBitbucketIntegrationConfig(buildConfig({ ...valid, apiBaseUrl: 7 })), + ).toThrow(/apiBaseUrl/); + expect(() => + readBitbucketIntegrationConfig(buildConfig({ ...valid, token: 7 })), + ).toThrow(/token/); + expect(() => + readBitbucketIntegrationConfig(buildConfig({ ...valid, username: 7 })), + ).toThrow(/username/); + expect(() => + readBitbucketIntegrationConfig(buildConfig({ ...valid, appPassword: 7 })), + ).toThrow(/appPassword/); + }); +}); + +describe('readBitbucketIntegrationConfigs', () => { + function buildConfig(data: Partial[]): Config[] { + return data.map(item => + ConfigReader.fromConfigs([{ context: '', data: item }]), + ); + } + + it('reads all values', () => { + const output = readBitbucketIntegrationConfigs( + buildConfig([ + { + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + token: 't', + username: 'u', + appPassword: 'p', + }, + ]), + ); + expect(output).toContainEqual({ + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + token: 't', + username: 'u', + appPassword: 'p', + }); + }); + + it('adds a default Bitbucket Cloud entry when missing', () => { + const output = readBitbucketIntegrationConfigs(buildConfig([])); + expect(output).toEqual([ + { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }, + ]); + }); + + it('injects the correct Bitbucket Cloud API base URL when missing', () => { + const output = readBitbucketIntegrationConfigs( + buildConfig([{ host: 'bitbucket.org' }]), + ); + expect(output).toEqual([ + { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }, + ]); + }); +}); diff --git a/packages/integration/src/bitbucket/config.ts b/packages/integration/src/bitbucket/config.ts new file mode 100644 index 0000000000..1997a5597d --- /dev/null +++ b/packages/integration/src/bitbucket/config.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const BITBUCKET_HOST = 'bitbucket.org'; +const BITBUCKET_API_BASE_URL = 'https://api.bitbucket.org/2.0'; + +/** + * The configuration parameters for a single Bitbucket API provider. + */ +export type BitbucketIntegrationConfig = { + /** + * The host of the target that this matches on, e.g. "bitbucket.org" + */ + host: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.bitbucket.org/2.0", + * with no trailing slash. + * + * May be omitted specifically for Bitbucket Cloud; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The authorization token to use for requests to a Bitbucket Server provider. + * + * See https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html + * + * If no token is specified, anonymous access is used. + */ + token?: string; + + /** + * The username to use for requests to Bitbucket Cloud (bitbucket.org). + */ + username?: string; + + /** + * Authentication with Bitbucket Cloud (bitbucket.org) is done using app passwords. + * + * See https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/ + */ + appPassword?: string; +}; + +/** + * Reads a single Bitbucket integration config. + * + * @param config The config object of a single integration + */ +export function readBitbucketIntegrationConfig( + config: Config, +): BitbucketIntegrationConfig { + const host = config.getOptionalString('host') ?? BITBUCKET_HOST; + let apiBaseUrl = config.getOptionalString('apiBaseUrl'); + const token = config.getOptionalString('token'); + const username = config.getOptionalString('username'); + const appPassword = config.getOptionalString('appPassword'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === BITBUCKET_HOST) { + apiBaseUrl = BITBUCKET_API_BASE_URL; + } + + return { + host, + apiBaseUrl, + token, + username, + appPassword, + }; +} + +/** + * Reads a set of Bitbucket integration configs, and inserts some defaults for + * public Bitbucket if not specified. + * + * @param configs All of the integration config objects + */ +export function readBitbucketIntegrationConfigs( + configs: Config[], +): BitbucketIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readBitbucketIntegrationConfig); + + // If no explicit bitbucket.org integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === BITBUCKET_HOST)) { + result.push({ + host: BITBUCKET_HOST, + apiBaseUrl: BITBUCKET_API_BASE_URL, + }); + } + + return result; +} diff --git a/packages/integration/src/bitbucket/index.ts b/packages/integration/src/bitbucket/index.ts new file mode 100644 index 0000000000..897c00d160 --- /dev/null +++ b/packages/integration/src/bitbucket/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + readBitbucketIntegrationConfig, + readBitbucketIntegrationConfigs, +} from './config'; +export type { BitbucketIntegrationConfig } from './config'; diff --git a/packages/integration/src/github/config.test.ts b/packages/integration/src/github/config.test.ts new file mode 100644 index 0000000000..d33cffb7be --- /dev/null +++ b/packages/integration/src/github/config.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + GitHubIntegrationConfig, + readGitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from './config'; + +describe('readGitHubIntegrationConfig', () => { + function buildConfig(provider: Partial) { + return ConfigReader.fromConfigs([{ context: '', data: provider }]); + } + + it('reads all values', () => { + const output = readGitHubIntegrationConfig( + buildConfig({ + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + rawBaseUrl: 'https://a.com/raw', + token: 't', + }), + ); + expect(output).toEqual({ + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + rawBaseUrl: 'https://a.com/raw', + token: 't', + }); + }); + + it('injects the correct GitHub API base URL when missing', () => { + const output = readGitHubIntegrationConfig( + buildConfig({ host: 'github.com' }), + ); + expect(output).toEqual({ + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }); + }); + + it('rejects funky configs', () => { + const valid: any = { + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + rawBaseUrl: 'https://a.com/raw', + token: 't', + }; + expect(() => + readGitHubIntegrationConfig(buildConfig({ ...valid, host: 7 })), + ).toThrow(/host/); + expect(() => + readGitHubIntegrationConfig(buildConfig({ ...valid, apiBaseUrl: 7 })), + ).toThrow(/apiBaseUrl/); + expect(() => + readGitHubIntegrationConfig(buildConfig({ ...valid, rawBaseUrl: 7 })), + ).toThrow(/rawBaseUrl/); + expect(() => + readGitHubIntegrationConfig(buildConfig({ ...valid, token: 7 })), + ).toThrow(/token/); + }); +}); + +describe('readGitHubIntegrationConfigs', () => { + function buildConfig( + providers: Partial[], + ): Config[] { + return providers.map(provider => + ConfigReader.fromConfigs([{ context: '', data: provider }]), + ); + } + + it('reads all values', () => { + const output = readGitHubIntegrationConfigs( + buildConfig([ + { + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + rawBaseUrl: 'https://a.com/raw', + token: 't', + }, + ]), + ); + expect(output).toContainEqual({ + host: 'a.com', + apiBaseUrl: 'https://a.com/api', + rawBaseUrl: 'https://a.com/raw', + token: 't', + }); + }); + + it('adds a default GitHub entry when missing', () => { + const output = readGitHubIntegrationConfigs(buildConfig([])); + expect(output).toEqual([ + { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }, + ]); + }); +}); diff --git a/packages/integration/src/github/config.ts b/packages/integration/src/github/config.ts new file mode 100644 index 0000000000..f646acc53b --- /dev/null +++ b/packages/integration/src/github/config.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const GITHUB_HOST = 'github.com'; +const GITHUB_API_BASE_URL = 'https://api.github.com'; +const GITHUB_RAW_BASE_URL = 'https://raw.githubusercontent.com'; + +/** + * The configuration parameters for a single GitHub integration. + */ +export type GitHubIntegrationConfig = { + /** + * The host of the target that this matches on, e.g. "github.com" + */ + host: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://raw.githubusercontent.com", with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +/** + * Reads a single GitHub integration config. + * + * @param config The config object of a single integration + */ +export function readGitHubIntegrationConfig( + config: Config, +): GitHubIntegrationConfig { + const host = config.getOptionalString('host') ?? GITHUB_HOST; + let apiBaseUrl = config.getOptionalString('apiBaseUrl'); + let rawBaseUrl = config.getOptionalString('rawBaseUrl'); + const token = config.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === GITHUB_HOST) { + apiBaseUrl = GITHUB_API_BASE_URL; + } + + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (host === GITHUB_HOST) { + rawBaseUrl = GITHUB_RAW_BASE_URL; + } + + return { host, apiBaseUrl, rawBaseUrl, token }; +} + +/** + * Reads a set of GitHub integration configs, and inserts some defaults for + * public GitHub if not specified. + * + * @param configs All of the integration config objects + */ +export function readGitHubIntegrationConfigs( + configs: Config[], +): GitHubIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readGitHubIntegrationConfig); + + // If no explicit github.com integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === GITHUB_HOST)) { + result.push({ + host: GITHUB_HOST, + apiBaseUrl: GITHUB_API_BASE_URL, + rawBaseUrl: GITHUB_RAW_BASE_URL, + }); + } + + return result; +} diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts new file mode 100644 index 0000000000..2099dd42e3 --- /dev/null +++ b/packages/integration/src/github/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + readGitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from './config'; +export type { GitHubIntegrationConfig } from './config'; diff --git a/packages/integration/src/gitlab/config.test.ts b/packages/integration/src/gitlab/config.test.ts new file mode 100644 index 0000000000..9998a530c6 --- /dev/null +++ b/packages/integration/src/gitlab/config.test.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + GitLabIntegrationConfig, + readGitLabIntegrationConfig, + readGitLabIntegrationConfigs, +} from './config'; + +describe('readGitLabIntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return ConfigReader.fromConfigs([{ context: '', data }]); + } + + it('reads all values', () => { + const output = readGitLabIntegrationConfig( + buildConfig({ + host: 'a.com', + token: 't', + }), + ); + expect(output).toEqual({ + host: 'a.com', + token: 't', + }); + }); + + it('inserts the defaults if missing', () => { + const output = readGitLabIntegrationConfig(buildConfig({})); + expect(output).toEqual({ host: 'gitlab.com' }); + }); + + it('rejects funky configs', () => { + const valid: any = { + host: 'a.com', + token: 't', + }; + expect(() => + readGitLabIntegrationConfig(buildConfig({ ...valid, host: 7 })), + ).toThrow(/host/); + expect(() => + readGitLabIntegrationConfig(buildConfig({ ...valid, token: 7 })), + ).toThrow(/token/); + }); +}); + +describe('readGitLabIntegrationConfigs', () => { + function buildConfig(data: Partial[]): Config[] { + return data.map(item => + ConfigReader.fromConfigs([{ context: '', data: item }]), + ); + } + + it('reads all values', () => { + const output = readGitLabIntegrationConfigs( + buildConfig([ + { + host: 'a.com', + token: 't', + }, + ]), + ); + expect(output).toContainEqual({ + host: 'a.com', + token: 't', + }); + }); + + it('adds a default entry when missing', () => { + const output = readGitLabIntegrationConfigs(buildConfig([])); + expect(output).toEqual([ + { + host: 'gitlab.com', + }, + ]); + }); +}); diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts new file mode 100644 index 0000000000..97c948c999 --- /dev/null +++ b/packages/integration/src/gitlab/config.ts @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const GITLAB_HOST = 'gitlab.com'; + +/** + * The configuration parameters for a single GitLab integration. + */ +export type GitLabIntegrationConfig = { + /** + * The host of the target that this matches on, e.g. "gitlab.com" + */ + host: string; + + /** + * The authorization token to use for requests this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +/** + * Reads a single GitLab integration config. + * + * @param config The config object of a single integration + */ +export function readGitLabIntegrationConfig( + config: Config, +): GitLabIntegrationConfig { + const host = config.getOptionalString('host') ?? GITLAB_HOST; + const token = config.getOptionalString('token'); + return { host, token }; +} + +/** + * Reads a set of GitLab integration configs, and inserts some defaults for + * public GitLab if not specified. + * + * @param configs All of the integration config objects + */ +export function readGitLabIntegrationConfigs( + configs: Config[], +): GitLabIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readGitLabIntegrationConfig); + + // As a convenience we always make sure there's at least an unauthenticated + // reader for public gitlab repos. + if (!result.some(c => c.host === GITLAB_HOST)) { + result.push({ host: GITLAB_HOST }); + } + + return result; +} diff --git a/packages/integration/src/gitlab/index.ts b/packages/integration/src/gitlab/index.ts new file mode 100644 index 0000000000..0801914fd4 --- /dev/null +++ b/packages/integration/src/gitlab/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + readGitLabIntegrationConfig, + readGitLabIntegrationConfigs, +} from './config'; +export type { GitLabIntegrationConfig } from './config'; diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts new file mode 100644 index 0000000000..bfed81824f --- /dev/null +++ b/packages/integration/src/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './azure'; +export * from './bitbucket'; +export * from './github'; +export * from './gitlab'; diff --git a/packages/integration/src/setupTests.ts b/packages/integration/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/packages/integration/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; From ba985b1357610282ce41926d001f47d2208ca3a9 Mon Sep 17 00:00:00 2001 From: Remi Date: Mon, 16 Nov 2020 16:18:50 +0100 Subject: [PATCH 071/259] feat(core): remove data-testid --- .../components/EmptyState/EmptyState.test.tsx | 2 +- .../EmptyState/EmptyStateImage.test.tsx | 16 ++++++++-------- .../components/EmptyState/EmptyStateImage.tsx | 10 +--------- .../FeatureCalloutCircular.test.tsx | 2 +- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/packages/core/src/components/EmptyState/EmptyState.test.tsx b/packages/core/src/components/EmptyState/EmptyState.test.tsx index c5bc7a9876..32e71f044d 100644 --- a/packages/core/src/components/EmptyState/EmptyState.test.tsx +++ b/packages/core/src/components/EmptyState/EmptyState.test.tsx @@ -34,6 +34,6 @@ describe('', () => { rendered.getByText('Your plugin is missing an annotation'), ).toBeInTheDocument(); expect(rendered.getByLabelText('button')).toBeInTheDocument(); - expect(rendered.getByTestId('missingAnnotation')).toBeInTheDocument(); + expect(rendered.getByAltText('annotation is missing')).toBeInTheDocument(); }); }); diff --git a/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx b/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx index 942f533dac..258eee943d 100644 --- a/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx +++ b/packages/core/src/components/EmptyState/EmptyStateImage.test.tsx @@ -20,30 +20,30 @@ import { EmptyStateImage } from './EmptyStateImage'; describe('', () => { it('render EmptyStateImage component with missing field', async () => { - const rendered = await renderWithEffects( + const { getByAltText } = await renderWithEffects( wrapInTestApp(), ); - expect(rendered.getByTestId('missingAnnotation')).toBeInTheDocument(); + expect(getByAltText('annotation is missing')).toBeInTheDocument(); }); it('render EmptyStateImage component with missing info', async () => { - const rendered = await renderWithEffects( + const { getByAltText } = await renderWithEffects( wrapInTestApp(), ); - expect(rendered.getByTestId('noInformation')).toBeInTheDocument(); + expect(getByAltText('no Information')).toBeInTheDocument(); }); it('render EmptyStateImage component with missing content', async () => { - const rendered = await renderWithEffects( + const { getByAltText } = await renderWithEffects( wrapInTestApp(), ); - expect(rendered.getByTestId('createComponent')).toBeInTheDocument(); + expect(getByAltText('create Component')).toBeInTheDocument(); }); it('render EmptyStateImage component with missing data', async () => { - const rendered = await renderWithEffects( + const { getByAltText } = await renderWithEffects( wrapInTestApp(), ); - expect(rendered.getByTestId('noBuild')).toBeInTheDocument(); + expect(getByAltText('no Build')).toBeInTheDocument(); }); }); diff --git a/packages/core/src/components/EmptyState/EmptyStateImage.tsx b/packages/core/src/components/EmptyState/EmptyStateImage.tsx index c7a61f7190..1973ff9a23 100644 --- a/packages/core/src/components/EmptyState/EmptyStateImage.tsx +++ b/packages/core/src/components/EmptyState/EmptyStateImage.tsx @@ -45,7 +45,6 @@ export const EmptyStateImage = ({ missing }: Props) => { src={missingAnnotation} className={classes.generalImg} alt="annotation is missing" - data-testid="missingAnnotation" /> ); case 'info': @@ -54,7 +53,6 @@ export const EmptyStateImage = ({ missing }: Props) => { src={noInformation} alt="no Information" className={classes.generalImg} - data-testid="noInformation" /> ); case 'content': @@ -63,17 +61,11 @@ export const EmptyStateImage = ({ missing }: Props) => { src={createComponent} alt="create Component" className={classes.generalImg} - data-testid="createComponent" /> ); case 'data': return ( - no Build + no Build ); default: return null; diff --git a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx index 5c7f17ba9f..83a31f198d 100644 --- a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx +++ b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.test.tsx @@ -136,7 +136,7 @@ describe('', () => { () => UPDATED_BOUNDING_RECT, ); - // Trigger the window resize event. + // Trigger the window scroll event. fireEvent(window, new Event('scroll')); }); From 3ad6320029670c37dbe25f564ba93c57094bc902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 16 Nov 2020 16:45:46 +0100 Subject: [PATCH 072/259] docs: catalog - extending the model, and relations (#3291) * docs: catalog - extending the model, and relations * Address comments --- .../software-catalog/descriptor-format.md | 71 ++++++++++++++++ .../software-catalog/extending-the-model.md | 63 ++++++++++++-- .../well-known-annotations.md | 4 +- .../software-catalog/well-known-relations.md | 83 +++++++++++++++++++ microsite/sidebars.json | 1 + 5 files changed, 213 insertions(+), 9 deletions(-) create mode 100644 docs/features/software-catalog/well-known-relations.md diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index a330ef196a..9081009385 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -22,11 +22,15 @@ we recommend that you name them `catalog-info.yaml`. - [Overall Shape Of An Entity](#overall-shape-of-an-entity) - [Common to All Kinds: The Envelope](#common-to-all-kinds-the-envelope) - [Common to All Kinds: The Metadata](#common-to-all-kinds-the-metadata) +- [Common to All Kinds: Relations](#common-to-all-kinds-relations) - [Kind: Component](#kind-component) - [Kind: Template](#kind-template) - [Kind: API](#kind-api) - [Kind: Group](#kind-group) - [Kind: User](#kind-user) +- [Kind: Resource](#kind-resource) +- [Kind: System](#kind-system) +- [Kind: Domain](#kind-domain) ## Overall Shape Of An Entity @@ -259,6 +263,61 @@ This field is optional, and currently has no special semantics. Each tag must be sequences of `[a-z0-9]` separated by `-`, at most 63 characters in total. +## Common to All Kinds: Relations + +The `relations` root field is a read-only list of relations, between the current +entity and other entities, described in the +[well-known relations section](well-known-relations.md). Relations are commonly +two-way, so that there's a pair of relation types each describing one direction +of the relation. + +A relation as part of a single entity that's read out of the API may look as +follows. + +```js +{ + // ... + "relations": [ + { + "target": { + "kind": "group", + "namespace": "default", + "name": "dev.infra" + }, + "type": "ownedBy" + } + ], + "spec": { + "owner": "dev.infra", + // ... + } +} +``` + +The fields of a relation are: + +| Field | Type | Description | +| ---------- | ------ | -------------------------------------------------------------------------------- | +| `target` | Object | A complete [compound reference](references.md) to the other end of the relation. | +| `type` | String | The type of relation FROM a source entity TO the target entity. | +| `metadata` | Object | Reserved for future use. | + +Entity descriptor YAML files are not supposed to contain this field. Instead, +catalog processors analyze the entity descriptor data and its surroundings, and +deduce relations that are then attached onto the entity as read from the +catalog. + +Where relations are produced, they are to be considered the authoritative source +for that piece of data. In the example above, a plugin would do better to +consume the relation rather than `spec.owner` for deducing the owner of the +entity, because it may even be the case that the owner isn't taken from the YAML +at all - it could be taken from a CODEOWNERS file nearby instead for example. +Also, the `spec.owner` is on a shortened form and may have semantics associated +with it (such as the default kind being `Group` if not specified). + +See the [well-known relations section](well-known-relations.md) for a list of +well-known / common relations and their semantics. + ## Kind: Component Describes the following entity kind: @@ -740,3 +799,15 @@ with the default kind `Group` and the default namespace equal to the same namespace as the user. Only `Group` entities may be referenced. Most commonly, these entries point to groups in the same namespace, so in those cases it is sufficient to enter only the `metadata.name` field of those groups. + +## Kind: Resource + +This kind is not yet defined, but is reserved [for future use](system-model.md). + +## Kind: System + +This kind is not yet defined, but is reserved [for future use](system-model.md). + +## Kind: Domain + +This kind is not yet defined, but is reserved [for future use](system-model.md). diff --git a/docs/features/software-catalog/extending-the-model.md b/docs/features/software-catalog/extending-the-model.md index 72170fb576..cbacce40c7 100644 --- a/docs/features/software-catalog/extending-the-model.md +++ b/docs/features/software-catalog/extending-the-model.md @@ -4,6 +4,36 @@ title: Extending the model description: Documentation on Extending the model --- +The Backstage catalog [entity data model](descriptor-format.md) is based on the +[Kubernetes objects format](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/), +and borrows a lot of its semantics as well. This page describes those semantics +at a higher level and how to extend them to fit your organization. + +Backstage comes with a number of catalog concepts out of the box: + +- There are a number of builtin versioned _kinds_, such as `Component`, `User` + etc. These encapsulate the high level concept of an entity, and define the + schema for its entity definition data. +- An entity has both a _metadata_ object and a _spec_ object at the root. +- Each kind may or may not have a _type_. For example, there are several well + known types of component, such as `service` and `website`. These clarify the + more detailed nature of the entity, and may affect what features are exposed + in the interface. +- Entities may have a number of _[annotations](well-known-annotations.md)_ on + them. These can be added either by humans into the descriptor files, or added + by automated processes when the entity is ingested into the catalog. +- Entities may have a number of _labels_ on them. +- Entities may have a number of _relations_, expressing how they relate to each + other in different ways. + +We'll list different possibilities for extending this below. + +## Adding a New Kind + +> TODO: Fill in + +## Adding a New Type of an Existing Kind + Backstage natively supports tracking of the following component [`type`](descriptor-format.md)'s: @@ -20,13 +50,6 @@ track in Backstage, it is possible to add your own software types that fit your organization's data model. Inside Spotify our model has grown significantly over the years, and now includes ML models, Apps, data pipelines and many more. -## Adding a new type - -TODO: Describe what changes are needed to add a new type that shows up in the -catalog. - -## The Other type - It might be tempting to put software that doesn't fit into any of the existing types into Other. There are a few reasons why we advise against this; firstly, we have found that it is preferred to match the conceptual model that your @@ -39,3 +62,29 @@ For example, the [Lighthouse plugin](https://github.com/backstage/backstage/tree/master/plugins/lighthouse) only makes sense for Websites. The more specific you can be in how you model your software, the easier it is to provide plugins that are contextual. + +> TODO: Fill in + +## Changing the Validation Rules for Core Entity Fields + +> TODO: Fill in + +## Adding New Fields to the Metadata Object + +> TODO: Fill in + +## Adding New Fields to the Spec Object of an Existing Kind + +> TODO: Fill in + +## Adding a New Annotation + +> TODO: Fill in + +## Adding a New Label + +> TODO: Fill in + +## Adding a New Relation Type + +> TODO: Fill in diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 5cb6837340..1f5e851064 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -2,9 +2,9 @@ id: well-known-annotations title: Well-known Annotations on Catalog Entities sidebar_label: Well-known Annotations -description: Documentation on lists a number of well known Annotations, that +description: Documentation that lists a number of well known Annotations, that have defined semantics. They can be attached to catalog entities and consumed -by plugins as needed +by plugins as needed. --- This section lists a number of well known diff --git a/docs/features/software-catalog/well-known-relations.md b/docs/features/software-catalog/well-known-relations.md new file mode 100644 index 0000000000..fe1c32b02f --- /dev/null +++ b/docs/features/software-catalog/well-known-relations.md @@ -0,0 +1,83 @@ +--- +id: well-known-relations +title: Well-known Relations between Catalog Entities +sidebar_label: Well-known Relations +description: Documentation that lists a number of well known Relations, that +have defined semantics. They can be attached to catalog entities and consumed +by plugins as needed. +--- + +This section lists a number of well known +[entity relation types](descriptor-format.md#common-to-all-kinds-relations), +that have defined semantics. They can be attached to catalog entities and +consumed by plugins as needed. + +If you are looking to extend the set of relations, see +[Extending the model](extending-the-model.md). + +## Relations + +This is a (non-exhaustive) list of relations that are known to be in active use. + +Each relation has a _source_ (implicitly: the entity that holds the relation), a +_target_ (the entity to which the source has a relation), and a _type_ that +tells what relation the source has with the target. The relation is directional; +there are commonly pairs of relation types and the entity at the other end will +have the opposite relation in the opposite direction (e.g. when querying for +`A`, you will see `A.ownedBy.B`, and when querying `B`, you will see +`B.ownerOf.A`). + +### `ownedBy` and `ownerOf` + +An ownership relation where the owner is usually an organizational entity +([User](descriptor-format.md#kind-user) or +[Group](descriptor-format.md#kind-group)), and the other entity can be anything. + +In Backstage, the owner of an entity is the singular entity (commonly a team) +that bears ultimate responsibility for the entity, and has the authority and +capability to develop and maintain it. They will be the point of contact if +something goes wrong, or if features are to be requested. The main purpose of +this relation is for display purposes in Backstage, so that people looking at +catalog entities can get an understanding of to whom this entity belongs. It is +not to be used by automated processes to for example assign authorization in +runtime systems. There may be others that also develop or otherwise touch the +entity, but there will always be one ultimate owner. + +This relation is commonly generated based on `spec.owner` of the owned entity, +where present. + +### `consumesApi` and `providesApi` + +A relation with an [API](descriptor-format.md#kind-api) entity, typically from a +[Component](descriptor-format.md#kind-component) or +[System](descriptor-format.md#kind-system). + +These relations express that a component or system either exposes an API - +meaning that it hosts callable endpoints from which you can consume that API - +or that they are dependent on being able to consume that API. + +This relation is commonly generated based on `spec.implementsApis` of the +component or system in question. + +### `dependsOn` and `dependencyOf` + +A relation denoting a dependency on another entity. + +This relation is a general expression of being in need of that other entity for +an entity to function. It can for example be used to express that a website +component needs a library component as part of its build, or that a service +component uses a persistent storage resource. + +### `parentOf` and `childOf` + +A parent/child relation to build up a tree, used for example to describe the +organizational structure between [Groups](descriptor-format.md#kind-group). + +This relation is commonly based on `spec.parent` and/or `spec.children`. + +### `memberOf` and `hasMember` + +A membership relation, typically for [Users](descriptor-format.md#kind-user) in +[Groups](descriptor-format.md#kind-group). + +This relation is commonly based on `spec.memberOf`. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 64fdd2f420..cb7cfcd6d5 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -45,6 +45,7 @@ "features/software-catalog/descriptor-format", "features/software-catalog/references", "features/software-catalog/well-known-annotations", + "features/software-catalog/well-known-relations", "features/software-catalog/extending-the-model", "features/software-catalog/external-integrations", "features/software-catalog/software-catalog-api" From 885526474eb7d62599c38e5f5a11d653a4c17175 Mon Sep 17 00:00:00 2001 From: Brenda Sukh Date: Mon, 16 Nov 2020 16:14:19 -0500 Subject: [PATCH 073/259] Fix styling on dropdowns --- .../CostInsightsPage/CostInsightsPage.tsx | 19 +++++++++++-------- .../CurrencySelect/CurrencySelect.tsx | 1 - 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx b/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx index 80464a56cb..8a9400d97a 100644 --- a/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx +++ b/plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx @@ -192,21 +192,24 @@ export const CostInsightsPage = () => { px={3} marginTop={10} display="flex" - flexDirection="row" justifyContent="space-between" + alignItems="center" + minHeight={40} > - + Cost Overview Billing data as of {lastCompleteBillingDate} - - + + + + {currencies.map((c: Currency) => ( Date: Mon, 16 Nov 2020 17:34:35 -0500 Subject: [PATCH 074/259] Add label to currency select --- .../CurrencySelect/CurrencySelect.tsx | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/plugins/cost-insights/src/components/CurrencySelect/CurrencySelect.tsx b/plugins/cost-insights/src/components/CurrencySelect/CurrencySelect.tsx index 1b5920ffcb..2db1b8ee38 100644 --- a/plugins/cost-insights/src/components/CurrencySelect/CurrencySelect.tsx +++ b/plugins/cost-insights/src/components/CurrencySelect/CurrencySelect.tsx @@ -15,7 +15,13 @@ */ import React from 'react'; -import { MenuItem, Select, SelectProps } from '@material-ui/core'; +import { + InputLabel, + FormControl, + MenuItem, + Select, + SelectProps, +} from '@material-ui/core'; import { Currency, CurrencyType } from '../../types'; import { findAlways } from '../../utils/assert'; import { useSelectStyles as useStyles } from '../../utils/styles'; @@ -51,24 +57,28 @@ export const CurrencySelect = ({ }; return ( - + + Convert to: + + ); }; From d33f5157c797afbe9358515510012dafc624d95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Tue, 17 Nov 2020 10:36:12 +0100 Subject: [PATCH 075/259] Extracted pushToRemote in scaffolder-backend --- .changeset/silent-suns-bathe.md | 5 + .../scaffolder/stages/publish/azure.test.ts | 125 +----------- .../src/scaffolder/stages/publish/azure.ts | 30 +-- .../scaffolder/stages/publish/github.test.ts | 181 +++++------------- .../src/scaffolder/stages/publish/github.ts | 35 +--- .../scaffolder/stages/publish/gitlab.test.ts | 138 ++----------- .../src/scaffolder/stages/publish/gitlab.ts | 29 +-- .../scaffolder/stages/publish/helpers.test.ts | 112 +++++++++++ .../src/scaffolder/stages/publish/helpers.ts | 55 ++++++ 9 files changed, 259 insertions(+), 451 deletions(-) create mode 100644 .changeset/silent-suns-bathe.md create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts diff --git a/.changeset/silent-suns-bathe.md b/.changeset/silent-suns-bathe.md new file mode 100644 index 0000000000..96e32ea125 --- /dev/null +++ b/.changeset/silent-suns-bathe.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Extracted pushToRemote function for reuse between publishers diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts index 8b84eb0edf..7ab6f81ae5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts @@ -17,10 +17,13 @@ jest.mock('nodegit'); jest.mock('azure-devops-node-api/GitApi'); jest.mock('azure-devops-node-api/interfaces/GitInterfaces'); +jest.mock('./helpers', () => ({ + pushToRemoteUserPass: jest.fn(), +})); import { AzurePublisher } from './azure'; import { GitApi } from 'azure-devops-node-api/GitApi'; -import * as NodeGit from 'nodegit'; +import { pushToRemoteUserPass } from './helpers'; const { mockGitApi } = require('azure-devops-node-api/GitApi') as { mockGitApi: { @@ -28,25 +31,6 @@ const { mockGitApi } = require('azure-devops-node-api/GitApi') as { }; }; -const { - Repository, - mockRepo, - mockIndex, - Signature, - Remote, - mockRemote, - Cred, -} = require('nodegit') as { - Repository: jest.Mocked<{ init: any }>; - Signature: jest.Mocked<{ now: any }>; - Cred: jest.Mocked<{ userpassPlaintextNew: any }>; - Remote: jest.Mocked<{ create: any }>; - - mockIndex: jest.Mocked; - mockRepo: jest.Mocked; - mockRemote: jest.Mocked; -}; - describe('Azure Publisher', () => { const publisher = new AzurePublisher(new GitApi('', []), 'fake-token'); @@ -60,7 +44,7 @@ describe('Azure Publisher', () => { remoteUrl: 'mockclone', } as { remoteUrl: string }); - await publisher.publish({ + const result = await publisher.publish({ values: { storePath: 'project/repo', owner: 'bob', @@ -68,109 +52,16 @@ describe('Azure Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect(mockGitApi.createRepository).toHaveBeenCalledWith( { name: 'repo', }, 'project', ); - }); - }); - - describe('publish: createGitDirectory', () => { - const values = { - isOrg: true, - storePath: 'blam/test', - owner: 'lols', - }; - - const mockDir = '/tmp/test/dir'; - - mockGitApi.createRepository.mockResolvedValue({ - remoteUrl: 'mockclone', - } as { remoteUrl: string }); - - it('should call init on the repo with the directory', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Repository.init).toHaveBeenCalledWith(mockDir, 0); - }); - - it('should call refresh index on the index and write the new files', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(mockRepo.refreshIndex).toHaveBeenCalled(); - }); - - it('should call add all files and write', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(mockIndex.addAll).toHaveBeenCalled(); - expect(mockIndex.write).toHaveBeenCalled(); - expect(mockIndex.writeTree).toHaveBeenCalled(); - }); - - it('should create a commit with on head with the right name and commiter', async () => { - const mockSignature = { mockSignature: 'bloblly' }; - Signature.now.mockReturnValue(mockSignature); - - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Signature.now).toHaveBeenCalledTimes(2); - expect(Signature.now).toHaveBeenCalledWith( - 'Scaffolder', - 'scaffolder@backstage.io', - ); - - expect(mockRepo.createCommit).toHaveBeenCalledWith( - 'HEAD', - mockSignature, - mockSignature, - 'initial commit', - 'mockoid', - [], - ); - }); - - it('creates a remote with the repo and remote', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Remote.create).toHaveBeenCalledWith( - mockRepo, - 'origin', + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', 'mockclone', - ); - }); - - it('shoud push to the remote repo', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - const [remotes, { callbacks }] = mockRemote.push.mock - .calls[0] as NodeGit.PushOptions[]; - - expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); - - callbacks?.credentials?.(); - - expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( 'notempty', 'fake-token', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 9dfc7b5b66..33f7f89199 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -17,10 +17,9 @@ import { PublisherBase } from './types'; import { GitApi } from 'azure-devops-node-api/GitApi'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; - +import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; -import { Repository, Remote, Signature, Cred } from 'nodegit'; export class AzurePublisher implements PublisherBase { private readonly client: GitApi; @@ -39,7 +38,7 @@ export class AzurePublisher implements PublisherBase { directory: string; }): Promise<{ remoteUrl: string }> { const remoteUrl = await this.createRemote(values); - await this.pushToRemote(directory, remoteUrl); + await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token); return { remoteUrl }; } @@ -54,29 +53,4 @@ export class AzurePublisher implements PublisherBase { return repo.remoteUrl || ''; } - - private async pushToRemote(directory: string, remote: string): Promise { - const repo = await Repository.init(directory, 0); - const index = await repo.refreshIndex(); - await index.addAll(); - await index.write(); - const oid = await index.writeTree(); - await repo.createCommit( - 'HEAD', - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - 'initial commit', - oid, - [], - ); - - const remoteRepo = await Remote.create(repo, 'origin', remote); - - await remoteRepo.push(['refs/heads/master:refs/heads/master'], { - callbacks: { - // Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat - credentials: () => Cred.userpassPlaintextNew('notempty', this.token), - }, - }); - } } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts index e68b0d0447..c85b4acba3 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts @@ -16,15 +16,18 @@ jest.mock('@octokit/rest'); jest.mock('nodegit'); +jest.mock('./helpers', () => ({ + pushToRemoteUserPass: jest.fn(), +})); import { Octokit } from '@octokit/rest'; -import * as NodeGit from 'nodegit'; import { OctokitResponse, ReposCreateInOrgResponseData, UsersGetByUsernameResponseData, } from '@octokit/types'; import { GithubPublisher } from './github'; +import { pushToRemoteUserPass } from './helpers'; const { mockGithubClient } = require('@octokit/rest') as { mockGithubClient: { @@ -34,25 +37,6 @@ const { mockGithubClient } = require('@octokit/rest') as { }; }; -const { - Repository, - mockRepo, - mockIndex, - Signature, - Remote, - mockRemote, - Cred, -} = require('nodegit') as { - Repository: jest.Mocked<{ init: any }>; - Signature: jest.Mocked<{ now: any }>; - Cred: jest.Mocked<{ userpassPlaintextNew: any }>; - Remote: jest.Mocked<{ create: any }>; - - mockIndex: jest.Mocked; - mockRepo: jest.Mocked; - mockRemote: jest.Mocked; -}; - describe('GitHub Publisher', () => { beforeEach(() => { jest.clearAllMocks(); @@ -72,8 +56,13 @@ describe('GitHub Publisher', () => { clone_url: 'mockclone', }, } as OctokitResponse); + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { + type: 'Organization', + }, + } as OctokitResponse); - await publisher.publish({ + const result = await publisher.publish({ values: { storePath: 'blam/test', owner: 'bob', @@ -82,6 +71,7 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ org: 'blam', name: 'test', @@ -97,6 +87,12 @@ describe('GitHub Publisher', () => { repo: 'test', permission: 'admin', }); + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', + 'mockclone', + 'abc', + 'x-oauth-basic', + ); }); it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => { @@ -111,7 +107,7 @@ describe('GitHub Publisher', () => { }, } as OctokitResponse); - await publisher.publish({ + const result = await publisher.publish({ values: { storePath: 'blam/test', owner: 'bob', @@ -120,6 +116,7 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -127,6 +124,12 @@ describe('GitHub Publisher', () => { private: false, }); expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled(); + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', + 'mockclone', + 'abc', + 'x-oauth-basic', + ); }); }); @@ -142,7 +145,7 @@ describe('GitHub Publisher', () => { }, } as OctokitResponse); - await publisher.publish({ + const result = await publisher.publish({ values: { storePath: 'blam/test', owner: 'bob', @@ -152,6 +155,7 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -165,113 +169,12 @@ describe('GitHub Publisher', () => { username: 'bob', permission: 'admin', }); - }); - - describe('publish: createGitDirectory', () => { - const values = { - storePath: 'blam/test', - owner: 'lols', - access: 'lols', - }; - - const mockDir = '/tmp/test/dir'; - - mockGithubClient.repos.createInOrg.mockResolvedValue({ - data: { - clone_url: 'mockclone', - }, - } as OctokitResponse); - mockGithubClient.users.getByUsername.mockResolvedValue({ - data: { - type: 'Organization', - }, - } as OctokitResponse); - - it('should call init on the repo with the directory', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Repository.init).toHaveBeenCalledWith(mockDir, 0); - }); - - it('should call refresh index on the index and write the new files', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(mockRepo.refreshIndex).toHaveBeenCalled(); - }); - - it('should call add all files and write', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(mockIndex.addAll).toHaveBeenCalled(); - expect(mockIndex.write).toHaveBeenCalled(); - expect(mockIndex.writeTree).toHaveBeenCalled(); - }); - - it('should create a commit with on head with the right name and commiter', async () => { - const mockSignature = { mockSignature: 'bloblly' }; - Signature.now.mockReturnValue(mockSignature); - - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Signature.now).toHaveBeenCalledTimes(2); - expect(Signature.now).toHaveBeenCalledWith( - 'Scaffolder', - 'scaffolder@backstage.io', - ); - - expect(mockRepo.createCommit).toHaveBeenCalledWith( - 'HEAD', - mockSignature, - mockSignature, - 'initial commit', - 'mockoid', - [], - ); - }); - - it('creates a remote with the repo and remote', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Remote.create).toHaveBeenCalledWith( - mockRepo, - 'origin', - 'mockclone', - ); - }); - - it('shoud push to the remote repo', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - const [remotes, { callbacks }] = mockRemote.push.mock - .calls[0] as NodeGit.PushOptions[]; - - expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); - - callbacks?.credentials?.(); - - expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( - 'abc', - 'x-oauth-basic', - ); - }); + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', + 'mockclone', + 'abc', + 'x-oauth-basic', + ); }); }); @@ -294,7 +197,7 @@ describe('GitHub Publisher', () => { }, } as OctokitResponse); - await publisher.publish({ + const result = await publisher.publish({ values: { isOrg: true, storePath: 'blam/test', @@ -303,12 +206,19 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ org: 'blam', name: 'test', private: true, visibility: 'internal', }); + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', + 'mockclone', + 'abc', + 'x-oauth-basic', + ); }); }); @@ -331,7 +241,7 @@ describe('GitHub Publisher', () => { }, } as OctokitResponse); - await publisher.publish({ + const result = await publisher.publish({ values: { storePath: 'blam/test', owner: 'bob', @@ -339,12 +249,19 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ name: 'test', private: true, }); + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', + 'mockclone', + 'abc', + 'x-oauth-basic', + ); }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 60ed86daa1..d5542e8800 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -16,10 +16,9 @@ import { PublisherBase } from './types'; import { Octokit } from '@octokit/rest'; - +import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; -import { Repository, Remote, Signature, Cred } from 'nodegit'; export type RepoVisibilityOptions = 'private' | 'internal' | 'public'; @@ -52,7 +51,12 @@ export class GithubPublisher implements PublisherBase { directory: string; }): Promise<{ remoteUrl: string }> { const remoteUrl = await this.createRemote(values); - await this.pushToRemote(directory, remoteUrl); + await pushToRemoteUserPass( + directory, + remoteUrl, + this.token, + 'x-oauth-basic', + ); return { remoteUrl }; } @@ -104,29 +108,4 @@ export class GithubPublisher implements PublisherBase { return data?.clone_url; } - - private async pushToRemote(directory: string, remote: string): Promise { - const repo = await Repository.init(directory, 0); - const index = await repo.refreshIndex(); - await index.addAll(); - await index.write(); - const oid = await index.writeTree(); - await repo.createCommit( - 'HEAD', - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - 'initial commit', - oid, - [], - ); - - const remoteRepo = await Remote.create(repo, 'origin', remote); - await remoteRepo.push(['refs/heads/master:refs/heads/master'], { - callbacks: { - credentials: () => { - return Cred.userpassPlaintextNew(this.token, 'x-oauth-basic'); - }, - }, - }); - } } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts index 43b19859de..a87fa6c6d9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts @@ -16,11 +16,14 @@ jest.mock('nodegit'); jest.mock('@gitbeaker/node'); +jest.mock('./helpers', () => ({ + pushToRemoteUserPass: jest.fn(), +})); import { GitlabPublisher } from './gitlab'; import { Gitlab as GitlabAPI } from '@gitbeaker/core'; import { Gitlab } from '@gitbeaker/node'; -import * as NodeGit from 'nodegit'; +import { pushToRemoteUserPass } from './helpers'; const { mockGitlabClient } = require('@gitbeaker/node') as { mockGitlabClient: { @@ -30,25 +33,6 @@ const { mockGitlabClient } = require('@gitbeaker/node') as { }; }; -const { - Repository, - mockRepo, - mockIndex, - Signature, - Remote, - mockRemote, - Cred, -} = require('nodegit') as { - Repository: jest.Mocked<{ init: any }>; - Signature: jest.Mocked<{ now: any }>; - Cred: jest.Mocked<{ userpassPlaintextNew: any }>; - Remote: jest.Mocked<{ create: any }>; - - mockIndex: jest.Mocked; - mockRepo: jest.Mocked; - mockRemote: jest.Mocked; -}; - describe('GitLab Publisher', () => { const publisher = new GitlabPublisher(new Gitlab({}), 'fake-token'); @@ -61,8 +45,11 @@ describe('GitLab Publisher', () => { mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 42, } as { id: number }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'mockclone', + } as { http_url_to_repo: string }); - await publisher.publish({ + const result = await publisher.publish({ values: { isOrg: true, storePath: 'blam/test', @@ -71,10 +58,17 @@ describe('GitLab Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ namespace_id: 42, name: 'test', }); + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', + 'mockclone', + 'oauth2', + 'fake-token', + ); }); it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => { @@ -86,7 +80,7 @@ describe('GitLab Publisher', () => { http_url_to_repo: 'mockclone', } as { http_url_to_repo: string }); - await publisher.publish({ + const result = await publisher.publish({ values: { storePath: 'blam/test', owner: 'bob', @@ -94,109 +88,15 @@ describe('GitLab Publisher', () => { directory: '/tmp/test', }); + expect(result).toEqual({ remoteUrl: 'mockclone' }); expect(mockGitlabClient.Users.current).toHaveBeenCalled(); - expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ namespace_id: 21, name: 'test', }); - }); - }); - - describe('publish: createGitDirectory', () => { - const values = { - isOrg: true, - storePath: 'blam/test', - owner: 'lols', - }; - - const mockDir = '/tmp/test/dir'; - - mockGitlabClient.Projects.create.mockResolvedValue({ - http_url_to_repo: 'mockclone', - } as { http_url_to_repo: string }); - - it('should call init on the repo with the directory', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Repository.init).toHaveBeenCalledWith(mockDir, 0); - }); - - it('should call refresh index on the index and write the new files', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(mockRepo.refreshIndex).toHaveBeenCalled(); - }); - - it('should call add all files and write', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(mockIndex.addAll).toHaveBeenCalled(); - expect(mockIndex.write).toHaveBeenCalled(); - expect(mockIndex.writeTree).toHaveBeenCalled(); - }); - - it('should create a commit with on head with the right name and commiter', async () => { - const mockSignature = { mockSignature: 'bloblly' }; - Signature.now.mockReturnValue(mockSignature); - - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Signature.now).toHaveBeenCalledTimes(2); - expect(Signature.now).toHaveBeenCalledWith( - 'Scaffolder', - 'scaffolder@backstage.io', - ); - - expect(mockRepo.createCommit).toHaveBeenCalledWith( - 'HEAD', - mockSignature, - mockSignature, - 'initial commit', - 'mockoid', - [], - ); - }); - - it('creates a remote with the repo and remote', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - expect(Remote.create).toHaveBeenCalledWith( - mockRepo, - 'origin', + expect(pushToRemoteUserPass).toHaveBeenCalledWith( + '/tmp/test', 'mockclone', - ); - }); - - it('shoud push to the remote repo', async () => { - await publisher.publish({ - values, - directory: mockDir, - }); - - const [remotes, { callbacks }] = mockRemote.push.mock - .calls[0] as NodeGit.PushOptions[]; - - expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); - - callbacks?.credentials?.(); - - expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( 'oauth2', 'fake-token', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts index a755a138b5..e748dc53dc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts @@ -16,10 +16,9 @@ import { PublisherBase } from './types'; import { Gitlab } from '@gitbeaker/core'; - +import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; import { RequiredTemplateValues } from '../templater'; -import { Repository, Remote, Signature, Cred } from 'nodegit'; export class GitlabPublisher implements PublisherBase { private readonly client: Gitlab; @@ -38,7 +37,7 @@ export class GitlabPublisher implements PublisherBase { directory: string; }): Promise<{ remoteUrl: string }> { const remoteUrl = await this.createRemote(values); - await this.pushToRemote(directory, remoteUrl); + await pushToRemoteUserPass(directory, remoteUrl, 'oauth2', this.token); return { remoteUrl }; } @@ -63,28 +62,4 @@ export class GitlabPublisher implements PublisherBase { return project?.http_url_to_repo; } - - private async pushToRemote(directory: string, remote: string): Promise { - const repo = await Repository.init(directory, 0); - const index = await repo.refreshIndex(); - await index.addAll(); - await index.write(); - const oid = await index.writeTree(); - await repo.createCommit( - 'HEAD', - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - Signature.now('Scaffolder', 'scaffolder@backstage.io'), - 'initial commit', - oid, - [], - ); - - const remoteRepo = await Remote.create(repo, 'origin', remote); - - await remoteRepo.push(['refs/heads/master:refs/heads/master'], { - callbacks: { - credentials: () => Cred.userpassPlaintextNew('oauth2', this.token), - }, - }); - } } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts new file mode 100644 index 0000000000..3a317adacd --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('nodegit'); +import * as NodeGit from 'nodegit'; +import { pushToRemoteCred } from './helpers'; + +const { + Repository, + mockRepo, + mockIndex, + Signature, + Remote, + mockRemote, + Cred, +} = require('nodegit') as { + Repository: jest.Mocked<{ init: any }>; + Signature: jest.Mocked<{ now: any }>; + Cred: jest.Mocked<{ userpassPlaintextNew: any }>; + Remote: jest.Mocked<{ create: any }>; + + mockIndex: jest.Mocked; + mockRepo: jest.Mocked; + mockRemote: jest.Mocked; +}; + +describe('pushToRemoteCred', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const directory = '/tmp/test/dir'; + const remote = 'mockclone'; + const credentialsProvider = () => + NodeGit.Cred.userpassPlaintextNew('username', 'password'); + + it('should call init on the repo with the directory', async () => { + await pushToRemoteCred(directory, remote, credentialsProvider); + + expect(Repository.init).toHaveBeenCalledWith(directory, 0); + }); + + it('should call refresh index on the index and write the new files', async () => { + await pushToRemoteCred(directory, remote, credentialsProvider); + + expect(mockRepo.refreshIndex).toHaveBeenCalled(); + }); + + it('should call add all files and write', async () => { + await pushToRemoteCred(directory, remote, credentialsProvider); + + expect(mockIndex.addAll).toHaveBeenCalled(); + expect(mockIndex.write).toHaveBeenCalled(); + expect(mockIndex.writeTree).toHaveBeenCalled(); + }); + + it('should create a commit with on head with the right name and commiter', async () => { + const mockSignature = { mockSignature: 'bloblly' }; + Signature.now.mockReturnValue(mockSignature); + + await pushToRemoteCred(directory, remote, credentialsProvider); + + expect(Signature.now).toHaveBeenCalledTimes(2); + expect(Signature.now).toHaveBeenCalledWith( + 'Scaffolder', + 'scaffolder@backstage.io', + ); + + expect(mockRepo.createCommit).toHaveBeenCalledWith( + 'HEAD', + mockSignature, + mockSignature, + 'initial commit', + 'mockoid', + [], + ); + }); + + it('creates a remote with the repo and remote', async () => { + await pushToRemoteCred(directory, remote, credentialsProvider); + + expect(Remote.create).toHaveBeenCalledWith(mockRepo, 'origin', 'mockclone'); + }); + + it('shoud push to the remote repo', async () => { + await pushToRemoteCred(directory, remote, credentialsProvider); + + const [remotes, { callbacks }] = mockRemote.push.mock + .calls[0] as NodeGit.PushOptions[]; + + expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); + + callbacks?.credentials?.(); + + expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( + 'username', + 'password', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts new file mode 100644 index 0000000000..51f5a4c854 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Repository, Remote, Signature, Cred } from 'nodegit'; + +export async function pushToRemoteCred( + directory: string, + remote: string, + credentialsProvider: () => Cred, +): Promise { + const repo = await Repository.init(directory, 0); + const index = await repo.refreshIndex(); + await index.addAll(); + await index.write(); + const oid = await index.writeTree(); + await repo.createCommit( + 'HEAD', + Signature.now('Scaffolder', 'scaffolder@backstage.io'), + Signature.now('Scaffolder', 'scaffolder@backstage.io'), + 'initial commit', + oid, + [], + ); + + const remoteRepo = await Remote.create(repo, 'origin', remote); + + await remoteRepo.push(['refs/heads/master:refs/heads/master'], { + callbacks: { + credentials: credentialsProvider, + }, + }); +} + +export async function pushToRemoteUserPass( + directory: string, + remote: string, + username: string, + password: string, +): Promise { + return pushToRemoteCred(directory, remote, () => + Cred.userpassPlaintextNew(username, password), + ); +} From 53b1a29968f3845c554e5a5b5d2db485f8986606 Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak Date: Tue, 17 Nov 2020 12:20:27 +0100 Subject: [PATCH 076/259] Feat: bump GitHub Insights Plugin version --- packages/app/package.json | 2 +- yarn.lock | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index f013200fbb..8dda24fdf1 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -34,7 +34,7 @@ "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@octokit/rest": "^18.0.0", - "@roadiehq/backstage-plugin-github-insights": "^0.2.12", + "@roadiehq/backstage-plugin-github-insights": "^0.2.14", "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.2", "@roadiehq/backstage-plugin-travis-ci": "^0.2.7", "@roadiehq/backstage-plugin-buildkite": "^0.1.2", diff --git a/yarn.lock b/yarn.lock index 10e70f52fb..947b33e89a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3706,10 +3706,10 @@ react-router-dom "6.0.0-beta.0" react-use "^15.3.3" -"@roadiehq/backstage-plugin-github-insights@^0.2.12": - version "0.2.12" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.12.tgz#aeda6306769f376cf6b5b9d74268b060a0f4e764" - integrity sha512-f9g5ajVWQkywoYo0zDUZV3g0SPxOTi3MvMuhUvgaZ/XKJr9mBZTe5qHnEYBSHOCB88l0Y2g0qHNdbjXhbFpokQ== +"@roadiehq/backstage-plugin-github-insights@^0.2.14": + version "0.2.14" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.14.tgz#2e4bf61495e650bb2b7a1711f97c9c4995215588" + integrity sha512-xhzHAmmTu7op1V7K3ytomGlmzHMs9jnb2Lc2YTrU3ame4WZcrrMCrqi8X9v8B8VHfwMBrwa55tyxRYObXrPr7A== dependencies: "@backstage/catalog-model" "^0.2.0" "@backstage/core" "^0.2.0" @@ -3722,10 +3722,8 @@ history "^5.0.0" react "^16.13.1" react-dom "^16.13.1" - react-markdown "^5.0.0" react-router "^6.0.0-beta.0" react-use "^15.3.3" - remark-gfm "^1.0.0" "@roadiehq/backstage-plugin-github-pull-requests@^0.6.2": version "0.6.2" @@ -19788,7 +19786,7 @@ react-markdown@^4.3.1: unist-util-visit "^1.3.0" xtend "^4.0.1" -react-markdown@^5.0.0, react-markdown@^5.0.2: +react-markdown@^5.0.2: version "5.0.2" resolved "https://registry.npmjs.org/react-markdown/-/react-markdown-5.0.2.tgz#d15a8beb37b4ec34fc23dd892e7755eb7040b8db" integrity sha512-kmkB4JbV7LqkDAjvaKRKtodB3n3Id76/DalaDun1U8FuLB0SenPfvH+jAQ5Pcpo54cACRQc1LB1yXmuuuIVecw== From d1bfe7b71fff1b2483729735a19a7b8f58909cfa Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 17 Nov 2020 13:19:28 +0100 Subject: [PATCH 077/259] Docs: Add TechDocs survey on docs site --- docs/features/techdocs/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index f1cf637997..16131a1ded 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -122,3 +122,12 @@ migrate Spotify's existing TechDocs features to open source. https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend [techdocs/container]: https://github.com/backstage/techdocs-container [techdocs/cli]: https://github.com/backstage/techdocs-cli + +## Feedback + +We have created a sweet and short TechDocs user survey - +https://docs.google.com/forms/d/e/1FAIpQLSdn5Vn3MQhCdyYRuW8cMzZkMQF0bFxXYN168gZRvESLfJWVVg/viewform + +This is to gather inputs from you (the Backstage community) which will help us +best serve TechDocs adopters and existing users. Your inputs will shape our +roadmap and we will share it in the open. From 22f7755d298833639046711a5ecb770adaea3e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 17 Nov 2020 13:26:24 +0100 Subject: [PATCH 078/259] docs: skip prettier for long metadata lines (#3311) --- docs/features/software-catalog/descriptor-format.md | 4 ++-- docs/features/software-catalog/external-integrations.md | 4 ++-- docs/features/software-catalog/index.md | 4 ++-- docs/features/software-catalog/well-known-annotations.md | 5 ++--- docs/features/software-catalog/well-known-relations.md | 5 ++--- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 9081009385..5d7a5b9af0 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -2,8 +2,8 @@ id: descriptor-format title: Descriptor Format of Catalog Entities sidebar_label: YAML File Format -description: Documentation on Descriptor Format of Catalog Entities which -describes the default data shape and semantics of catalog entities +# prettier-ignore +description: Documentation on Descriptor Format of Catalog Entities which describes the default data shape and semantics of catalog entities --- This section describes the default data shape and semantics of catalog entities. diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 325bed5e4a..e7d253f11e 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -1,8 +1,8 @@ --- id: external-integrations title: External integrations -description: Documentation on External integrations to integrate systems -with Backstage +# prettier-ignore +description: Documentation on External integrations to integrate systems with Backstage --- Backstage natively supports importing catalog data through the use of diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index 08e6260e80..70541b85db 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -2,8 +2,8 @@ id: software-catalog-overview title: Backstage Service Catalog (alpha) sidebar_label: Overview -description: The Backstage Service Catalog — actually, a software catalog, since -it includes more than just services +# prettier-ignore +description: The Backstage Service Catalog — actually, a software catalog, since it includes more than just services --- ## What is a Service Catalog? diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 1f5e851064..4d227ea568 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -2,9 +2,8 @@ id: well-known-annotations title: Well-known Annotations on Catalog Entities sidebar_label: Well-known Annotations -description: Documentation that lists a number of well known Annotations, that -have defined semantics. They can be attached to catalog entities and consumed -by plugins as needed. +# prettier-ignore +description: Documentation that lists a number of well known Annotations, that have defined semantics. They can be attached to catalog entities and consumed by plugins as needed. --- This section lists a number of well known diff --git a/docs/features/software-catalog/well-known-relations.md b/docs/features/software-catalog/well-known-relations.md index fe1c32b02f..07a4fc0a5c 100644 --- a/docs/features/software-catalog/well-known-relations.md +++ b/docs/features/software-catalog/well-known-relations.md @@ -2,9 +2,8 @@ id: well-known-relations title: Well-known Relations between Catalog Entities sidebar_label: Well-known Relations -description: Documentation that lists a number of well known Relations, that -have defined semantics. They can be attached to catalog entities and consumed -by plugins as needed. +# prettier-ignore +description: Documentation that lists a number of well known Relations, that have defined semantics. They can be attached to catalog entities and consumed by plugins as needed. --- This section lists a number of well known From ea0fe99f82d3fe504cd58bc0cd32db138fe87710 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 17 Nov 2020 12:40:52 -0500 Subject: [PATCH 079/259] Add tooltips --- plugins/catalog/src/components/AboutCard/AboutCard.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/catalog/src/components/AboutCard/AboutCard.tsx b/plugins/catalog/src/components/AboutCard/AboutCard.tsx index 8da67d7493..89630d6f10 100644 --- a/plugins/catalog/src/components/AboutCard/AboutCard.tsx +++ b/plugins/catalog/src/components/AboutCard/AboutCard.tsx @@ -119,6 +119,7 @@ export function AboutCard({ entity, variant }: AboutCardProps) { action={ { window.open(codeLink.edithref || '#', '_blank'); }} @@ -133,7 +134,12 @@ export function AboutCard({ entity, variant }: AboutCardProps) { disabled={ !entity.metadata.annotations?.['backstage.io/techdocs-ref'] } - label="View Techdocs" + label="View TechDocs" + title={ + !entity.metadata.annotations?.['backstage.io/techdocs-ref'] + ? 'No TechDocs available' + : '' + } icon={} href={`/docs/${ entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE @@ -142,6 +148,7 @@ export function AboutCard({ entity, variant }: AboutCardProps) { } href="api" /> From 8b7737d0b8f1dbe2e1f52ba30d9777241243b253 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 17 Nov 2020 12:44:13 -0500 Subject: [PATCH 080/259] Add changeset --- .changeset/friendly-dodos-remember.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/friendly-dodos-remember.md diff --git a/.changeset/friendly-dodos-remember.md b/.changeset/friendly-dodos-remember.md new file mode 100644 index 0000000000..463d71f806 --- /dev/null +++ b/.changeset/friendly-dodos-remember.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Add About Card tooltips From 5efc00c6a6c2b90c9c102c7b9ca93306893d9cbc Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 17 Nov 2020 13:45:11 -0500 Subject: [PATCH 081/259] Add typescript support for title --- .../components/AboutCard/IconLinkVertical/IconLinkVertical.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/catalog/src/components/AboutCard/IconLinkVertical/IconLinkVertical.tsx b/plugins/catalog/src/components/AboutCard/IconLinkVertical/IconLinkVertical.tsx index c054bc0d4e..dd267dde35 100644 --- a/plugins/catalog/src/components/AboutCard/IconLinkVertical/IconLinkVertical.tsx +++ b/plugins/catalog/src/components/AboutCard/IconLinkVertical/IconLinkVertical.tsx @@ -23,6 +23,7 @@ export type IconLinkVerticalProps = { icon?: React.ReactNode; href?: string; disabled?: boolean; + title?: string; label: string; }; @@ -57,6 +58,7 @@ export function IconLinkVertical({ {icon} From b3a5e3b150f85517b6127959ddcedfd28794741c Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 17 Nov 2020 13:50:50 -0500 Subject: [PATCH 082/259] Add tooltips plural --- .github/styles/vocab.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 3c5fa34adc..043b83a51f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -192,6 +192,7 @@ tolerations Tolerations toolsets tooltip +tooltips touchpoints ui upvote From 65edcdee22f4f17184652542d44b5c2dd693b50b Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 17 Nov 2020 20:03:44 +0100 Subject: [PATCH 083/259] TechDocs: Remove hardcoded backend URL from techdocs-backend --- packages/backend/src/plugins/techdocs.ts | 2 +- .../techdocs-backend/src/techdocs/stages/publish/local.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index a9d6293c33..ecc67b528f 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -50,7 +50,7 @@ export default async function createPlugin({ const urlPreparer = new UrlPreparer(reader, logger); preparers.register('url', urlPreparer); - const publisher = new LocalPublish(logger); + const publisher = new LocalPublish(logger, config); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts index 6d655d4632..221b0d48e7 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts @@ -18,12 +18,15 @@ import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; import { PublisherBase } from './types'; import { resolvePackagePath } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; export class LocalPublish implements PublisherBase { private readonly logger: Logger; + private readonly config: Config; - constructor(logger: Logger) { + constructor(logger: Logger, config: Config) { this.logger = logger; + this.config = config; } publish({ @@ -63,8 +66,9 @@ export class LocalPublish implements PublisherBase { reject(err); } + const backendBaseUrl = this.config.getString('backend.baseUrl'); resolve({ - remoteUrl: `http://localhost:7000/api/techdocs/static/docs/${entity.metadata.name}`, + remoteUrl: `${backendBaseUrl}/api/techdocs/static/docs/${entity.metadata.name}`, }); }); }); From c01bf361a44e2e1c93b7602ea7ebd24fc7cbd983 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 17 Nov 2020 20:19:30 +0100 Subject: [PATCH 084/259] Modify tests to include new config param in LocalPublish --- plugins/techdocs-backend/src/service/standaloneServer.ts | 2 +- .../src/techdocs/stages/publish/local.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 79e8b0b945..7dda1b833b 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -53,7 +53,7 @@ export async function startStandaloneServer( const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); - const publisher = new LocalPublish(logger); + const publisher = new LocalPublish(logger, config); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts index d97a4014f6..199e6346e8 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts @@ -18,6 +18,7 @@ import fs from 'fs-extra'; import path from 'path'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { LocalPublish } from './local'; const createMockEntity = (annotations = {}) => { @@ -37,7 +38,8 @@ const logger = getVoidLogger(); describe('local publisher', () => { it('should publish generated documentation dir', async () => { - const publisher = new LocalPublish(logger); + const testConfig = ConfigReader.fromConfigs([{ context: '', data: {} }]); + const publisher = new LocalPublish(logger, testConfig); const mockEntity = createMockEntity(); From 3b314807c63e7b922b7d3538b4aaac3af5832b71 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 17 Nov 2020 21:03:56 +0100 Subject: [PATCH 085/259] TechDocs: Use discovery instead of config to read base url --- packages/backend/src/plugins/techdocs.ts | 2 +- plugins/techdocs-backend/package.json | 1 + .../src/service/standaloneServer.ts | 2 +- .../src/techdocs/stages/publish/local.test.ts | 19 ++++++++++++++++--- .../src/techdocs/stages/publish/local.ts | 15 ++++++++------- yarn.lock | 16 +++++----------- 6 files changed, 32 insertions(+), 23 deletions(-) diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index ecc67b528f..de48280e64 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -50,7 +50,7 @@ export default async function createPlugin({ const urlPreparer = new UrlPreparer(reader, logger); preparers.register('url', urlPreparer); - const publisher = new LocalPublish(logger, config); + const publisher = new LocalPublish(logger, discovery); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index c1d1f6ad27..236f53b9d1 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -23,6 +23,7 @@ "@backstage/backend-common": "^0.2.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", + "@backstage/core-api": "^0.2.1", "@types/dockerode": "^2.5.34", "@types/express": "^4.17.6", "command-exists-promise": "^2.0.2", diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 7dda1b833b..4082a36172 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -53,7 +53,7 @@ export async function startStandaloneServer( const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); - const publisher = new LocalPublish(logger, config); + const publisher = new LocalPublish(logger, discovery); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts index 199e6346e8..4b6503f620 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts @@ -17,7 +17,7 @@ /* eslint-disable no-restricted-syntax */ import fs from 'fs-extra'; import path from 'path'; -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, SingleHostDiscovery } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { LocalPublish } from './local'; @@ -38,8 +38,21 @@ const logger = getVoidLogger(); describe('local publisher', () => { it('should publish generated documentation dir', async () => { - const testConfig = ConfigReader.fromConfigs([{ context: '', data: {} }]); - const publisher = new LocalPublish(logger, testConfig); + const testConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + backend: { + baseUrl: 'http://localhost:7000', + listen: { + port: 7000, + }, + }, + }, + }, + ]); + const testDiscovery = SingleHostDiscovery.fromConfig(testConfig); + const publisher = new LocalPublish(logger, testDiscovery); const mockEntity = createMockEntity(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts index 221b0d48e7..9f066a919c 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts @@ -18,15 +18,15 @@ import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; import { PublisherBase } from './types'; import { resolvePackagePath } from '@backstage/backend-common'; -import { Config } from '@backstage/config'; +import { DiscoveryApi } from '@backstage/core-api'; export class LocalPublish implements PublisherBase { private readonly logger: Logger; - private readonly config: Config; + private readonly discoveryApi: DiscoveryApi; - constructor(logger: Logger, config: Config) { + constructor(logger: Logger, discoveryApi: DiscoveryApi) { this.logger = logger; - this.config = config; + this.discoveryApi = discoveryApi; } publish({ @@ -66,9 +66,10 @@ export class LocalPublish implements PublisherBase { reject(err); } - const backendBaseUrl = this.config.getString('backend.baseUrl'); - resolve({ - remoteUrl: `${backendBaseUrl}/api/techdocs/static/docs/${entity.metadata.name}`, + this.discoveryApi.getBaseUrl('techdocs').then(techdocsApiUrl => { + resolve({ + remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + }); }); }); }); diff --git a/yarn.lock b/yarn.lock index 10e70f52fb..9d4984e152 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1290,40 +1290,34 @@ to-fast-properties "^2.0.0" "@backstage/core@^0.2.0": - version "0.3.0" + version "0.2.0" + resolved "https://registry.npmjs.org/@backstage/core/-/core-0.2.0.tgz#543246b2d87563c9aa4d9fb96e40fdfc7e827520" + integrity sha512-75m2u3FoUngBOvt9l65xZcYTzzB+49OXpY1A9VNFUR1+jMs3cL/0HDfByQV2H0xXaHzMngQ8C5u/sWhkQsij1w== dependencies: "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.1" - "@backstage/theme" "^0.2.1" + "@backstage/core-api" "^0.2.0" + "@backstage/theme" "^0.2.0" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" - "@types/dagre" "^0.7.44" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" clsx "^1.1.0" - d3-selection "^2.0.0" - d3-shape "^2.0.0" - d3-zoom "^2.0.0" - dagre "^0.8.5" immer "^7.0.9" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" - qs "^6.9.4" rc-progress "^3.0.0" react "^16.12.0" react-dom "^16.12.0" react-helmet "6.1.0" react-hook-form "^6.6.0" - react-markdown "^5.0.2" react-router "6.0.0-beta.0" react-router-dom "6.0.0-beta.0" react-sparklines "^1.7.0" react-syntax-highlighter "^13.5.1" react-use "^15.3.3" - remark-gfm "^1.0.0" "@bcoe/v8-coverage@^0.2.3": version "0.2.3" From 2723a607915021dcfd71222187aa17c01fe4edee Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 17 Nov 2020 21:17:19 +0100 Subject: [PATCH 086/259] Update template app to use new API params for LocalPublish --- .../default-app/packages/backend/src/plugins/techdocs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index 5506228962..ac4d81a8e8 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -28,7 +28,7 @@ export default async function createPlugin({ preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); - const publisher = new LocalPublish(logger); + const publisher = new LocalPublish(logger, discovery); const dockerClient = new Docker(); From a6a61f192c89c65c1293b07ac3398237d2a6a957 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Tue, 17 Nov 2020 17:28:30 -0500 Subject: [PATCH 087/259] fix tooltip for long period separated titles --- .../src/components/BarChart/BarChartTooltip.tsx | 7 ++++--- plugins/cost-insights/src/utils/styles.ts | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx index 2cb09b724e..8ab787a443 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx @@ -35,7 +35,6 @@ export const BarChartTooltip = ({ children, }: PropsWithChildren) => { const classes = useStyles(); - return ( - {title} + + {title} + {subtitle && ( {subtitle} )} - {topRight} + {topRight && {topRight}} {content && ( diff --git a/plugins/cost-insights/src/utils/styles.ts b/plugins/cost-insights/src/utils/styles.ts index de743d8459..1e8fd50899 100644 --- a/plugins/cost-insights/src/utils/styles.ts +++ b/plugins/cost-insights/src/utils/styles.ts @@ -386,7 +386,7 @@ export const useTooltipStyles = makeStyles( boxShadow: theme.shadows[1], color: theme.palette.tooltip.color, fontSize: theme.typography.fontSize, - width: 250, + maxWidth: 300, }, actions: { padding: theme.spacing(2), @@ -403,6 +403,12 @@ export const useTooltipStyles = makeStyles( divider: { backgroundColor: emphasize(theme.palette.divider, 1), }, + truncate: { + maxWidth: 200, + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + }, subtitle: { fontStyle: 'italic', }, From 74b9561a2bf77d9e462125a8b68033c6f4db663c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 01:44:17 +0100 Subject: [PATCH 088/259] github/workdflows: bump setup-msbuild to fix windows builds (#3321) --- .github/workflows/e2e-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-win.yml b/.github/workflows/e2e-win.yml index 570d857c48..1844700cea 100644 --- a/.github/workflows/e2e-win.yml +++ b/.github/workflows/e2e-win.yml @@ -32,7 +32,7 @@ jobs: with: node-version: ${{ matrix.node-version }} - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v1.0.1 + uses: microsoft/setup-msbuild@v1.0.2 - name: yarn install run: yarn install --frozen-lockfile From 3efd03c0e8304d8d58b79e0ed27346f645c7bf5e Mon Sep 17 00:00:00 2001 From: Andrew Thauer <6507159+andrewthauer@users.noreply.github.com> Date: Tue, 17 Nov 2020 22:11:16 -0500 Subject: [PATCH 089/259] chore: remove obsolete circleci proxy config --- .changeset/loud-pets-clap.md | 5 +++++ packages/app/package.json | 11 +---------- 2 files changed, 6 insertions(+), 10 deletions(-) create mode 100644 .changeset/loud-pets-clap.md diff --git a/.changeset/loud-pets-clap.md b/.changeset/loud-pets-clap.md new file mode 100644 index 0000000000..0d50c066f9 --- /dev/null +++ b/.changeset/loud-pets-clap.md @@ -0,0 +1,5 @@ +--- +'example-app': patch +--- + +Removed obsolete CircleCI proxy config from example-app diff --git a/packages/app/package.json b/packages/app/package.json index f013200fbb..1686306232 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -86,14 +86,5 @@ "last 1 safari version" ] }, - "license": "Apache-2.0", - "proxy": { - "/circleci/api": { - "target": "https://circleci.com/api/v1.1", - "changeOrigin": true, - "pathRewrite": { - "^/circleci/api/": "/" - } - } - } + "license": "Apache-2.0" } From 9023404510fc9579a8f755482d695cf49a67a5c9 Mon Sep 17 00:00:00 2001 From: Joel Low Date: Wed, 18 Nov 2020 14:11:15 +0800 Subject: [PATCH 090/259] feat(cli): Support specifying listen host/port for frontend --- .changeset/modern-boats-knock.md | 5 +++++ packages/cli/src/lib/bundler/server.ts | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 .changeset/modern-boats-knock.md diff --git a/.changeset/modern-boats-knock.md b/.changeset/modern-boats-knock.md new file mode 100644 index 0000000000..16f80fe041 --- /dev/null +++ b/.changeset/modern-boats-knock.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Support specifying listen host/port for frontend diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index e1fb560e22..2710145d83 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -25,7 +25,12 @@ import { resolveBundlingPaths } from './paths'; export async function serveBundle(options: ServeOptions) { const url = resolveBaseUrl(options.config); - const port = Number(url.port) || (url.protocol === 'https:' ? 443 : 80); + const host = + options.config.getOptionalString('app.listen.host') || url.hostname; + const port = + options.config.getOptionalNumber('app.listen.port') || + Number(url.port) || + (url.protocol === 'https:' ? 443 : 80); const paths = resolveBundlingPaths(options); const pkgPath = paths.targetPackageJson; @@ -50,7 +55,7 @@ export async function serveBundle(options: ServeOptions) { clientLogLevel: 'warning', stats: 'errors-warnings', https: url.protocol === 'https:', - host: url.hostname, + host, port, proxy: pkg.proxy, }); From 3457c40f81dc178a1e401acbb703314fc96e501a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 10:24:15 +0100 Subject: [PATCH 091/259] Apply suggestions from code review Co-authored-by: Adam Harvey --- docs/overview/stability-index.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index 96ff33542f..1ec90e106d 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -349,12 +349,15 @@ Stability: `0` ### [`kubernetes`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes/) The frontend component of the Kubernetes plugin, used to browse and visualize -Kubernetes resources. Stability: `1`. +Kubernetes resources. + +Stability: `1`. ### [`kubernetes-backend`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend/) The backend component of the Kubernetes plugin, used to fetch Kubernetes resources from clusters and associate them with entities in the Catalog. + Stability: `1`. ### [`lighthouse`](https://github.com/backstage/backstage/tree/master/plugins/lighthouse/) @@ -367,7 +370,9 @@ Stability: `0` ### [`newrelic`](https://github.com/backstage/backstage/tree/master/plugins/newrelic/) Observability platform built to help engineers create and monitor their -software. Stability: `0` +software. + +Stability: `0` ### [`proxy-backend`](https://github.com/backstage/backstage/tree/master/plugins/proxy-backend/) From 7d680c075e9fd75122a4a3ac3dc8cb50e7f7bcdc Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Wed, 18 Nov 2020 11:20:37 +0100 Subject: [PATCH 092/259] [Search] Search frontend plugin (#3086) * feat(search) search plugin * feat(search): search page components * feat(search): api * feat(search): filters wip * feat(search): wip filters * fix(search): delete unused useParams hook * fix(search): update docs * fix(search): use latest versions of dependencies * fix(search): change version of catalog plugin * fix(search api): pass instance of catalog api to search api * fix(filters): rename component from FilterButton to FiltersButton * fixup * fix(filters): use list of checkboxes to match style of catalog page filters * fix(styles): use theme spacing for margins and paddings, delete unused styles * fix(search): change terminology of search input to be more consistent and clear * fix(search): restructure component exports according to ADR * fix(search): replace sm with xs on Grid components to support smaller screens * fix(search): add types * fixup * fix(search): bump backstage core * change versions of backstage theme and dev-utils --- .github/CODEOWNERS | 1 + packages/app/package.json | 1 + packages/app/src/plugins.ts | 1 + plugins/search/.eslintrc.js | 3 + plugins/search/README.md | 9 + plugins/search/dev/index.tsx | 19 ++ plugins/search/package.json | 49 ++++ plugins/search/src/apis.ts | 52 ++++ .../search/src/components/Filters/Filters.tsx | 132 ++++++++++ .../src/components/Filters/FiltersButton.tsx | 56 +++++ .../search/src/components/Filters/index.tsx | 19 ++ .../src/components/SearchBar/SearchBar.tsx | 69 ++++++ .../search/src/components/SearchBar/index.tsx | 17 ++ .../src/components/SearchPage/SearchPage.tsx | 55 +++++ .../src/components/SearchPage/index.tsx | 17 ++ .../components/SearchResult/SearchResult.tsx | 227 ++++++++++++++++++ .../src/components/SearchResult/index.tsx | 17 ++ plugins/search/src/components/index.tsx | 20 ++ plugins/search/src/index.ts | 16 ++ plugins/search/src/plugin.test.ts | 22 ++ plugins/search/src/plugin.ts | 29 +++ plugins/search/src/setupTests.ts | 17 ++ 22 files changed, 848 insertions(+) create mode 100644 plugins/search/.eslintrc.js create mode 100644 plugins/search/README.md create mode 100644 plugins/search/dev/index.tsx create mode 100644 plugins/search/package.json create mode 100644 plugins/search/src/apis.ts create mode 100644 plugins/search/src/components/Filters/Filters.tsx create mode 100644 plugins/search/src/components/Filters/FiltersButton.tsx create mode 100644 plugins/search/src/components/Filters/index.tsx create mode 100644 plugins/search/src/components/SearchBar/SearchBar.tsx create mode 100644 plugins/search/src/components/SearchBar/index.tsx create mode 100644 plugins/search/src/components/SearchPage/SearchPage.tsx create mode 100644 plugins/search/src/components/SearchPage/index.tsx create mode 100644 plugins/search/src/components/SearchResult/SearchResult.tsx create mode 100644 plugins/search/src/components/SearchResult/index.tsx create mode 100644 plugins/search/src/components/index.tsx create mode 100644 plugins/search/src/index.ts create mode 100644 plugins/search/src/plugin.test.ts create mode 100644 plugins/search/src/plugin.ts create mode 100644 plugins/search/src/setupTests.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d818c0d6b5..b3c9bc653d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,5 +9,6 @@ /plugins/cost-insights @backstage/silver-lining /plugins/cloudbuild @trivago/ebarrios /plugins/techdocs @backstage/techdocs-core +/plugins/search @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /.changeset/cost-insights-* @backstage/silver-lining diff --git a/packages/app/package.json b/packages/app/package.json index f013200fbb..dea63e3a4c 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -25,6 +25,7 @@ "@backstage/plugin-rollbar": "^0.2.1", "@backstage/plugin-scaffolder": "^0.3.0", "@backstage/plugin-sentry": "^0.2.1", + "@backstage/plugin-search": "^0.2.0", "@backstage/plugin-tech-radar": "^0.3.0", "@backstage/plugin-techdocs": "^0.2.1", "@backstage/plugin-user-settings": "^0.2.1", diff --git a/packages/app/src/plugins.ts b/packages/app/src/plugins.ts index c61a16e59c..d6577b4ce4 100644 --- a/packages/app/src/plugins.ts +++ b/packages/app/src/plugins.ts @@ -39,3 +39,4 @@ export { plugin as CostInsights } from '@backstage/plugin-cost-insights'; export { plugin as GitHubInsights } from '@roadiehq/backstage-plugin-github-insights'; export { plugin as UserSettings } from '@backstage/plugin-user-settings'; export { plugin as BuildKite } from '@roadiehq/backstage-plugin-buildkite'; +export { plugin as Search } from '@backstage/plugin-search'; diff --git a/plugins/search/.eslintrc.js b/plugins/search/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/plugins/search/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/plugins/search/README.md b/plugins/search/README.md new file mode 100644 index 0000000000..fb4b047811 --- /dev/null +++ b/plugins/search/README.md @@ -0,0 +1,9 @@ +# Backstage Search + +**This plugin is still under development.** + +You can follow the progress under the Global search in Backstage [milestone](https://github.com/backstage/backstage/milestone/21) or reach out to us in the #search Discord channel. + +## Getting started + +Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search)to check out the plugin. diff --git a/plugins/search/dev/index.tsx b/plugins/search/dev/index.tsx new file mode 100644 index 0000000000..264d6f801f --- /dev/null +++ b/plugins/search/dev/index.tsx @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createDevApp } from '@backstage/dev-utils'; +import { plugin } from '../src/plugin'; + +createDevApp().registerPlugin(plugin).render(); diff --git a/plugins/search/package.json b/plugins/search/package.json new file mode 100644 index 0000000000..6f356d1bf4 --- /dev/null +++ b/plugins/search/package.json @@ -0,0 +1,49 @@ +{ + "name": "@backstage/plugin-search", + "version": "0.2.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "build": "backstage-cli plugin:build", + "start": "backstage-cli plugin:serve", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "diff": "backstage-cli plugin:diff", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", + "@material-ui/core": "^4.11.0", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "4.0.0-alpha.45", + "@backstage/plugin-catalog": "^0.2.0", + "react-router-dom": "6.0.0-beta.0", + "react": "^16.13.1", + "react-dom": "^16.13.1", + "react-use": "^15.3.3" + }, + "devDependencies": { + "@backstage/cli": "^0.2.0", + "@backstage/dev-utils": "^0.1.3", + "@backstage/test-utils": "^0.1.2", + "@testing-library/jest-dom": "^5.10.1", + "@testing-library/react": "^10.4.1", + "@testing-library/user-event": "^12.0.7", + "@types/jest": "^26.0.7", + "@types/node": "^12.0.0", + "msw": "^0.21.2", + "cross-fetch": "^3.0.6" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts new file mode 100644 index 0000000000..1783b91d2a --- /dev/null +++ b/plugins/search/src/apis.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogApi } from '@backstage/plugin-catalog'; + +export type Result = { + name: string; + description: string; + owner: string; + kind: string; + lifecycle: string; +}; + +export type SearchResults = Array; + +class SearchApi { + private catalogApi: CatalogApi; + + constructor(catalogApi: CatalogApi) { + this.catalogApi = catalogApi; + } + + private async entities() { + const entities = await this.catalogApi.getEntities(); + return entities.map((result: any) => ({ + name: result.metadata.name, + description: result.metadata.description, + owner: result.spec.owner, + kind: result.kind, + lifecycle: result.spec.lifecycle, + })); + } + + public getSearchResult(): Promise { + return this.entities(); + } +} + +export default SearchApi; diff --git a/plugins/search/src/components/Filters/Filters.tsx b/plugins/search/src/components/Filters/Filters.tsx new file mode 100644 index 0000000000..19d4e53d72 --- /dev/null +++ b/plugins/search/src/components/Filters/Filters.tsx @@ -0,0 +1,132 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { + makeStyles, + Typography, + Divider, + Card, + CardHeader, + Button, + CardContent, + Select, + Checkbox, + List, + ListItem, + ListItemText, + MenuItem, +} from '@material-ui/core'; + +const useStyles = makeStyles(theme => ({ + filters: { + background: 'transparent', + boxShadow: '0px 0px 0px 0px', + }, + checkbox: { + padding: theme.spacing(0, 1, 0, 1), + }, + dropdown: { + width: '100%', + }, +})); + +export type FiltersState = { + selected: string; + checked: Array; +}; + +type FiltersProps = { + filters: FiltersState; + resetFilters: () => void; + updateSelected: (filter: string) => void; + updateChecked: (filter: string) => void; +}; + +export const Filters = ({ + filters, + resetFilters, + updateSelected, + updateChecked, +}: FiltersProps) => { + const classes = useStyles(); + + // TODO: move mocked filters out of filters component to make it more generic + const filter1 = ['All', 'API', 'Component', 'Location', 'Template']; + const filter2 = ['deprecated', 'recommended', 'experimental', 'production']; + + return ( + + Filters} + action={ + + } + /> + + + Kind + + + + Lifecycle + + {filter2.map(filter => ( + updateChecked(filter)} + > + + + + ))} + + + + ); +}; diff --git a/plugins/search/src/components/Filters/FiltersButton.tsx b/plugins/search/src/components/Filters/FiltersButton.tsx new file mode 100644 index 0000000000..4775e9d8b8 --- /dev/null +++ b/plugins/search/src/components/Filters/FiltersButton.tsx @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import FilterListIcon from '@material-ui/icons/FilterList'; +import { makeStyles, IconButton, Typography } from '@material-ui/core'; + +const useStyles = makeStyles(theme => ({ + filters: { + width: '250px', + display: 'flex', + }, + icon: { + margin: theme.spacing(-1, 0, 0, 0), + }, +})); + +type FiltersButtonProps = { + numberOfSelectedFilters: number; + handleToggleFilters: () => void; +}; + +export const FiltersButton = ({ + numberOfSelectedFilters, + handleToggleFilters, +}: FiltersButtonProps) => { + const classes = useStyles(); + + return ( +
+ + + + + Filters ({numberOfSelectedFilters ? numberOfSelectedFilters : 0}) + +
+ ); +}; diff --git a/plugins/search/src/components/Filters/index.tsx b/plugins/search/src/components/Filters/index.tsx new file mode 100644 index 0000000000..ea431a3c01 --- /dev/null +++ b/plugins/search/src/components/Filters/index.tsx @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { FiltersButton } from './FiltersButton'; +export { Filters } from './Filters'; +export type { FiltersState } from './Filters'; diff --git a/plugins/search/src/components/SearchBar/SearchBar.tsx b/plugins/search/src/components/SearchBar/SearchBar.tsx new file mode 100644 index 0000000000..9aa48e7284 --- /dev/null +++ b/plugins/search/src/components/SearchBar/SearchBar.tsx @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { makeStyles } from '@material-ui/core/styles'; +import { Paper } from '@material-ui/core'; +import InputBase from '@material-ui/core/InputBase'; +import IconButton from '@material-ui/core/IconButton'; +import SearchIcon from '@material-ui/icons/Search'; +import ClearButton from '@material-ui/icons/Clear'; + +const useStyles = makeStyles(() => ({ + root: { + display: 'flex', + alignItems: 'center', + }, + input: { + flex: 1, + }, +})); + +type SearchBarProps = { + searchQuery: string; + handleSearch: any; + handleClearSearchBar: any; +}; + +export const SearchBar = ({ + searchQuery, + handleSearch, + handleClearSearchBar, +}: SearchBarProps) => { + const classes = useStyles(); + + return ( + handleSearch(e)} + className={classes.root} + > + + + + handleSearch(e)} + inputProps={{ 'aria-label': 'search backstage' }} + /> + handleClearSearchBar()}> + + + + ); +}; diff --git a/plugins/search/src/components/SearchBar/index.tsx b/plugins/search/src/components/SearchBar/index.tsx new file mode 100644 index 0000000000..e2b8af1946 --- /dev/null +++ b/plugins/search/src/components/SearchBar/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchBar } from './SearchBar'; diff --git a/plugins/search/src/components/SearchPage/SearchPage.tsx b/plugins/search/src/components/SearchPage/SearchPage.tsx new file mode 100644 index 0000000000..b5779a3d6e --- /dev/null +++ b/plugins/search/src/components/SearchPage/SearchPage.tsx @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState } from 'react'; + +import { Header, Content, Page } from '@backstage/core'; +import { Grid } from '@material-ui/core'; + +import { SearchBar } from '../SearchBar'; +import { SearchResult } from '../SearchResult'; + +export const SearchPage = () => { + const [searchQuery, setSearchQuery] = useState(''); + + const handleSearch = (event: React.ChangeEvent) => { + event.preventDefault(); + setSearchQuery(event.target.value); + }; + + const handleClearSearchBar = () => { + setSearchQuery(''); + }; + + return ( + +
+ + + + + + + + + + + + ); +}; diff --git a/plugins/search/src/components/SearchPage/index.tsx b/plugins/search/src/components/SearchPage/index.tsx new file mode 100644 index 0000000000..acdc0967ab --- /dev/null +++ b/plugins/search/src/components/SearchPage/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchPage } from './SearchPage'; diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx new file mode 100644 index 0000000000..a4985c20fb --- /dev/null +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -0,0 +1,227 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState, useEffect } from 'react'; +import { useAsync } from 'react-use'; + +import { makeStyles, Typography, Grid, Divider } from '@material-ui/core'; +import { Table, TableColumn, useApi } from '@backstage/core'; +import { catalogApiRef } from '@backstage/plugin-catalog'; + +import { FiltersButton, Filters, FiltersState } from '../Filters'; +import SearchApi, { Result, SearchResults } from '../../apis'; + +const useStyles = makeStyles(theme => ({ + searchTerm: { + background: '#eee', + borderRadius: '10%', + }, + tableHeader: { + margin: theme.spacing(1, 0, 0, 0), + display: 'flex', + }, + divider: { + width: '1px', + margin: theme.spacing(0, 2), + padding: theme.spacing(2, 0), + }, +})); + +type SearchResultProps = { + searchQuery?: string; +}; + +type TableHeaderProps = { + searchQuery?: string; + numberOfSelectedFilters: number; + numberOfResults: number; + handleToggleFilters: () => void; +}; + +type Filters = { + selected: string; + checked: Array; +}; + +// TODO: move out column to make the search result component more generic +const columns: TableColumn[] = [ + { + title: 'Component Id', + field: 'name', + highlight: true, + }, + { + title: 'Description', + field: 'description', + }, + { + title: 'Owner', + field: 'owner', + }, + { + title: 'Kind', + field: 'kind', + }, + { + title: 'LifeCycle', + field: 'lifecycle', + }, +]; + +const TableHeader = ({ + searchQuery, + numberOfSelectedFilters, + numberOfResults, + handleToggleFilters, +}: TableHeaderProps) => { + const classes = useStyles(); + + return ( +
+ + + + {searchQuery ? ( + + {`${numberOfResults} `} + {numberOfResults > 1 ? `results for ` : `result for `} + "{searchQuery}"{' '} + + ) : ( + {`${numberOfResults} results`} + )} + +
+ ); +}; + +export const SearchResult = ({ searchQuery }: SearchResultProps) => { + const catalogApi = useApi(catalogApiRef); + + const [showFilters, toggleFilters] = useState(false); + const [filters, setFilters] = useState({ + selected: 'All', + checked: [], + }); + + const [filteredResults, setFilteredResults] = useState([]); + + const searchApi = new SearchApi(catalogApi); + + const { loading, error, value: results } = useAsync(() => { + return searchApi.getSearchResult(); + }, []); + + useEffect(() => { + if (results) { + let withFilters = results; + + // apply filters + + // filter on selected + if (filters.selected !== 'All') { + withFilters = results.filter((result: Result) => + filters.selected.includes(result.kind), + ); + } + + // filter on checked + if (filters.checked.length > 0) { + withFilters = withFilters.filter((result: Result) => + filters.checked.includes(result.lifecycle), + ); + } + + // filter on searchQuery + if (searchQuery) { + withFilters = withFilters.filter( + (result: Result) => + result.name?.toLowerCase().includes(searchQuery) || + result.description?.toLowerCase().includes(searchQuery), + ); + } + + setFilteredResults(withFilters); + } + }, [filters, searchQuery, results]); + + if (loading || error || !results) return null; + + const resetFilters = () => { + setFilters({ + selected: 'All', + checked: [], + }); + }; + + const updateSelected = (filter: string) => { + setFilters(prevState => ({ + ...prevState, + selected: filter, + })); + }; + + const updateChecked = (filter: string) => { + if (filters.checked.includes(filter)) { + setFilters(prevState => ({ + ...prevState, + checked: prevState.checked.filter(item => item !== filter), + })); + return; + } + + setFilters(prevState => ({ + ...prevState, + checked: [...prevState.checked, filter], + })); + }; + + return ( + <> + + {showFilters && ( + + + + )} + +
toggleFilters(!showFilters)} + /> + } + /> + + + + ); +}; diff --git a/plugins/search/src/components/SearchResult/index.tsx b/plugins/search/src/components/SearchResult/index.tsx new file mode 100644 index 0000000000..cf10135fd0 --- /dev/null +++ b/plugins/search/src/components/SearchResult/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchResult } from './SearchResult'; diff --git a/plugins/search/src/components/index.tsx b/plugins/search/src/components/index.tsx new file mode 100644 index 0000000000..f8e6a5a09e --- /dev/null +++ b/plugins/search/src/components/index.tsx @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Filters'; +export * from './SearchBar'; +export * from './SearchPage'; +export * from './SearchResult'; diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts new file mode 100644 index 0000000000..224e293890 --- /dev/null +++ b/plugins/search/src/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { plugin } from './plugin'; diff --git a/plugins/search/src/plugin.test.ts b/plugins/search/src/plugin.test.ts new file mode 100644 index 0000000000..92b8d5bcbf --- /dev/null +++ b/plugins/search/src/plugin.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { plugin } from './plugin'; + +describe('search', () => { + it('should export plugin', () => { + expect(plugin).toBeDefined(); + }); +}); diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts new file mode 100644 index 0000000000..44c7bcb042 --- /dev/null +++ b/plugins/search/src/plugin.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createPlugin, createRouteRef } from '@backstage/core'; +import { SearchPage } from './components/SearchPage'; + +export const rootRouteRef = createRouteRef({ + path: '/search', + title: 'search', +}); + +export const plugin = createPlugin({ + id: 'search', + register({ router }) { + router.addRoute(rootRouteRef, SearchPage); + }, +}); diff --git a/plugins/search/src/setupTests.ts b/plugins/search/src/setupTests.ts new file mode 100644 index 0000000000..43b8421558 --- /dev/null +++ b/plugins/search/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; From ddb5a38f7782089da0f0875bbe0339daa6adfb4a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 7 Nov 2020 17:51:16 +0100 Subject: [PATCH 093/259] backend-common,core,auth-backend: added initial config schemas --- packages/backend-common/config-schema.json | 80 ++++++++++++++++++++++ packages/backend-common/package.json | 3 +- packages/core/package.json | 51 +++++++++++++- plugins/auth-backend/package.json | 35 +++++++++- 4 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 packages/backend-common/config-schema.json diff --git a/packages/backend-common/config-schema.json b/packages/backend-common/config-schema.json new file mode 100644 index 0000000000..2012447a11 --- /dev/null +++ b/packages/backend-common/config-schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/backend-common", + "type": "object", + "properties": { + "integrations": { + "type": "object", + "additionalProperties": false, + "properties": { + "github": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "token": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "gitlab": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "token": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "bitbucket": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "username": { + "type": "string" + }, + "appPassword": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "azure": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "token": { + "type": "string", + "visibility": "secret" + } + } + } + } + } + } + } +} diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index ea84d1a383..2f58bd0536 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -90,5 +90,6 @@ }, "files": [ "dist" - ] + ], + "configSchema": "config-schema.json" } diff --git a/packages/core/package.json b/packages/core/package.json index 848f0847bd..3d4b53d35b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -80,5 +80,54 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/core", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "title": { + "type": "string", + "visibility": "frontend" + }, + "baseUrl": { + "type": "string", + "visibility": "frontend" + } + }, + "required": [ + "title", + "baseUrl" + ] + }, + "backend": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "visibility": "frontend" + } + }, + "required": [ + "baseUrl" + ] + }, + "organization": { + "type": "object", + "properties": { + "name": { + "type": "string", + "visibility": "frontend" + } + } + } + }, + "required": [ + "app", + "backend" + ] + } } diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index af3d056b28..2bc00f56a2 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -66,5 +66,38 @@ "files": [ "dist", "migrations" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/plugin-auth-backend", + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "providers": { + "type": "object", + "properties": { + "google": { + "type": "object", + "visibility": "frontend", + "additionalProperties": { + "type": "object", + "properties": { + "clientId": { + "type": "string" + }, + "clientSecret": { + "type": "string", + "visibility": "secret" + } + } + } + } + } + } + } + } + } + } } From 6fec95079a18d2da1c8036d16809dac0d67388dc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 7 Nov 2020 18:08:19 +0100 Subject: [PATCH 094/259] config-loader: initial incomplete config schema implementation --- packages/config-loader/package.json | 6 + packages/config-loader/src/lib/index.ts | 1 + packages/config-loader/src/lib/schema.ts | 216 +++++++++++++++++++++++ packages/config-loader/src/loader.ts | 10 +- yarn.lock | 32 +++- 5 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 packages/config-loader/src/lib/schema.ts diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 2d3474f3af..39735ff2fd 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,8 +30,14 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", + "@types/json-schema": "^7.0.6", + "@types/json-schema-merge-allof": "^0.6.0", + "ajv": "^7.0.0-beta.2", "fs-extra": "^9.0.0", + "json-schema": "^0.2.5", + "json-schema-merge-allof": "^0.7.0", "yaml": "^1.9.2", "yup": "^0.29.3" }, diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 1b5ad2ef50..29812053bc 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; +export { loadSchema } from './schema'; diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts new file mode 100644 index 0000000000..c3441bce38 --- /dev/null +++ b/packages/config-loader/src/lib/schema.ts @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; +import Ajv from 'ajv'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; +import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; +import { AppConfig, ConfigReader } from '@backstage/config'; + +const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; + +type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; + +type ConfigSchema = { + load(appConfigs: AppConfig[]): AppConfig[]; +}; + +type ConfigSchemaPackageEntry = { + value: JSONSchema; + path: string; +}; + +type Options = { + dependencies: string[]; +}; + +type ValidationError = string; + +type ValidationResult = { + errors?: ValidationError[]; + visibilities: Map; +}; + +type ValidationFunc = (configs: AppConfig[]) => ValidationResult; + +export async function loadSchema(options: Options): Promise { + const start = process.hrtime(); + + const schemas = await collectConfigSchemas(options.dependencies[0]); + + const [durS, durNs] = process.hrtime(start); + const dur = (durS + durNs / 10 ** 9).toFixed(3); + console.log(`DEBUG: collected config schemas in ${dur}s`); + console.log('DEBUG: schemas =', schemas); + + const validate = compileConfigSchemas(schemas); + + return { + load(configs: AppConfig[]): AppConfig[] { + const result = validate(configs); + console.log('DEBUG: result =', result); + if (result.errors) { + throw new Error( + `Config validation failed, ${result.errors.join('; ')}`, + ); + } + + return configs; + }, + }; +} + +function compileConfigSchemas( + schemas: ConfigSchemaPackageEntry[], +): ValidationFunc { + const visibilities = new Map(); + + const ajv = new Ajv({ + strict: true, + allErrors: true, + defaultMeta: 'http://json-schema.org/draft-07/schema#', + schemas: { + 'https://backstage.io/schema/config-v1': true, + }, + keywords: [ + { + keyword: 'visibility', + schemaType: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, ctx) => { + if (!ctx) { + return false; + } + if (visibility === 'frontend') { + visibilities.set(ctx.dataPath, visibility); + } + return true; + }; + }, + }, + ], + }); + + const merged = mergeAllOf( + { allOf: schemas.map(_ => _.value) }, + { + ignoreAdditionalProperties: true, + resolvers: { + visibility(values: string[], path: string[]) { + const hasApp = values.some(_ => _ === 'frontend'); + const hasSecret = values.some(_ => _ === 'secret'); + if (hasApp && hasSecret) { + throw new Error( + `Config schema visibility is both 'frontend' and 'secret' for ${path.join( + '/', + )}`, + ); + } else if (hasApp) { + return 'frontend'; + } else if (hasSecret) { + return 'secret'; + } + + return 'backend'; + }, + } as Partial>, + }, + ); + + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilities.clear(); + const valid = validate(config); + if (!valid) { + const errors = ajv.errorsText(validate.errors); + return { + errors: [errors], + visibilities: new Map(), + }; + } + + return { + visibilities: new Map(visibilities), + }; + }; +} + +const req = + typeof __non_webpack_require__ === 'undefined' + ? require + : __non_webpack_require__; + +async function collectConfigSchemas( + name: string, + opts: unknown = {}, + visited: Set = new Set(), + schemas: ConfigSchemaPackageEntry[] = [], +): Promise { + const pkgPath = req.resolve(`${name}/package.json`, opts); + if (visited.has(pkgPath)) { + return schemas; + } + visited.add(pkgPath); + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return schemas; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + for (const depName of depNames) { + await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); + } + + return schemas; +} diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index e7eb2a38fa..318eaa7915 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname, isAbsolute } from 'path'; import { AppConfig, JsonObject } from '@backstage/config'; -import { readConfigFile, readEnvConfig, readSecret } from './lib'; +import { readConfigFile, readEnvConfig, readSecret, loadSchema } from './lib'; export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -71,13 +71,17 @@ class Context { } } +type LoadedConfig = AppConfig[]; + export async function loadConfig( options: LoadConfigOptions, -): Promise { +): Promise { const configs = []; const { configRoot } = options; const configPaths = options.configPaths.slice(); + const schema = await loadSchema({ dependencies: ['example-backend'] }); + // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` if (configPaths.length === 0) { @@ -126,5 +130,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return configs; + return schema.load(configs); } diff --git a/yarn.lock b/yarn.lock index 10e70f52fb..98e1b2476f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5369,7 +5369,14 @@ resolved "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-3.12.5.tgz#136d5e6a57a931e1cce6f9d8126aa98a9c92a6bb" integrity sha512-JCcp6J0GV66Y4ZMDAQCXot4xprYB+Zfd3meK9+INSJeVZwJmHAW30BBEEkPzXswMXuiyReUGOP3GxrADc9wPww== -"@types/json-schema@*", "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5": +"@types/json-schema-merge-allof@^0.6.0": + version "0.6.0" + resolved "https://registry.npmjs.org/@types/json-schema-merge-allof/-/json-schema-merge-allof-0.6.0.tgz#0f587d8a3bcb41a55ef2e91d3ba96430c9bc1813" + integrity sha512-v6iCEk4Sxy1twlCTtrRxMqSHX0vuLJ7Ql4hiIUZRMOswxtlUeybIfYaZIj7pX747RBczG2YtBm4Fn6sqKzeY2Q== + dependencies: + "@types/json-schema" "*" + +"@types/json-schema@*", "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6": version "7.0.6" resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== @@ -6562,6 +6569,15 @@ ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.1, ajv@^6.10.2, ajv@^6.12.4, ajv@^6.12.5, ajv json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ajv@^7.0.0-beta.2: + version "7.0.0-beta.3" + resolved "https://registry.npmjs.org/ajv/-/ajv-7.0.0-beta.3.tgz#d34861ccfbdebb55bf9f49a08b29f76bf656fc5c" + integrity sha512-gUGVvM4NmyqrFvCNAQnP4P7FC0RjxMQyRnrXpozNglBkDJnTysVbvycyOZUy5n6yLKSqVDUqWZBXj7dXINrSqw== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^0.5.0" + uri-js "^4.2.2" + alphanum-sort@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3" @@ -15022,11 +15038,25 @@ json-schema-merge-allof@^0.6.0: json-schema-compare "^0.2.2" lodash "^4.17.4" +json-schema-merge-allof@^0.7.0: + version "0.7.0" + resolved "https://registry.npmjs.org/json-schema-merge-allof/-/json-schema-merge-allof-0.7.0.tgz#84d3e8c3e03d3060014286958eb8834fa9d76304" + integrity sha512-kvsuSVnl1n5xnNEu5ed4o8r8ujSA4/IgRtHmpgfMfa7FOMIRAzN4F9qbuklouTn5J8bi83y6MQ11n+ERMMTXZg== + dependencies: + compute-lcm "^1.1.0" + json-schema-compare "^0.2.2" + lodash "^4.17.4" + json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== +json-schema-traverse@^0.5.0: + version "0.5.0" + resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.5.0.tgz#1069a6097f9e9a567bfc6cc215e85f606ebf6480" + integrity sha512-x+TRJIQFskrNnFKE2Viz9FCSjK1vIh+H/uaBiOYszh/IcZmAFneQ35H4osWDJp1NPXccuV2I0RMXmi2ZS6Kqcg== + json-schema@0.2.3: version "0.2.3" resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" From 47952780392b522827707c6ffbcf5a58771c89c2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:05:56 +0100 Subject: [PATCH 095/259] config-loader: add config filtering by visibility --- packages/config-loader/src/lib/schema.ts | 97 +++++++++++++++++++++--- packages/config-loader/src/loader.ts | 2 +- 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts index c3441bce38..0d820ab173 100644 --- a/packages/config-loader/src/lib/schema.ts +++ b/packages/config-loader/src/lib/schema.ts @@ -19,14 +19,24 @@ import { resolve as resolvePath, dirname } from 'path'; import Ajv from 'ajv'; import { JSONSchema7 as JSONSchema } from 'json-schema'; import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; -import { AppConfig, ConfigReader } from '@backstage/config'; +import { + AppConfig, + ConfigReader, + JsonObject, + JsonValue, +} from '@backstage/config'; const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; +const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; + type ConfigSchema = { - load(appConfigs: AppConfig[]): AppConfig[]; + process( + appConfigs: AppConfig[], + options?: ConfigProcessingOptions, + ): AppConfig[]; }; type ConfigSchemaPackageEntry = { @@ -42,11 +52,61 @@ type ValidationError = string; type ValidationResult = { errors?: ValidationError[]; - visibilities: Map; + visibilityByPath: Map; }; type ValidationFunc = (configs: AppConfig[]) => ValidationResult; +type ConfigProcessingOptions = { + visibilities?: ConfigVisibility[]; +}; + +export function filterByVisibility( + data: JsonObject, + includeVisibilities: ConfigVisibility[], + visibilityByPath: Map, +): JsonObject { + function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + if (typeof jsonVal !== 'object') { + const visibility = + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + if (includeVisibilities.includes(visibility)) { + return jsonVal; + } + return undefined; + } else if (jsonVal === null) { + return undefined; + } else if (Array.isArray(jsonVal)) { + const arr = new Array(); + + for (const [index, value] of jsonVal.entries()) { + const out = transform(value, `${path}/${index}`); + if (out !== undefined) { + arr.push(out); + } + } + + return arr.length === 0 ? undefined : arr; + } + + const outObj: JsonObject = {}; + + for (const [key, value] of Object.entries(jsonVal)) { + if (value === undefined) { + continue; + } + const out = transform(value, `${path}/${key}`); + if (out !== undefined) { + outObj[key] = out; + } + } + + return Object.keys(outObj).length === 0 ? undefined : outObj; + } + + return (transform(data, '') as JsonObject) ?? {}; +} + export async function loadSchema(options: Options): Promise { const start = process.hrtime(); @@ -60,16 +120,28 @@ export async function loadSchema(options: Options): Promise { const validate = compileConfigSchemas(schemas); return { - load(configs: AppConfig[]): AppConfig[] { + process( + configs: AppConfig[], + { visibilities }: ConfigProcessingOptions = {}, + ): AppConfig[] { const result = validate(configs); - console.log('DEBUG: result =', result); if (result.errors) { throw new Error( `Config validation failed, ${result.errors.join('; ')}`, ); } - return configs; + let processedConfigs = configs; + + if (visibilities) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility(data, visibilities, result.visibilityByPath), + })); + } + console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); + + return processedConfigs; }, }; } @@ -77,7 +149,7 @@ export async function loadSchema(options: Options): Promise { function compileConfigSchemas( schemas: ConfigSchemaPackageEntry[], ): ValidationFunc { - const visibilities = new Map(); + const visibilityByPath = new Map(); const ajv = new Ajv({ strict: true, @@ -99,8 +171,8 @@ function compileConfigSchemas( if (!ctx) { return false; } - if (visibility === 'frontend') { - visibilities.set(ctx.dataPath, visibility); + if (visibility) { + visibilityByPath.set(ctx.dataPath, visibility); } return true; }; @@ -140,18 +212,19 @@ function compileConfigSchemas( return configs => { const config = ConfigReader.fromConfigs(configs).get(); - visibilities.clear(); + visibilityByPath.clear(); + const valid = validate(config); if (!valid) { const errors = ajv.errorsText(validate.errors); return { errors: [errors], - visibilities: new Map(), + visibilityByPath: new Map(), }; } return { - visibilities: new Map(visibilities), + visibilityByPath: new Map(visibilityByPath), }; }; } diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 318eaa7915..2d3c2afd9c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -130,5 +130,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return schema.load(configs); + return schema.process(configs, { visibilities: ['frontend', 'secret'] }); } From 8c65cec932d2818914b0ef124059e832016952a9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:18:58 +0100 Subject: [PATCH 096/259] config-loader: split up config schema implementation and move into separate lib --- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/lib/index.ts | 2 +- packages/config-loader/src/lib/schema.ts | 289 ------------------ .../config-loader/src/lib/schema/collect.ts | 78 +++++ .../config-loader/src/lib/schema/compile.ts | 109 +++++++ .../config-loader/src/lib/schema/filtering.ts | 64 ++++ .../config-loader/src/lib/schema/index.ts | 17 ++ packages/config-loader/src/lib/schema/load.ts | 63 ++++ .../config-loader/src/lib/schema/types.ts | 49 +++ packages/config-loader/src/loader.ts | 6 +- 10 files changed, 384 insertions(+), 295 deletions(-) delete mode 100644 packages/config-loader/src/lib/schema.ts create mode 100644 packages/config-loader/src/lib/schema/collect.ts create mode 100644 packages/config-loader/src/lib/schema/compile.ts create mode 100644 packages/config-loader/src/lib/schema/filtering.ts create mode 100644 packages/config-loader/src/lib/schema/index.ts create mode 100644 packages/config-loader/src/lib/schema/load.ts create mode 100644 packages/config-loader/src/lib/schema/types.ts diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 02db134fe5..415b6d4022 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export { readEnvConfig } from './lib'; +export { readEnvConfig, loadConfigSchema } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 29812053bc..4da0c97788 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,4 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; -export { loadSchema } from './schema'; +export { loadConfigSchema } from './schema'; diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts deleted file mode 100644 index 0d820ab173..0000000000 --- a/packages/config-loader/src/lib/schema.ts +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { resolve as resolvePath, dirname } from 'path'; -import Ajv from 'ajv'; -import { JSONSchema7 as JSONSchema } from 'json-schema'; -import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; -import { - AppConfig, - ConfigReader, - JsonObject, - JsonValue, -} from '@backstage/config'; - -const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; - -type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; - -const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; - -type ConfigSchema = { - process( - appConfigs: AppConfig[], - options?: ConfigProcessingOptions, - ): AppConfig[]; -}; - -type ConfigSchemaPackageEntry = { - value: JSONSchema; - path: string; -}; - -type Options = { - dependencies: string[]; -}; - -type ValidationError = string; - -type ValidationResult = { - errors?: ValidationError[]; - visibilityByPath: Map; -}; - -type ValidationFunc = (configs: AppConfig[]) => ValidationResult; - -type ConfigProcessingOptions = { - visibilities?: ConfigVisibility[]; -}; - -export function filterByVisibility( - data: JsonObject, - includeVisibilities: ConfigVisibility[], - visibilityByPath: Map, -): JsonObject { - function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { - if (typeof jsonVal !== 'object') { - const visibility = - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; - if (includeVisibilities.includes(visibility)) { - return jsonVal; - } - return undefined; - } else if (jsonVal === null) { - return undefined; - } else if (Array.isArray(jsonVal)) { - const arr = new Array(); - - for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}/${index}`); - if (out !== undefined) { - arr.push(out); - } - } - - return arr.length === 0 ? undefined : arr; - } - - const outObj: JsonObject = {}; - - for (const [key, value] of Object.entries(jsonVal)) { - if (value === undefined) { - continue; - } - const out = transform(value, `${path}/${key}`); - if (out !== undefined) { - outObj[key] = out; - } - } - - return Object.keys(outObj).length === 0 ? undefined : outObj; - } - - return (transform(data, '') as JsonObject) ?? {}; -} - -export async function loadSchema(options: Options): Promise { - const start = process.hrtime(); - - const schemas = await collectConfigSchemas(options.dependencies[0]); - - const [durS, durNs] = process.hrtime(start); - const dur = (durS + durNs / 10 ** 9).toFixed(3); - console.log(`DEBUG: collected config schemas in ${dur}s`); - console.log('DEBUG: schemas =', schemas); - - const validate = compileConfigSchemas(schemas); - - return { - process( - configs: AppConfig[], - { visibilities }: ConfigProcessingOptions = {}, - ): AppConfig[] { - const result = validate(configs); - if (result.errors) { - throw new Error( - `Config validation failed, ${result.errors.join('; ')}`, - ); - } - - let processedConfigs = configs; - - if (visibilities) { - processedConfigs = processedConfigs.map(({ data, context }) => ({ - context, - data: filterByVisibility(data, visibilities, result.visibilityByPath), - })); - } - console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); - - return processedConfigs; - }, - }; -} - -function compileConfigSchemas( - schemas: ConfigSchemaPackageEntry[], -): ValidationFunc { - const visibilityByPath = new Map(); - - const ajv = new Ajv({ - strict: true, - allErrors: true, - defaultMeta: 'http://json-schema.org/draft-07/schema#', - schemas: { - 'https://backstage.io/schema/config-v1': true, - }, - keywords: [ - { - keyword: 'visibility', - schemaType: 'string', - metaSchema: { - type: 'string', - enum: CONFIG_VISIBILITIES, - }, - compile(visibility: ConfigVisibility) { - return (_data, ctx) => { - if (!ctx) { - return false; - } - if (visibility) { - visibilityByPath.set(ctx.dataPath, visibility); - } - return true; - }; - }, - }, - ], - }); - - const merged = mergeAllOf( - { allOf: schemas.map(_ => _.value) }, - { - ignoreAdditionalProperties: true, - resolvers: { - visibility(values: string[], path: string[]) { - const hasApp = values.some(_ => _ === 'frontend'); - const hasSecret = values.some(_ => _ === 'secret'); - if (hasApp && hasSecret) { - throw new Error( - `Config schema visibility is both 'frontend' and 'secret' for ${path.join( - '/', - )}`, - ); - } else if (hasApp) { - return 'frontend'; - } else if (hasSecret) { - return 'secret'; - } - - return 'backend'; - }, - } as Partial>, - }, - ); - - const validate = ajv.compile(merged); - - return configs => { - const config = ConfigReader.fromConfigs(configs).get(); - - visibilityByPath.clear(); - - const valid = validate(config); - if (!valid) { - const errors = ajv.errorsText(validate.errors); - return { - errors: [errors], - visibilityByPath: new Map(), - }; - } - - return { - visibilityByPath: new Map(visibilityByPath), - }; - }; -} - -const req = - typeof __non_webpack_require__ === 'undefined' - ? require - : __non_webpack_require__; - -async function collectConfigSchemas( - name: string, - opts: unknown = {}, - visited: Set = new Set(), - schemas: ConfigSchemaPackageEntry[] = [], -): Promise { - const pkgPath = req.resolve(`${name}/package.json`, opts); - if (visited.has(pkgPath)) { - return schemas; - } - visited.add(pkgPath); - const pkg = await fs.readJson(pkgPath); - const depNames = [ - ...Object.keys(pkg.dependencies ?? {}), - ...Object.keys(pkg.peerDependencies ?? {}), - ]; - - // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, - // since that's pretty slow. We probably need a better way to determine when - // we've left the Backstage ecosystem, but this will do for now. - const hasSchema = 'configSchema' in pkg; - const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); - if (!hasSchema && !hasBackstageDep) { - return schemas; - } - if (hasSchema) { - if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { - throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, - ); - } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: pkgPath, - }); - } else { - schemas.push({ - value: pkg.configSchema, - path: pkgPath, - }); - } - } - - for (const depName of depNames) { - await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); - } - - return schemas; -} diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts new file mode 100644 index 0000000000..b6d0bffaef --- /dev/null +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; +import { ConfigSchemaPackageEntry } from './types'; + +const req = + typeof __non_webpack_require__ === 'undefined' + ? require + : __non_webpack_require__; + +export async function collectConfigSchemas( + name: string, + opts: unknown = {}, + visited: Set = new Set(), + schemas: ConfigSchemaPackageEntry[] = [], +): Promise { + const pkgPath = req.resolve(`${name}/package.json`, opts); + if (visited.has(pkgPath)) { + return schemas; + } + visited.add(pkgPath); + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return schemas; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + for (const depName of depNames) { + await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); + } + + return schemas; +} diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts new file mode 100644 index 0000000000..9ab8627fce --- /dev/null +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ajv from 'ajv'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; +import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; +import { ConfigReader } from '@backstage/config'; +import { + ConfigSchemaPackageEntry, + ValidationFunc, + CONFIG_VISIBILITIES, + ConfigVisibility, +} from './types'; + +export function compileConfigSchemas( + schemas: ConfigSchemaPackageEntry[], +): ValidationFunc { + const visibilityByPath = new Map(); + + const ajv = new Ajv({ + strict: true, + allErrors: true, + defaultMeta: 'http://json-schema.org/draft-07/schema#', + schemas: { + 'https://backstage.io/schema/config-v1': true, + }, + keywords: [ + { + keyword: 'visibility', + schemaType: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, ctx) => { + if (!ctx) { + return false; + } + if (visibility) { + visibilityByPath.set(ctx.dataPath, visibility); + } + return true; + }; + }, + }, + ], + }); + + const merged = mergeAllOf( + { allOf: schemas.map(_ => _.value) }, + { + ignoreAdditionalProperties: true, + resolvers: { + visibility(values: string[], path: string[]) { + const hasApp = values.some(_ => _ === 'frontend'); + const hasSecret = values.some(_ => _ === 'secret'); + if (hasApp && hasSecret) { + throw new Error( + `Config schema visibility is both 'frontend' and 'secret' for ${path.join( + '/', + )}`, + ); + } else if (hasApp) { + return 'frontend'; + } else if (hasSecret) { + return 'secret'; + } + + return 'backend'; + }, + } as Partial>, + }, + ); + + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilityByPath.clear(); + + const valid = validate(config); + if (!valid) { + const errors = ajv.errorsText(validate.errors); + return { + errors: [errors], + visibilityByPath: new Map(), + }; + } + + return { + visibilityByPath: new Map(visibilityByPath), + }; + }; +} diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts new file mode 100644 index 0000000000..64ccfa5e24 --- /dev/null +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject, JsonValue } from '@backstage/config'; +import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; + +export function filterByVisibility( + data: JsonObject, + includeVisibilities: ConfigVisibility[], + visibilityByPath: Map, +): JsonObject { + function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + if (typeof jsonVal !== 'object') { + const visibility = + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + if (includeVisibilities.includes(visibility)) { + return jsonVal; + } + return undefined; + } else if (jsonVal === null) { + return undefined; + } else if (Array.isArray(jsonVal)) { + const arr = new Array(); + + for (const [index, value] of jsonVal.entries()) { + const out = transform(value, `${path}/${index}`); + if (out !== undefined) { + arr.push(out); + } + } + + return arr.length === 0 ? undefined : arr; + } + + const outObj: JsonObject = {}; + + for (const [key, value] of Object.entries(jsonVal)) { + if (value === undefined) { + continue; + } + const out = transform(value, `${path}/${key}`); + if (out !== undefined) { + outObj[key] = out; + } + } + + return Object.keys(outObj).length === 0 ? undefined : outObj; + } + + return (transform(data, '') as JsonObject) ?? {}; +} diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts new file mode 100644 index 0000000000..990c0dc182 --- /dev/null +++ b/packages/config-loader/src/lib/schema/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { loadConfigSchema } from './load'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts new file mode 100644 index 0000000000..75fcb86ba7 --- /dev/null +++ b/packages/config-loader/src/lib/schema/load.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { compileConfigSchemas } from './compile'; +import { collectConfigSchemas } from './collect'; +import { filterByVisibility } from './filtering'; +import { ConfigSchema } from './types'; + +type Options = { + dependencies: string[]; +}; + +export async function loadConfigSchema( + options: Options, +): Promise { + const start = process.hrtime(); + + const schemas = await collectConfigSchemas(options.dependencies[0]); + + const [durS, durNs] = process.hrtime(start); + const dur = (durS + durNs / 10 ** 9).toFixed(3); + console.log(`DEBUG: collected config schemas in ${dur}s`); + console.log('DEBUG: schemas =', schemas); + + const validate = compileConfigSchemas(schemas); + + return { + process(configs: AppConfig[], { visibilities } = {}): AppConfig[] { + const result = validate(configs); + if (result.errors) { + throw new Error( + `Config validation failed, ${result.errors.join('; ')}`, + ); + } + + let processedConfigs = configs; + + if (visibilities) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility(data, visibilities, result.visibilityByPath), + })); + } + console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); + + return processedConfigs; + }, + }; +} diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts new file mode 100644 index 0000000000..9e5624ca6a --- /dev/null +++ b/packages/config-loader/src/lib/schema/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; + +export type ConfigSchemaPackageEntry = { + value: JSONSchema; + path: string; +}; + +export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; + +export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; + +export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; + +type ValidationError = string; + +type ValidationResult = { + errors?: ValidationError[]; + visibilityByPath: Map; +}; + +export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; + +type ConfigProcessingOptions = { + visibilities?: ConfigVisibility[]; +}; + +export type ConfigSchema = { + process( + appConfigs: AppConfig[], + options?: ConfigProcessingOptions, + ): AppConfig[]; +}; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2d3c2afd9c..00414f86eb 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname, isAbsolute } from 'path'; import { AppConfig, JsonObject } from '@backstage/config'; -import { readConfigFile, readEnvConfig, readSecret, loadSchema } from './lib'; +import { readConfigFile, readEnvConfig, readSecret } from './lib'; export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -80,8 +80,6 @@ export async function loadConfig( const { configRoot } = options; const configPaths = options.configPaths.slice(); - const schema = await loadSchema({ dependencies: ['example-backend'] }); - // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` if (configPaths.length === 0) { @@ -130,5 +128,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return schema.process(configs, { visibilities: ['frontend', 'secret'] }); + return configs; } From b652ebc4e7e276d963804f3d1a6e5c98c81e0a62 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:39:07 +0100 Subject: [PATCH 097/259] config-loader: add visibility filtering tests + fix filtering logic --- .../src/lib/schema/filtering.test.ts | 105 ++++++++++++++++++ .../config-loader/src/lib/schema/filtering.ts | 23 ++-- 2 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 packages/config-loader/src/lib/schema/filtering.test.ts diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts new file mode 100644 index 0000000000..8cc907417b --- /dev/null +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject } from '@backstage/config'; +import { ConfigVisibility } from './types'; +import { filterByVisibility } from './filtering'; + +const data = { + arr: ['f', 'b', 's'], + objArr: [ + { f: 1, b: 2, s: 3 }, + { f: 4, b: 5, s: 6 }, + ], + obj: { + f: 'a', + b: { + s: true, + }, + }, + arrF: [{ never: 'here' }], + arrB: [{ never: 'here' }], + arrS: [{ never: 'here' }], + objF: { never: 'here' }, + objB: { never: 'here' }, + objS: { never: 'here' }, +}; + +const visibilities = new Map( + Object.entries({ + '/arr/0': 'frontend', + '/arr/1': 'backend', + '/arr/2': 'secret', + '/obj/f': 'frontend', + '/obj/b': 'backend', + '/obj/b/s': 'secret', + '/objArr/0/f': 'frontend', + '/objArr/0/b': 'backend', + '/objArr/0/s': 'secret', + '/objArr/1/f': 'frontend', + '/objArr/1/b': 'backend', + '/objArr/1/s': 'secret', + '/arrF': 'frontend', + '/arrB': 'backend', + '/arrS': 'secret', + '/objF': 'frontend', + '/objB': 'backend', + '/objS': 'secret', + }), +); + +describe('filterByVisibility', () => { + test.each<[ConfigVisibility[], JsonObject]>([ + [[], {}], + [ + ['frontend'], + { + arr: ['f'], + objArr: [{ f: 1 }, { f: 4 }], + obj: { f: 'a' }, + arrF: [], + objF: {}, + }, + ], + [ + ['backend'], + { + arr: ['b'], + objArr: [{ b: 2 }, { b: 5 }], + obj: { b: {} }, + arrF: [{ never: 'here' }], + arrB: [{ never: 'here' }], + arrS: [{ never: 'here' }], + objF: { never: 'here' }, + objB: { never: 'here' }, + objS: { never: 'here' }, + }, + ], + [ + ['secret'], + { + arr: ['s'], + objArr: [{ s: 3 }, { s: 6 }], + obj: { b: { s: true } }, + arrS: [], + objS: {}, + }, + ], + [['frontend', 'backend', 'secret'], data], + ])('should filter correctly with %p', (filter, expected) => { + expect(filterByVisibility(data, filter, visibilities)).toEqual(expected); + }); +}); diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 64ccfa5e24..3ed4c958b8 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -23,13 +23,12 @@ export function filterByVisibility( visibilityByPath: Map, ): JsonObject { function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + const isVisible = includeVisibilities.includes( + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY, + ); + if (typeof jsonVal !== 'object') { - const visibility = - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; - if (includeVisibilities.includes(visibility)) { - return jsonVal; - } - return undefined; + return isVisible ? jsonVal : undefined; } else if (jsonVal === null) { return undefined; } else if (Array.isArray(jsonVal)) { @@ -42,10 +41,14 @@ export function filterByVisibility( } } - return arr.length === 0 ? undefined : arr; + if (arr.length > 0 || isVisible) { + return arr; + } + return undefined; } const outObj: JsonObject = {}; + let hasOutput = false; for (const [key, value] of Object.entries(jsonVal)) { if (value === undefined) { @@ -54,10 +57,14 @@ export function filterByVisibility( const out = transform(value, `${path}/${key}`); if (out !== undefined) { outObj[key] = out; + hasOutput = true; } } - return Object.keys(outObj).length === 0 ? undefined : outObj; + if (hasOutput || isVisible) { + return outObj; + } + return undefined; } return (transform(data, '') as JsonObject) ?? {}; From a533c15b0979745739a86456742f6ff7ba06aa2f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:57:50 +0100 Subject: [PATCH 098/259] config-loader: document schema lib --- .../config-loader/src/lib/schema/collect.ts | 3 ++ .../config-loader/src/lib/schema/compile.ts | 16 +++++++ .../config-loader/src/lib/schema/filtering.ts | 4 ++ packages/config-loader/src/lib/schema/load.ts | 3 ++ .../config-loader/src/lib/schema/types.ts | 45 +++++++++++++++++++ 5 files changed, 71 insertions(+) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index b6d0bffaef..25bdbe7f55 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -23,6 +23,9 @@ const req = ? require : __non_webpack_require__; +/** + * This collects all known config schemas across all dependencies of the app. + */ export async function collectConfigSchemas( name: string, opts: unknown = {}, diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 9ab8627fce..76642690d5 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -25,9 +25,19 @@ import { ConfigVisibility, } from './types'; +/** + * This takes a collection of Backstage configuration schemas from various + * sources and compiles them down into a single schema validation function. + * + * It also handles the implementation of the custom "visibility" keyword used + * to specify the scope of different config paths. + */ export function compileConfigSchemas( schemas: ConfigSchemaPackageEntry[], ): ValidationFunc { + // The ajv instance below is stateful and doesn't really allow for additional + // output during validation. We work around this by having this extra piece + // of state that we reset before each validation. const visibilityByPath = new Map(); const ajv = new Ajv({ @@ -63,8 +73,13 @@ export function compileConfigSchemas( const merged = mergeAllOf( { allOf: schemas.map(_ => _.value) }, { + // JSONSchema is typically subtractive, as in it always reduces the set of allowed + // inputs through constraints. This changes the object property merging to be additive + // rather than subtractive. ignoreAdditionalProperties: true, resolvers: { + // This ensures that the visibilities across different schemas are sound, and + // selects the most specific visibility for each path. visibility(values: string[], path: string[]) { const hasApp = values.some(_ => _ === 'frontend'); const hasSecret = values.some(_ => _ === 'secret'); @@ -95,6 +110,7 @@ export function compileConfigSchemas( const valid = validate(config); if (!valid) { + // TODO(Rugvip): better messages here, with more context such as which file the error occurred in const errors = ajv.errorsText(validate.errors); return { errors: [errors], diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 3ed4c958b8..d6dcaf2dd7 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -17,6 +17,10 @@ import { JsonObject, JsonValue } from '@backstage/config'; import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; +/** + * This filters data by visibility by discovering the visibility of each + * value, and then only keeping the ones that are specified in `includeVisibilities`. + */ export function filterByVisibility( data: JsonObject, includeVisibilities: ConfigVisibility[], diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 75fcb86ba7..286b363b50 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -24,6 +24,9 @@ type Options = { dependencies: string[]; }; +/** + * Loads config schema for a Backstage instance. + */ export async function loadConfigSchema( options: Options, ): Promise { diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 9e5624ca6a..7e831a7a1c 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -17,30 +17,75 @@ import { AppConfig } from '@backstage/config'; import { JSONSchema7 as JSONSchema } from 'json-schema'; +/** + * An sub-set of configuration schema. + */ export type ConfigSchemaPackageEntry = { + /** + * The configuration schema itself, as JSONSchema draft-07 + */ value: JSONSchema; + /** + * The path that the configuration schema was discovered at. + */ path: string; }; +/** + * A list of all possible configuration value visibilities. + */ export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; +/** + * A type representing the possible configuration value visibilities + */ export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; +/** + * The default configuration visibility if no other values is given. + */ export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; +/** + * An explanation of a configuration validation error. + */ type ValidationError = string; +/** + * The result of validating configuration data using a schema. + */ type ValidationResult = { + /** + * Errors that where emitted during validation, if any. + */ errors?: ValidationError[]; + /** + * The configuration visibilities the where discovered during validation. + * + * The path in the key uses the form `////` + */ visibilityByPath: Map; }; +/** + * A function used validate configuration data. + */ export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; +/** + * Options used to process configuration data with a schema. + */ type ConfigProcessingOptions = { + /** + * The visibilities that should be included in the output data. + * If omitted, the data will not be filtered by visibility. + */ visibilities?: ConfigVisibility[]; }; +/** + * A loaded configuration schema that is ready to process configuration data. + */ export type ConfigSchema = { process( appConfigs: AppConfig[], From 4ccba1aa9620af569ec4c424806bafb510982570 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:59:35 +0100 Subject: [PATCH 099/259] config-loader: remove debugging output in schema lib --- packages/config-loader/src/lib/schema/load.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 286b363b50..46e1358ba9 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -30,15 +30,8 @@ type Options = { export async function loadConfigSchema( options: Options, ): Promise { - const start = process.hrtime(); - const schemas = await collectConfigSchemas(options.dependencies[0]); - const [durS, durNs] = process.hrtime(start); - const dur = (durS + durNs / 10 ** 9).toFixed(3); - console.log(`DEBUG: collected config schemas in ${dur}s`); - console.log('DEBUG: schemas =', schemas); - const validate = compileConfigSchemas(schemas); return { @@ -58,7 +51,6 @@ export async function loadConfigSchema( data: filterByVisibility(data, visibilities, result.visibilityByPath), })); } - console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); return processedConfigs; }, From f4af5d1dced13c29bf6fdb07fd024979453920d7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 15:22:28 +0100 Subject: [PATCH 100/259] config-loader: consider all deps when discovering schema and parallelize processing --- .../config-loader/src/lib/schema/collect.ts | 109 +++++++++++------- packages/config-loader/src/lib/schema/load.ts | 2 +- 2 files changed, 66 insertions(+), 45 deletions(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 25bdbe7f55..6e3755926c 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -18,6 +18,11 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname } from 'path'; import { ConfigSchemaPackageEntry } from './types'; +type Item = { + name: string; + parentPath?: string; +}; + const req = typeof __non_webpack_require__ === 'undefined' ? require @@ -27,55 +32,71 @@ const req = * This collects all known config schemas across all dependencies of the app. */ export async function collectConfigSchemas( - name: string, - opts: unknown = {}, - visited: Set = new Set(), - schemas: ConfigSchemaPackageEntry[] = [], + packageNames: string[], ): Promise { - const pkgPath = req.resolve(`${name}/package.json`, opts); - if (visited.has(pkgPath)) { - return schemas; - } - visited.add(pkgPath); - const pkg = await fs.readJson(pkgPath); - const depNames = [ - ...Object.keys(pkg.dependencies ?? {}), - ...Object.keys(pkg.peerDependencies ?? {}), - ]; + const visitedPackages = new Set(); + const schemas = Array(); - // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, - // since that's pretty slow. We probably need a better way to determine when - // we've left the Backstage ecosystem, but this will do for now. - const hasSchema = 'configSchema' in pkg; - const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); - if (!hasSchema && !hasBackstageDep) { - return schemas; - } - if (hasSchema) { - if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { - throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, - ); - } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: pkgPath, - }); - } else { - schemas.push({ - value: pkg.configSchema, - path: pkgPath, - }); + async function process({ name, parentPath }: Item) { + // Ensures that we only process each package once. We don't bother with + // loading in schemas from duplicates of different versions, as that's not + // supported by Backstage right now anyway. We may want to change that in + // the future though, if it for example becomes possible to load in two + // different versions of e.g. @backstage/core at once. + if (visitedPackages.has(name)) { + return; } + visitedPackages.add(name); + + const pkgPath = req.resolve( + `${name}/package.json`, + parentPath && { + paths: [parentPath], + }, + ); + + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + await Promise.all( + depNames.map(name => process({ name, parentPath: pkgPath })), + ); } - for (const depName of depNames) { - await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); - } + await Promise.all(packageNames.map(name => process({ name }))); return schemas; } diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 46e1358ba9..6352da65af 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -30,7 +30,7 @@ type Options = { export async function loadConfigSchema( options: Options, ): Promise { - const schemas = await collectConfigSchemas(options.dependencies[0]); + const schemas = await collectConfigSchemas(options.dependencies); const validate = compileConfigSchemas(schemas); From 7298f8b26576042b05b37ec38fac7e6a8ac1f2b6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 15:30:03 +0100 Subject: [PATCH 101/259] config-loader: remove support for conditionally reading secrets --- packages/backend-common/src/config.ts | 1 - packages/cli/src/lib/config.ts | 6 +- packages/config-loader/src/lib/reader.test.ts | 19 ------ packages/config-loader/src/lib/reader.ts | 4 -- .../config-loader/src/lib/secrets.test.ts | 1 - packages/config-loader/src/lib/types.ts | 1 - packages/config-loader/src/loader.test.ts | 59 ++----------------- packages/config-loader/src/loader.ts | 27 +-------- 8 files changed, 7 insertions(+), 111 deletions(-) diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 69e46bd5b4..86beb65805 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -40,7 +40,6 @@ export async function loadBackendConfig(options: Options): Promise { env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development', configRoot: paths.targetRoot, configPaths: configOpts.map(opt => resolvePath(opt)), - shouldReadSecrets: true, }); options.logger.info( diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 5801632ea5..0ce19c80f5 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -18,14 +18,10 @@ import { loadConfig } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; -export async function loadCliConfig( - configArgs: string[], - shouldReadSecrets: boolean = false, -) { +export async function loadCliConfig(configArgs: string[]) { const configPaths = configArgs.map(arg => paths.resolveTarget(arg)); const appConfigs = await loadConfig({ - shouldReadSecrets, env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production', configRoot: paths.targetRoot, configPaths, diff --git a/packages/config-loader/src/lib/reader.test.ts b/packages/config-loader/src/lib/reader.test.ts index 6ccceef60d..a0a8495714 100644 --- a/packages/config-loader/src/lib/reader.test.ts +++ b/packages/config-loader/src/lib/reader.test.ts @@ -28,7 +28,6 @@ function memoryFiles(files: { [path: string]: string }) { const mockContext: ReaderContext = { env: {}, - skip: () => false, readFile: jest.fn(), readSecret: jest.fn(), }; @@ -179,22 +178,4 @@ describe('readConfigFile', () => { await expect(config).rejects.toThrow('Invalid secret at .app: NOPE'); }); - - it('should omit skipped values', async () => { - const readFile = memoryFiles({ - './app-config.yaml': 'app: { title: skip, name: include }', - }); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - skip: (path: string) => path === '.app.title', - readSecret: jest.fn() as ReadSecretFunc, - }); - - await expect(config).resolves.toEqual({ - context: 'app-config.yaml', - data: { app: { name: 'include' } }, - }); - }); }); diff --git a/packages/config-loader/src/lib/reader.ts b/packages/config-loader/src/lib/reader.ts index 8eadae0fe0..9eba58be97 100644 --- a/packages/config-loader/src/lib/reader.ts +++ b/packages/config-loader/src/lib/reader.ts @@ -37,10 +37,6 @@ export async function readConfigFile( obj: JsonValue, path: string, ): Promise { - if (ctx.skip(path)) { - return undefined; - } - if (typeof obj !== 'object') { return obj; } else if (obj === null) { diff --git a/packages/config-loader/src/lib/secrets.test.ts b/packages/config-loader/src/lib/secrets.test.ts index cfc4150d68..d80ada193b 100644 --- a/packages/config-loader/src/lib/secrets.test.ts +++ b/packages/config-loader/src/lib/secrets.test.ts @@ -21,7 +21,6 @@ const ctx: ReaderContext = { env: { SECRET: 'my-secret', }, - skip: () => false, readSecret: jest.fn(), async readFile(path) { const content = ({ diff --git a/packages/config-loader/src/lib/types.ts b/packages/config-loader/src/lib/types.ts index 02b8a9d053..e189aef20d 100644 --- a/packages/config-loader/src/lib/types.ts +++ b/packages/config-loader/src/lib/types.ts @@ -28,7 +28,6 @@ export type SkipFunc = (path: string) => boolean; */ export type ReaderContext = { env: { [name in string]?: string }; - skip: SkipFunc; readFile: ReadFileFunc; readSecret: ReadSecretFunc; }; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3b0d8e6e92..a9857a784f 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -44,47 +44,6 @@ describe('loadConfig', () => { configRoot: '/root', configPaths: [], env: 'production', - shouldReadSecrets: false, - }), - ).resolves.toEqual([ - { - context: 'app-config.yaml', - data: { - app: { - title: 'Example App', - }, - }, - }, - ]); - }); - - it('loads config without secrets', async () => { - await expect( - loadConfig({ - configRoot: '/root', - configPaths: ['/root/app-config.yaml'], - env: 'production', - shouldReadSecrets: false, - }), - ).resolves.toEqual([ - { - context: 'app-config.yaml', - data: { - app: { - title: 'Example App', - }, - }, - }, - ]); - }); - - it('loads config with secrets', async () => { - await expect( - loadConfig({ - configRoot: '/root', - configPaths: ['/root/app-config.yaml'], - env: 'production', - shouldReadSecrets: true, }), ).resolves.toEqual([ { @@ -99,16 +58,12 @@ describe('loadConfig', () => { ]); }); - it('loads development config without secrets', async () => { + it('loads config with secrets', async () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', - ], - env: 'development', - shouldReadSecrets: false, + configPaths: ['/root/app-config.yaml'], + env: 'production', }), ).resolves.toEqual([ { @@ -116,15 +71,10 @@ describe('loadConfig', () => { data: { app: { title: 'Example App', + sessionKey: 'abc123', }, }, }, - { - context: 'app-config.development.yaml', - data: { - app: {}, - }, - }, ]); }); @@ -137,7 +87,6 @@ describe('loadConfig', () => { '/root/app-config.development.yaml', ], env: 'development', - shouldReadSecrets: true, }), ).resolves.toEqual([ { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 00414f86eb..a647367469 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -28,18 +28,13 @@ export type LoadConfigOptions = { // TODO(Rugvip): This will be removed in the future, but for now we use it to warn about possible mistakes. env: string; - - // Whether to read secrets or omit them, defaults to false. - shouldReadSecrets?: boolean; }; class Context { constructor( private readonly options: { - secretPaths: Set; env: { [name in string]?: string }; rootPath: string; - shouldReadSecrets: boolean; }, ) {} @@ -47,35 +42,21 @@ class Context { return this.options.env; } - skip(path: string): boolean { - if (this.options.shouldReadSecrets) { - return false; - } - return this.options.secretPaths.has(path); - } - async readFile(path: string): Promise { return fs.readFile(resolvePath(this.options.rootPath, path), 'utf8'); } async readSecret( - path: string, + _path: string, desc: JsonObject, ): Promise { - this.options.secretPaths.add(path); - if (!this.options.shouldReadSecrets) { - return undefined; - } - return readSecret(desc, this); } } -type LoadedConfig = AppConfig[]; - export async function loadConfig( options: LoadConfigOptions, -): Promise { +): Promise { const configs = []; const { configRoot } = options; const configPaths = options.configPaths.slice(); @@ -102,8 +83,6 @@ export async function loadConfig( } try { - const secretPaths = new Set(); - for (const configPath of configPaths) { if (!isAbsolute(configPath)) { throw new Error(`Config load path is not absolute: '${configPath}'`); @@ -111,10 +90,8 @@ export async function loadConfig( const config = await readConfigFile( configPath, new Context({ - secretPaths, env: process.env, rootPath: dirname(configPath), - shouldReadSecrets: Boolean(options.shouldReadSecrets), }), ); From 564f438edd4a4c016e2ac73ff132c420ae371a69 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 16:10:39 +0100 Subject: [PATCH 102/259] cli: load config schema and use to filter config + add --frontend flag to print --- packages/cli/package.json | 1 + packages/cli/src/commands/config/print.ts | 35 ++++++++++++++++++++--- packages/cli/src/commands/index.ts | 1 + packages/cli/src/lib/bundler/bundle.ts | 2 +- packages/cli/src/lib/bundler/config.ts | 10 +++---- packages/cli/src/lib/bundler/server.ts | 2 +- packages/cli/src/lib/bundler/types.ts | 12 ++++---- packages/cli/src/lib/config.ts | 20 +++++++++++-- 8 files changed, 64 insertions(+), 19 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 317e78ca16..66c56dd8cc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -76,6 +76,7 @@ "jest": "^26.0.1", "jest-css-modules": "^2.1.0", "jest-esm-transformer": "^1.0.0", + "lodash": "^4.17.19", "mini-css-extract-plugin": "^0.9.0", "ora": "^4.0.3", "raw-loader": "^4.0.1", diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index eb814a5bee..ce0d9976b1 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -16,16 +16,43 @@ import { Command } from 'commander'; import { stringify as stringifyYaml } from 'yaml'; +import { ConfigReader } from '@backstage/config'; import { loadCliConfig } from '../../lib/config'; +import cloneDeepWith from 'lodash/cloneDeepWith'; export default async (cmd: Command) => { - const { config } = await loadCliConfig(cmd.config, cmd.withSecrets ?? false); + const { schema, appConfigs } = await loadCliConfig(cmd.config); + let data; - const flatConfig = config.get(); + if (cmd.frontend) { + const frontendConfigs = schema.process(appConfigs, { + visibilities: ['frontend'], + }); + data = ConfigReader.fromConfigs(frontendConfigs).get(); + } else if (cmd.withSecrets) { + data = ConfigReader.fromConfigs(appConfigs).get(); + } else { + let secretConfigs = schema.process(appConfigs, { + visibilities: ['secret'], + }); + + secretConfigs = secretConfigs.map(entry => ({ + context: entry.context, + data: cloneDeepWith(entry.data, value => { + console.log('DEBUG: value =', value); + if (typeof value !== 'object') { + return ''; + } + return undefined; + }), + })); + + data = ConfigReader.fromConfigs(appConfigs.concat(secretConfigs)).get(); + } if (cmd.format === 'json') { - process.stdout.write(`${JSON.stringify(flatConfig, null, 2)}\n`); + process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); } else { - process.stdout.write(`${stringifyYaml(flatConfig)}\n`); + process.stdout.write(`${stringifyYaml(data)}\n`); } }; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 5b0dae6244..bcbde20d84 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -136,6 +136,7 @@ export function registerCommands(program: CommanderStatic) { program .command('config:print') + .option('--frontend', 'Print only the frontend configuration') .option('--with-secrets', 'Include secrets in the printed configuration') .option( '--format ', diff --git a/packages/cli/src/lib/bundler/bundle.ts b/packages/cli/src/lib/bundler/bundle.ts index 8746c807e7..a5d581c60a 100644 --- a/packages/cli/src/lib/bundler/bundle.ts +++ b/packages/cli/src/lib/bundler/bundle.ts @@ -40,7 +40,7 @@ export async function buildBundle(options: BuildOptions) { ...options, checksEnabled: false, isDev: false, - baseUrl: resolveBaseUrl(options.config), + baseUrl: resolveBaseUrl(options.frontendConfig), }); const compiler = webpack(config); diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 34b9f498ad..9e150e0ccb 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -74,11 +74,11 @@ export async function createConfig( paths: BundlingPaths, options: BundlingOptions, ): Promise { - const { checksEnabled, isDev } = options; + const { checksEnabled, isDev, frontendConfig } = options; const { plugins, loaders } = transforms(options); - const baseUrl = options.config.getString('app.baseUrl'); + const baseUrl = frontendConfig.getString('app.baseUrl'); const validBaseUrl = new URL(baseUrl); if (checksEnabled) { @@ -99,7 +99,7 @@ export async function createConfig( plugins.push( new webpack.EnvironmentPlugin({ - APP_CONFIG: options.appConfigs, + APP_CONFIG: options.frontendAppConfigs, }), ); @@ -109,9 +109,9 @@ export async function createConfig( templateParameters: { publicPath: validBaseUrl.pathname.replace(/\/$/, ''), app: { - title: options.config.getString('app.title'), + title: frontendConfig.getString('app.title'), baseUrl: validBaseUrl.href, - googleAnalyticsTrackingId: options.config.getOptionalString( + googleAnalyticsTrackingId: frontendConfig.getOptionalString( 'app.googleAnalyticsTrackingId', ), }, diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index 2710145d83..f540e01084 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -23,7 +23,7 @@ import { ServeOptions } from './types'; import { resolveBundlingPaths } from './paths'; export async function serveBundle(options: ServeOptions) { - const url = resolveBaseUrl(options.config); + const url = resolveBaseUrl(options.frontendConfig); const host = options.config.getOptionalString('app.listen.host') || url.hostname; diff --git a/packages/cli/src/lib/bundler/types.ts b/packages/cli/src/lib/bundler/types.ts index 00f1895725..d0af0a9937 100644 --- a/packages/cli/src/lib/bundler/types.ts +++ b/packages/cli/src/lib/bundler/types.ts @@ -21,23 +21,23 @@ import { ParallelOption } from '../parallel'; export type BundlingOptions = { checksEnabled: boolean; isDev: boolean; - config: Config; - appConfigs: AppConfig[]; + frontendConfig: Config; + frontendAppConfigs: AppConfig[]; baseUrl: URL; parallel?: ParallelOption; }; export type ServeOptions = BundlingPathsOptions & { checksEnabled: boolean; - config: Config; - appConfigs: AppConfig[]; + frontendConfig: Config; + frontendAppConfigs: AppConfig[]; }; export type BuildOptions = BundlingPathsOptions & { statsJsonEnabled: boolean; parallel?: ParallelOption; - config: Config; - appConfigs: AppConfig[]; + frontendConfig: Config; + frontendAppConfigs: AppConfig[]; }; export type BackendBundlingOptions = { diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 0ce19c80f5..a282bbf88c 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,13 +14,22 @@ * limitations under the License. */ -import { loadConfig } from '@backstage/config-loader'; +import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; export async function loadCliConfig(configArgs: string[]) { const configPaths = configArgs.map(arg => paths.resolveTarget(arg)); + // Consider all packages in the monorepo when loading in config + const LernaProject = require('@lerna/project'); + const project = new LernaProject(paths.targetDir); + const packages = await project.getPackages(); + const localPackageNames = packages.map((p: any) => p.name); + const schema = await loadConfigSchema({ + dependencies: localPackageNames, + }); + const appConfigs = await loadConfig({ env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production', configRoot: paths.targetRoot, @@ -31,8 +40,15 @@ export async function loadCliConfig(configArgs: string[]) { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); + const frontendAppConfigs = schema.process(appConfigs, { + visibilities: ['frontend'], + }); + const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); + return { + schema, appConfigs, - config: ConfigReader.fromConfigs(appConfigs), + frontendConfig, + frontendAppConfigs, }; } From 3849e24112fefdea9a9a28d56176472b7c80a4e6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 00:19:30 +0100 Subject: [PATCH 103/259] config-loader: add support for serializing and deserializing schema --- packages/config-loader/src/index.ts | 1 + packages/config-loader/src/lib/index.ts | 2 +- .../config-loader/src/lib/schema/collect.ts | 17 ++++++---- .../config-loader/src/lib/schema/index.ts | 1 + packages/config-loader/src/lib/schema/load.ts | 34 +++++++++++++++---- .../config-loader/src/lib/schema/types.ts | 7 ++-- 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 415b6d4022..b08380d8ce 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,5 +15,6 @@ */ export { readEnvConfig, loadConfigSchema } from './lib'; +export type { ConfigSchema } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 4da0c97788..ceb7c34222 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,4 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; -export { loadConfigSchema } from './schema'; +export * from './schema'; diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 6e3755926c..f7fed94748 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -15,7 +15,11 @@ */ import fs from 'fs-extra'; -import { resolve as resolvePath, dirname } from 'path'; +import { + resolve as resolvePath, + relative as relativePath, + dirname, +} from 'path'; import { ConfigSchemaPackageEntry } from './types'; type Item = { @@ -36,8 +40,9 @@ export async function collectConfigSchemas( ): Promise { const visitedPackages = new Set(); const schemas = Array(); + const currentDir = process.cwd(); - async function process({ name, parentPath }: Item) { + async function processItem({ name, parentPath }: Item) { // Ensures that we only process each package once. We don't bother with // loading in schemas from duplicates of different versions, as that's not // supported by Backstage right now anyway. We may want to change that in @@ -81,22 +86,22 @@ export async function collectConfigSchemas( ); schemas.push({ value, - path: pkgPath, + path: relativePath(currentDir, pkgPath), }); } else { schemas.push({ value: pkg.configSchema, - path: pkgPath, + path: relativePath(currentDir, pkgPath), }); } } await Promise.all( - depNames.map(name => process({ name, parentPath: pkgPath })), + depNames.map(name => processItem({ name, parentPath: pkgPath })), ); } - await Promise.all(packageNames.map(name => process({ name }))); + await Promise.all(packageNames.map(name => processItem({ name }))); return schemas; } diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index 990c0dc182..bc3c7d8ebc 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -15,3 +15,4 @@ */ export { loadConfigSchema } from './load'; +export type { ConfigSchema } from './types'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 6352da65af..cdfc8f9549 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -14,15 +14,19 @@ * limitations under the License. */ -import { AppConfig } from '@backstage/config'; +import { AppConfig, JsonObject } from '@backstage/config'; import { compileConfigSchemas } from './compile'; import { collectConfigSchemas } from './collect'; import { filterByVisibility } from './filtering'; -import { ConfigSchema } from './types'; +import { ConfigSchema, ConfigSchemaPackageEntry } from './types'; -type Options = { - dependencies: string[]; -}; +type Options = + | { + dependencies: string[]; + } + | { + serialized: JsonObject; + }; /** * Loads config schema for a Backstage instance. @@ -30,7 +34,19 @@ type Options = { export async function loadConfigSchema( options: Options, ): Promise { - const schemas = await collectConfigSchemas(options.dependencies); + let schemas: ConfigSchemaPackageEntry[]; + + if ('dependencies' in options) { + schemas = await collectConfigSchemas(options.dependencies); + } else { + const { serialized } = options; + if (serialized?.backstageConfigSchemaVersion !== 1) { + throw new Error( + 'Serialized configuration schema is invalid or has an invalid version number', + ); + } + schemas = serialized.schemas as ConfigSchemaPackageEntry[]; + } const validate = compileConfigSchemas(schemas); @@ -54,5 +70,11 @@ export async function loadConfigSchema( return processedConfigs; }, + serialize(): JsonObject { + return { + schemas, + backstageConfigSchemaVersion: 1, + }; + }, }; } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 7e831a7a1c..cec2da0160 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -14,8 +14,7 @@ * limitations under the License. */ -import { AppConfig } from '@backstage/config'; -import { JSONSchema7 as JSONSchema } from 'json-schema'; +import { AppConfig, JsonObject } from '@backstage/config'; /** * An sub-set of configuration schema. @@ -24,7 +23,7 @@ export type ConfigSchemaPackageEntry = { /** * The configuration schema itself, as JSONSchema draft-07 */ - value: JSONSchema; + value: JsonObject; /** * The path that the configuration schema was discovered at. */ @@ -91,4 +90,6 @@ export type ConfigSchema = { appConfigs: AppConfig[], options?: ConfigProcessingOptions, ): AppConfig[]; + + serialize(): JsonObject; }; From e43aa0dc60012aa11c68a8dfd3d40c27381bdfcd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 00:27:24 +0100 Subject: [PATCH 104/259] cli: write serialized config schema to app output --- packages/cli/src/lib/bundler/bundle.ts | 10 +++++++++- packages/cli/src/lib/bundler/types.ts | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/lib/bundler/bundle.ts b/packages/cli/src/lib/bundler/bundle.ts index a5d581c60a..30fef2e8b7 100644 --- a/packages/cli/src/lib/bundler/bundle.ts +++ b/packages/cli/src/lib/bundler/bundle.ts @@ -33,7 +33,7 @@ const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024; const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024; export async function buildBundle(options: BuildOptions) { - const { statsJsonEnabled } = options; + const { statsJsonEnabled, schema: configSchema } = options; const paths = resolveBundlingPaths(options); const config = await createConfig(paths, { @@ -56,6 +56,14 @@ export async function buildBundle(options: BuildOptions) { }); } + if (configSchema) { + await fs.writeJson( + resolvePath(paths.targetDist, '.config-schema.json'), + configSchema.serialize(), + { spaces: 2 }, + ); + } + const { stats } = await build(compiler, isCi).catch(error => { console.log(chalk.red('Failed to compile.\n')); throw new Error(`Failed to compile.\n${error.message || error}`); diff --git a/packages/cli/src/lib/bundler/types.ts b/packages/cli/src/lib/bundler/types.ts index d0af0a9937..71343a1761 100644 --- a/packages/cli/src/lib/bundler/types.ts +++ b/packages/cli/src/lib/bundler/types.ts @@ -17,6 +17,7 @@ import { AppConfig, Config } from '@backstage/config'; import { BundlingPathsOptions } from './paths'; import { ParallelOption } from '../parallel'; +import { ConfigSchema } from '@backstage/config-loader'; export type BundlingOptions = { checksEnabled: boolean; @@ -36,6 +37,7 @@ export type ServeOptions = BundlingPathsOptions & { export type BuildOptions = BundlingPathsOptions & { statsJsonEnabled: boolean; parallel?: ParallelOption; + schema?: ConfigSchema; frontendConfig: Config; frontendAppConfigs: AppConfig[]; }; From 2e4de993383b72be6ee93bdf6ce60aa30070ce7a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 10:11:50 +0100 Subject: [PATCH 105/259] config-loader: switch to ajv v6 --- packages/config-loader/package.json | 2 +- .../config-loader/src/lib/schema/compile.ts | 40 ++++++++----------- .../src/lib/schema/filtering.test.ts | 36 ++++++++--------- .../config-loader/src/lib/schema/filtering.ts | 4 +- yarn.lock | 14 ------- 5 files changed, 38 insertions(+), 58 deletions(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 39735ff2fd..32776d71b7 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -34,7 +34,7 @@ "@backstage/config": "^0.1.1", "@types/json-schema": "^7.0.6", "@types/json-schema-merge-allof": "^0.6.0", - "ajv": "^7.0.0-beta.2", + "ajv": "^6.12.5", "fs-extra": "^9.0.0", "json-schema": "^0.2.5", "json-schema-merge-allof": "^0.7.0", diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 76642690d5..f63dc3e636 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -41,33 +41,27 @@ export function compileConfigSchemas( const visibilityByPath = new Map(); const ajv = new Ajv({ - strict: true, allErrors: true, - defaultMeta: 'http://json-schema.org/draft-07/schema#', schemas: { 'https://backstage.io/schema/config-v1': true, }, - keywords: [ - { - keyword: 'visibility', - schemaType: 'string', - metaSchema: { - type: 'string', - enum: CONFIG_VISIBILITIES, - }, - compile(visibility: ConfigVisibility) { - return (_data, ctx) => { - if (!ctx) { - return false; - } - if (visibility) { - visibilityByPath.set(ctx.dataPath, visibility); - } - return true; - }; - }, - }, - ], + }).addKeyword('visibility', { + type: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, dataPath) => { + if (!dataPath) { + return false; + } + if (visibility) { + visibilityByPath.set(dataPath, visibility); + } + return true; + }; + }, }); const merged = mergeAllOf( diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 8cc907417b..191edf4824 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -40,24 +40,24 @@ const data = { const visibilities = new Map( Object.entries({ - '/arr/0': 'frontend', - '/arr/1': 'backend', - '/arr/2': 'secret', - '/obj/f': 'frontend', - '/obj/b': 'backend', - '/obj/b/s': 'secret', - '/objArr/0/f': 'frontend', - '/objArr/0/b': 'backend', - '/objArr/0/s': 'secret', - '/objArr/1/f': 'frontend', - '/objArr/1/b': 'backend', - '/objArr/1/s': 'secret', - '/arrF': 'frontend', - '/arrB': 'backend', - '/arrS': 'secret', - '/objF': 'frontend', - '/objB': 'backend', - '/objS': 'secret', + '.arr[0]': 'frontend', + '.arr[1]': 'backend', + '.arr[2]': 'secret', + '.obj.f': 'frontend', + '.obj.b': 'backend', + '.obj.b.s': 'secret', + '.objArr[0].f': 'frontend', + '.objArr[0].b': 'backend', + '.objArr[0].s': 'secret', + '.objArr[1].f': 'frontend', + '.objArr[1].b': 'backend', + '.objArr[1].s': 'secret', + '.arrF': 'frontend', + '.arrB': 'backend', + '.arrS': 'secret', + '.objF': 'frontend', + '.objB': 'backend', + '.objS': 'secret', }), ); diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index d6dcaf2dd7..908e9e4739 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -39,7 +39,7 @@ export function filterByVisibility( const arr = new Array(); for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}/${index}`); + const out = transform(value, `${path}[${index}]`); if (out !== undefined) { arr.push(out); } @@ -58,7 +58,7 @@ export function filterByVisibility( if (value === undefined) { continue; } - const out = transform(value, `${path}/${key}`); + const out = transform(value, `${path}.${key}`); if (out !== undefined) { outObj[key] = out; hasOutput = true; diff --git a/yarn.lock b/yarn.lock index 98e1b2476f..a85e24210f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6569,15 +6569,6 @@ ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.1, ajv@^6.10.2, ajv@^6.12.4, ajv@^6.12.5, ajv json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^7.0.0-beta.2: - version "7.0.0-beta.3" - resolved "https://registry.npmjs.org/ajv/-/ajv-7.0.0-beta.3.tgz#d34861ccfbdebb55bf9f49a08b29f76bf656fc5c" - integrity sha512-gUGVvM4NmyqrFvCNAQnP4P7FC0RjxMQyRnrXpozNglBkDJnTysVbvycyOZUy5n6yLKSqVDUqWZBXj7dXINrSqw== - dependencies: - fast-deep-equal "^3.1.1" - json-schema-traverse "^0.5.0" - uri-js "^4.2.2" - alphanum-sort@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3" @@ -15052,11 +15043,6 @@ json-schema-traverse@^0.4.1: resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== -json-schema-traverse@^0.5.0: - version "0.5.0" - resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.5.0.tgz#1069a6097f9e9a567bfc6cc215e85f606ebf6480" - integrity sha512-x+TRJIQFskrNnFKE2Viz9FCSjK1vIh+H/uaBiOYszh/IcZmAFneQ35H4osWDJp1NPXccuV2I0RMXmi2ZS6Kqcg== - json-schema@0.2.3: version "0.2.3" resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" From 7f95e40fbb8e43cd75a9e61f62ac6fda728037a6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 10:28:11 +0100 Subject: [PATCH 106/259] app-backend,cli: add support for injecting full config based on serialized schema --- packages/backend/src/plugins/app.ts | 6 ++- packages/core/src/api-wrappers/createApp.tsx | 6 ++- plugins/app-backend/package.json | 1 + plugins/app-backend/src/lib/config.test.ts | 26 +++++------ plugins/app-backend/src/lib/config.ts | 46 +++++++++++++------ .../app-backend/src/service/router.test.ts | 4 +- plugins/app-backend/src/service/router.ts | 25 ++++++---- .../src/service/standaloneServer.ts | 5 +- 8 files changed, 79 insertions(+), 40 deletions(-) diff --git a/packages/backend/src/plugins/app.ts b/packages/backend/src/plugins/app.ts index c9f7c0622a..637af80974 100644 --- a/packages/backend/src/plugins/app.ts +++ b/packages/backend/src/plugins/app.ts @@ -17,9 +17,13 @@ import { createRouter } from '@backstage/plugin-app-backend'; import { PluginEnvironment } from '../types'; -export default async function createPlugin({ logger }: PluginEnvironment) { +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { return await createRouter({ logger, + config, appPackageName: 'example-app', }); } diff --git a/packages/core/src/api-wrappers/createApp.tsx b/packages/core/src/api-wrappers/createApp.tsx index 592a2417ca..4a1e58db42 100644 --- a/packages/core/src/api-wrappers/createApp.tsx +++ b/packages/core/src/api-wrappers/createApp.tsx @@ -61,7 +61,11 @@ export const defaultConfigLoader: AppConfigLoader = async ( if (runtimeConfigJson !== '__app_injected_runtime_config__'.toUpperCase()) { try { const data = JSON.parse(runtimeConfigJson) as JsonObject; - configs.push({ data, context: 'env' }); + if (Array.isArray(data)) { + configs.push(...data); + } else { + configs.push({ data, context: 'env' }); + } } catch (error) { throw new Error(`Failed to load runtime configuration, ${error}`); } diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index c9b82cf6f1..08266acb68 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -22,6 +22,7 @@ "dependencies": { "@backstage/backend-common": "^0.2.0", "@backstage/config-loader": "^0.2.0", + "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", diff --git a/plugins/app-backend/src/lib/config.test.ts b/plugins/app-backend/src/lib/config.test.ts index 16c5600b1d..475f72f3b1 100644 --- a/plugins/app-backend/src/lib/config.test.ts +++ b/plugins/app-backend/src/lib/config.test.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'path'; import { getVoidLogger } from '@backstage/backend-common'; -import { injectEnvConfig } from './config'; +import { injectConfig } from './config'; jest.mock('fs-extra'); @@ -29,12 +29,12 @@ const readFileMock = (fsMock.readFile as unknown) as jest.MockedFunction< const MOCK_DIR = 'mock-dir'; const baseOptions = { - env: {}, + appConfigs: [], staticDir: MOCK_DIR, logger: getVoidLogger(), }; -describe('injectEnvConfig', () => { +describe('injectConfig', () => { beforeEach(() => { fsMock.readdir.mockResolvedValue(['main.js']); }); @@ -44,7 +44,7 @@ describe('injectEnvConfig', () => { }); it('should not inject without config', async () => { - await injectEnvConfig(baseOptions); + await injectConfig(baseOptions); expect(fsMock.readdir).toHaveBeenCalledTimes(0); expect(fsMock.readFile).toHaveBeenCalledTimes(0); expect(fsMock.writeFile).toHaveBeenCalledTimes(0); @@ -64,7 +64,10 @@ describe('injectEnvConfig', () => { return 'NO_PLACEHOLDER_HERE'; }); - await injectEnvConfig({ ...baseOptions, env: { APP_CONFIG_x: '0' } }); + await injectConfig({ + ...baseOptions, + appConfigs: [{ data: { x: 0 }, context: 'test' }], + }); expect(fsMock.readFile).toHaveBeenCalledTimes(2); expect(fsMock.readFile).toHaveBeenNthCalledWith( 1, @@ -96,11 +99,9 @@ describe('injectEnvConfig', () => { 'JSON.parse("__APP_INJECTED_RUNTIME_CONFIG__")', ); - await injectEnvConfig({ + await injectConfig({ ...baseOptions, - env: { - APP_CONFIG_x: '0', - }, + appConfigs: [{ data: { x: 0 }, context: 'test' }], }); expect(fsMock.writeFile).toHaveBeenCalledTimes(1); @@ -115,12 +116,9 @@ describe('injectEnvConfig', () => { readFileMock.mockResolvedValue(fsMock.writeFile.mock.calls[0][1]); - await injectEnvConfig({ + await injectConfig({ ...baseOptions, - env: { - APP_CONFIG_x: '1', - APP_CONFIG_y: '2', - }, + appConfigs: [{ data: { x: 1, y: 2 }, context: 'test' }], }); expect(fsMock.writeFile).toHaveBeenCalledTimes(2); diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 0ed56ef722..60b5d28806 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -16,12 +16,12 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'path'; -import { readEnvConfig } from '@backstage/config-loader'; import { Logger } from 'winston'; +import { AppConfig, Config, JsonObject } from '@backstage/config'; +import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader'; -type Options = { - // Environment to read config from - env: { [name: string]: string | undefined }; +type InjectOptions = { + appConfigs: AppConfig[]; // Directory of the static JS files to search for file to inject staticDir: string; logger: Logger; @@ -31,19 +31,13 @@ type Options = { * Injects config from APP_CONFIG_ env vars, replacing existing * injected config if it has already been injected. */ -export async function injectEnvConfig(options: Options) { - const { env, staticDir, logger } = options; - - const envConfig = readEnvConfig(env); - if (envConfig.length === 0) { - return; - } +export async function injectConfig(options: InjectOptions) { + const { staticDir, logger, appConfigs } = options; const files = await fs.readdir(staticDir); const jsFiles = files.filter(file => file.endsWith('.js')); - const [{ data }] = envConfig; - const escapedData = JSON.stringify(data).replace(/("|'|\\)/g, '\\$1'); + const escapedData = JSON.stringify(appConfigs).replace(/("|'|\\)/g, '\\$1'); const injected = `/*__APP_INJECTED_CONFIG_MARKER__*/"${escapedData}"/*__INJECTED_END__*/`; for (const jsFile of jsFiles) { @@ -72,3 +66,29 @@ export async function injectEnvConfig(options: Options) { } logger.info('Env config not injected'); } + +type ReadOptions = { + env: { [name: string]: string | undefined }; + appDistDir: string; + config: Config; +}; + +export async function readConfigs(options: ReadOptions): Promise { + const { env, appDistDir, config } = options; + + const appConfigs = readEnvConfig(env); + + const schemaPath = resolvePath(appDistDir, '.config-schema.json'); + if (await fs.pathExists(schemaPath)) { + const serializedSchema = await fs.readJson(schemaPath); + const schema = await loadConfigSchema({ serialized: serializedSchema }); + + const frontendConfigs = await schema.process( + [{ data: config.get() as JsonObject, context: 'app' }], + { visibilities: ['frontend'] }, + ); + appConfigs.push(...frontendConfigs); + } + + return appConfigs; +} diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts index aed4280ca5..b0cc3c59f7 100644 --- a/plugins/app-backend/src/service/router.test.ts +++ b/plugins/app-backend/src/service/router.test.ts @@ -16,10 +16,10 @@ import { resolve as resolvePath } from 'path'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; import request from 'supertest'; - import { createRouter } from './router'; jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() })); @@ -35,6 +35,7 @@ describe('createRouter', () => { beforeAll(async () => { const router = await createRouter({ logger: getVoidLogger(), + config: new ConfigReader({}), appPackageName: 'example-app', }); app = express().use(router); @@ -76,6 +77,7 @@ describe('createRouter with static fallback handler', () => { const router = await createRouter({ logger: getVoidLogger(), + config: new ConfigReader({}), appPackageName: 'example-app', staticFallbackHandler, }); diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index c9a06e4dda..b435ec15f1 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -15,13 +15,15 @@ */ import { resolve as resolvePath } from 'path'; -import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { injectEnvConfig } from '../lib/config'; +import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; +import { injectConfig, readConfigs } from '../lib/config'; export interface RouterOptions { + config: Config; logger: Logger; appPackageName: string; staticFallbackHandler?: express.Handler; @@ -30,22 +32,27 @@ export interface RouterOptions { export async function createRouter( options: RouterOptions, ): Promise { - const appDistDir = resolvePackagePath(options.appPackageName, 'dist'); - options.logger.info(`Serving static app content from ${appDistDir}`); + const { config, logger, appPackageName, staticFallbackHandler } = options; - await injectEnvConfig({ + const appDistDir = resolvePackagePath(appPackageName, 'dist'); + logger.info(`Serving static app content from ${appDistDir}`); + const staticDir = resolvePath(appDistDir, 'static'); + + const appConfigs = await readConfigs({ + config, + appDistDir, env: process.env, - logger: options.logger, - staticDir: resolvePath(appDistDir, 'static'), }); + await injectConfig({ appConfigs, logger, staticDir }); + const router = Router(); // Use a separate router for static content so that a fallback can be provided by backend const staticRouter = Router(); staticRouter.use(express.static(resolvePath(appDistDir, 'static'))); - if (options.staticFallbackHandler) { - staticRouter.use(options.staticFallbackHandler); + if (staticFallbackHandler) { + staticRouter.use(staticFallbackHandler); } staticRouter.use(notFoundHandler()); diff --git a/plugins/app-backend/src/service/standaloneServer.ts b/plugins/app-backend/src/service/standaloneServer.ts index 8abf3b81f2..005d80027b 100644 --- a/plugins/app-backend/src/service/standaloneServer.ts +++ b/plugins/app-backend/src/service/standaloneServer.ts @@ -14,14 +14,16 @@ * limitations under the License. */ -import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; +import { createServiceBuilder } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; import { createRouter } from './router'; export interface ServerOptions { port: number; enableCors: boolean; + config: Config; logger: Logger; } @@ -32,6 +34,7 @@ export async function startStandaloneServer( logger.debug('Starting application server...'); const router = await createRouter({ logger, + config: options.config, appPackageName: 'example-app', }); From eff7b6e3ba05901a44cc99f836535669f222b857 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 10:32:25 +0100 Subject: [PATCH 107/259] cli: remove debug print --- packages/cli/src/commands/config/print.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index ce0d9976b1..66b709e601 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -35,11 +35,9 @@ export default async (cmd: Command) => { let secretConfigs = schema.process(appConfigs, { visibilities: ['secret'], }); - secretConfigs = secretConfigs.map(entry => ({ context: entry.context, data: cloneDeepWith(entry.data, value => { - console.log('DEBUG: value =', value); if (typeof value !== 'object') { return ''; } From c18769cc00dd85017886cd03ca22a58f3ddbb9b6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 14:21:53 +0100 Subject: [PATCH 108/259] config-loader: visibilities -> visiblity --- packages/cli/src/commands/config/print.ts | 4 ++-- packages/cli/src/lib/config.ts | 2 +- packages/config-loader/src/lib/schema/filtering.test.ts | 4 ++-- packages/config-loader/src/lib/schema/load.ts | 6 +++--- packages/config-loader/src/lib/schema/types.ts | 2 +- plugins/app-backend/src/lib/config.ts | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index 66b709e601..246847b610 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -26,14 +26,14 @@ export default async (cmd: Command) => { if (cmd.frontend) { const frontendConfigs = schema.process(appConfigs, { - visibilities: ['frontend'], + visiblity: ['frontend'], }); data = ConfigReader.fromConfigs(frontendConfigs).get(); } else if (cmd.withSecrets) { data = ConfigReader.fromConfigs(appConfigs).get(); } else { let secretConfigs = schema.process(appConfigs, { - visibilities: ['secret'], + visiblity: ['secret'], }); secretConfigs = secretConfigs.map(entry => ({ context: entry.context, diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index a282bbf88c..63e3958a6f 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -41,7 +41,7 @@ export async function loadCliConfig(configArgs: string[]) { ); const frontendAppConfigs = schema.process(appConfigs, { - visibilities: ['frontend'], + visiblity: ['frontend'], }); const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 191edf4824..8e3fb61ae5 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -38,7 +38,7 @@ const data = { objS: { never: 'here' }, }; -const visibilities = new Map( +const visiblity = new Map( Object.entries({ '.arr[0]': 'frontend', '.arr[1]': 'backend', @@ -100,6 +100,6 @@ describe('filterByVisibility', () => { ], [['frontend', 'backend', 'secret'], data], ])('should filter correctly with %p', (filter, expected) => { - expect(filterByVisibility(data, filter, visibilities)).toEqual(expected); + expect(filterByVisibility(data, filter, visiblity)).toEqual(expected); }); }); diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index cdfc8f9549..d886926647 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -51,7 +51,7 @@ export async function loadConfigSchema( const validate = compileConfigSchemas(schemas); return { - process(configs: AppConfig[], { visibilities } = {}): AppConfig[] { + process(configs: AppConfig[], { visiblity } = {}): AppConfig[] { const result = validate(configs); if (result.errors) { throw new Error( @@ -61,10 +61,10 @@ export async function loadConfigSchema( let processedConfigs = configs; - if (visibilities) { + if (visiblity) { processedConfigs = processedConfigs.map(({ data, context }) => ({ context, - data: filterByVisibility(data, visibilities, result.visibilityByPath), + data: filterByVisibility(data, visiblity, result.visibilityByPath), })); } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index cec2da0160..6ab0602fb3 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -79,7 +79,7 @@ type ConfigProcessingOptions = { * The visibilities that should be included in the output data. * If omitted, the data will not be filtered by visibility. */ - visibilities?: ConfigVisibility[]; + visiblity?: ConfigVisibility[]; }; /** diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 60b5d28806..82b431c33f 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -85,7 +85,7 @@ export async function readConfigs(options: ReadOptions): Promise { const frontendConfigs = await schema.process( [{ data: config.get() as JsonObject, context: 'app' }], - { visibilities: ['frontend'] }, + { visiblity: ['frontend'] }, ); appConfigs.push(...frontendConfigs); } From 051e0fd17e7b58cbfc8a0f548f8b891139f5859e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 14:28:48 +0100 Subject: [PATCH 109/259] config-loader: hasApp -> hasFrontend --- packages/config-loader/src/lib/schema/compile.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index f63dc3e636..7dc65be910 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -75,15 +75,15 @@ export function compileConfigSchemas( // This ensures that the visibilities across different schemas are sound, and // selects the most specific visibility for each path. visibility(values: string[], path: string[]) { - const hasApp = values.some(_ => _ === 'frontend'); + const hasFrontend = values.some(_ => _ === 'frontend'); const hasSecret = values.some(_ => _ === 'secret'); - if (hasApp && hasSecret) { + if (hasFrontend && hasSecret) { throw new Error( `Config schema visibility is both 'frontend' and 'secret' for ${path.join( '/', )}`, ); - } else if (hasApp) { + } else if (hasFrontend) { return 'frontend'; } else if (hasSecret) { return 'secret'; From 49ab961b8bb659336aead47f786740a9959b6fb6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 15:48:02 +0100 Subject: [PATCH 110/259] config-loader: add support for transforming values during schema processing --- packages/config-loader/src/index.ts | 2 +- .../config-loader/src/lib/schema/filtering.ts | 20 +++++++++---- .../config-loader/src/lib/schema/index.ts | 2 +- packages/config-loader/src/lib/schema/load.ts | 28 +++++++++++++++++-- .../config-loader/src/lib/schema/types.ts | 16 +++++++++++ 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index b08380d8ce..9ad54c5f18 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,6 +15,6 @@ */ export { readEnvConfig, loadConfigSchema } from './lib'; -export type { ConfigSchema } from './lib'; +export type { ConfigSchema, ConfigVisibility } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 908e9e4739..74e75f465d 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -15,7 +15,11 @@ */ import { JsonObject, JsonValue } from '@backstage/config'; -import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; +import { + ConfigVisibility, + DEFAULT_CONFIG_VISIBILITY, + TransformFunc, +} from './types'; /** * This filters data by visibility by discovering the visibility of each @@ -25,14 +29,20 @@ export function filterByVisibility( data: JsonObject, includeVisibilities: ConfigVisibility[], visibilityByPath: Map, + transformFunc?: TransformFunc, ): JsonObject { function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { - const isVisible = includeVisibilities.includes( - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY, - ); + const visibility = visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + const isVisible = includeVisibilities.includes(visibility); if (typeof jsonVal !== 'object') { - return isVisible ? jsonVal : undefined; + if (isVisible) { + if (transformFunc) { + return transformFunc(jsonVal, { visibility }); + } + return jsonVal; + } + return undefined; } else if (jsonVal === null) { return undefined; } else if (Array.isArray(jsonVal)) { diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index bc3c7d8ebc..8cefb93b3c 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -15,4 +15,4 @@ */ export { loadConfigSchema } from './load'; -export type { ConfigSchema } from './types'; +export type { ConfigSchema, ConfigVisibility } from './types'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index d886926647..010317849c 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -18,7 +18,11 @@ import { AppConfig, JsonObject } from '@backstage/config'; import { compileConfigSchemas } from './compile'; import { collectConfigSchemas } from './collect'; import { filterByVisibility } from './filtering'; -import { ConfigSchema, ConfigSchemaPackageEntry } from './types'; +import { + ConfigSchema, + ConfigSchemaPackageEntry, + CONFIG_VISIBILITIES, +} from './types'; type Options = | { @@ -51,7 +55,10 @@ export async function loadConfigSchema( const validate = compileConfigSchemas(schemas); return { - process(configs: AppConfig[], { visiblity } = {}): AppConfig[] { + process( + configs: AppConfig[], + { visiblity, valueTransform } = {}, + ): AppConfig[] { const result = validate(configs); if (result.errors) { throw new Error( @@ -64,7 +71,22 @@ export async function loadConfigSchema( if (visiblity) { processedConfigs = processedConfigs.map(({ data, context }) => ({ context, - data: filterByVisibility(data, visiblity, result.visibilityByPath), + data: filterByVisibility( + data, + visiblity, + result.visibilityByPath, + valueTransform, + ), + })); + } else if (valueTransform) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility( + data, + Array.from(CONFIG_VISIBILITIES), + result.visibilityByPath, + valueTransform, + ), })); } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 6ab0602fb3..c82036bbf4 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -71,6 +71,14 @@ type ValidationResult = { */ export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; +/** + * A function used to transform primitive configuration values. + */ +export type TransformFunc = ( + value: T, + context: { visibility: ConfigVisibility }, +) => T | undefined; + /** * Options used to process configuration data with a schema. */ @@ -80,6 +88,14 @@ type ConfigProcessingOptions = { * If omitted, the data will not be filtered by visibility. */ visiblity?: ConfigVisibility[]; + + /** + * A transform function that can be used to transform primitive configuration values + * during validation. The value returned from the transform function will be used + * instead of the original value. If the transform returns `undefined`, the value + * will be omitted. + */ + valueTransform?: TransformFunc; }; /** From b6e59e6368ccc62c713a6d253b331e3c603ad737 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 15:50:41 +0100 Subject: [PATCH 111/259] cli: refactor and fix config:print to properly filter secrets --- packages/cli/src/commands/config/print.ts | 65 +++++++++++++---------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index 246847b610..9649c70ef3 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -16,37 +16,14 @@ import { Command } from 'commander'; import { stringify as stringifyYaml } from 'yaml'; -import { ConfigReader } from '@backstage/config'; +import { AppConfig, ConfigReader } from '@backstage/config'; import { loadCliConfig } from '../../lib/config'; -import cloneDeepWith from 'lodash/cloneDeepWith'; +import { ConfigSchema, ConfigVisibility } from '@backstage/config-loader'; export default async (cmd: Command) => { const { schema, appConfigs } = await loadCliConfig(cmd.config); - let data; - - if (cmd.frontend) { - const frontendConfigs = schema.process(appConfigs, { - visiblity: ['frontend'], - }); - data = ConfigReader.fromConfigs(frontendConfigs).get(); - } else if (cmd.withSecrets) { - data = ConfigReader.fromConfigs(appConfigs).get(); - } else { - let secretConfigs = schema.process(appConfigs, { - visiblity: ['secret'], - }); - secretConfigs = secretConfigs.map(entry => ({ - context: entry.context, - data: cloneDeepWith(entry.data, value => { - if (typeof value !== 'object') { - return ''; - } - return undefined; - }), - })); - - data = ConfigReader.fromConfigs(appConfigs.concat(secretConfigs)).get(); - } + const visibility = getVisiblityOption(cmd); + const data = serializeConfigData(appConfigs, schema, visibility); if (cmd.format === 'json') { process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); @@ -54,3 +31,37 @@ export default async (cmd: Command) => { process.stdout.write(`${stringifyYaml(data)}\n`); } }; + +function getVisiblityOption(cmd: Command): ConfigVisibility { + if (cmd.frontend && cmd.withSecrets) { + throw new Error('Not allowed to combine frontend and secret config'); + } + if (cmd.frontend) { + return 'frontend'; + } else if (cmd.withSecrets) { + return 'secret'; + } + return 'backend'; +} + +function serializeConfigData( + appConfigs: AppConfig[], + schema: ConfigSchema, + visiblity: ConfigVisibility, +) { + if (visiblity === 'frontend') { + const frontendConfigs = schema.process(appConfigs, { + visiblity: ['frontend'], + }); + return ConfigReader.fromConfigs(frontendConfigs).get(); + } else if (visiblity === 'secret') { + return ConfigReader.fromConfigs(appConfigs).get(); + } + + const sanitizedConfigs = schema.process(appConfigs, { + valueTransform: (value, { visibility }) => + visibility === 'secret' ? '' : value, + }); + + return ConfigReader.fromConfigs(sanitizedConfigs).get(); +} From eb9e293fcef38a29cb75020f0afaa92e02581c84 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 18:03:10 +0100 Subject: [PATCH 112/259] config-loader: skip schema collection from packages that don't export package.json --- .../config-loader/src/lib/schema/collect.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index f7fed94748..6f441a2735 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -53,12 +53,19 @@ export async function collectConfigSchemas( } visitedPackages.add(name); - const pkgPath = req.resolve( - `${name}/package.json`, - parentPath && { - paths: [parentPath], - }, - ); + let pkgPath: string; + try { + pkgPath = req.resolve( + `${name}/package.json`, + parentPath && { + paths: [parentPath], + }, + ); + } catch { + // We can somewhat safely ignore packages that don't export package.json, + // as they are likely not part of the Backstage ecosystem anyway. + return; + } const pkg = await fs.readJson(pkgPath); const depNames = [ From d02d79e39276c1c61e695d3b3995685d0e643d9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 13:07:13 +0100 Subject: [PATCH 113/259] config-loader,cli: better presentation of config schema validation errors --- packages/cli/src/lib/config.ts | 29 ++++++++++++------- .../config-loader/src/lib/schema/compile.ts | 10 +++++-- packages/config-loader/src/lib/schema/load.ts | 4 ++- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 63e3958a6f..08a8193818 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -40,15 +40,24 @@ export async function loadCliConfig(configArgs: string[]) { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); - const frontendAppConfigs = schema.process(appConfigs, { - visiblity: ['frontend'], - }); - const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); + try { + const frontendAppConfigs = schema.process(appConfigs, { + visiblity: ['frontend'], + }); + const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); - return { - schema, - appConfigs, - frontendConfig, - frontendAppConfigs, - }; + return { + schema, + appConfigs, + frontendConfig, + frontendAppConfigs, + }; + } catch (error) { + const maybeSchemaError = error as Error & { messages?: string[] }; + if (maybeSchemaError.messages) { + const messages = maybeSchemaError.messages.join('\n '); + throw new Error(`Configuration does not match schema\n\n ${messages}`); + } + throw error; + } } diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 7dc65be910..6ee24f8d21 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -104,10 +104,14 @@ export function compileConfigSchemas( const valid = validate(config); if (!valid) { - // TODO(Rugvip): better messages here, with more context such as which file the error occurred in - const errors = ajv.errorsText(validate.errors); + const errors = validate.errors ?? []; return { - errors: [errors], + errors: errors.map(({ dataPath, message, params }) => { + const paramStr = Object.entries(params) + .map(([name, value]) => `${name}=${value}`) + .join(' '); + return `Config ${message || ''} { ${paramStr} } at ${dataPath}`; + }), visibilityByPath: new Map(), }; } diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 010317849c..01a9499983 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -61,9 +61,11 @@ export async function loadConfigSchema( ): AppConfig[] { const result = validate(configs); if (result.errors) { - throw new Error( + const error = new Error( `Config validation failed, ${result.errors.join('; ')}`, ); + (error as any).messages = result.errors; + throw error; } let processedConfigs = configs; From 28a49c2ffa49a2316c97a738f7e13da2c13477e3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:09:07 +0100 Subject: [PATCH 114/259] config-loader: make visibility schema keyword work for all types --- packages/config-loader/src/lib/schema/compile.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 6ee24f8d21..5009dccce7 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -46,7 +46,6 @@ export function compileConfigSchemas( 'https://backstage.io/schema/config-v1': true, }, }).addKeyword('visibility', { - type: 'string', metaSchema: { type: 'string', enum: CONFIG_VISIBILITIES, From 06205426d8b285709c068b1fad98e62cd2f172f2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:13:42 +0100 Subject: [PATCH 115/259] config-loader: add support for defining config schema using .d.ts files --- packages/config-loader/package.json | 1 + .../config-loader/src/lib/schema/collect.ts | 72 ++++++++++++++++--- yarn.lock | 21 ++++-- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 32776d71b7..9abd3c754e 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -38,6 +38,7 @@ "fs-extra": "^9.0.0", "json-schema": "^0.2.5", "json-schema-merge-allof": "^0.7.0", + "typescript-json-schema": "^0.43.0", "yaml": "^1.9.2", "yup": "^0.29.3" }, diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 6f441a2735..dd0776213c 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -21,6 +21,8 @@ import { dirname, } from 'path'; import { ConfigSchemaPackageEntry } from './types'; +import { getProgramFromFiles, generateSchema } from 'typescript-json-schema'; +import { JsonObject } from '@backstage/config'; type Item = { name: string; @@ -40,6 +42,7 @@ export async function collectConfigSchemas( ): Promise { const visitedPackages = new Set(); const schemas = Array(); + const tsSchemaPaths = Array(); const currentDir = process.cwd(); async function processItem({ name, parentPath }: Item) { @@ -83,18 +86,23 @@ export async function collectConfigSchemas( } if (hasSchema) { if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { + const isJson = pkg.configSchema.endsWith('.json'); + const isDts = pkg.configSchema.endsWith('.d.ts'); + if (!isJson && !isDts) { throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, + `Config schema files must be .json or .d.ts, got ${pkg.configSchema}`, ); } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: relativePath(currentDir, pkgPath), - }); + if (isDts) { + tsSchemaPaths.push(resolvePath(dirname(pkgPath), pkg.configSchema)); + } else { + const path = resolvePath(dirname(pkgPath), pkg.configSchema); + const value = await fs.readJson(path); + schemas.push({ + value, + path: relativePath(currentDir, path), + }); + } } else { schemas.push({ value: pkg.configSchema, @@ -110,5 +118,49 @@ export async function collectConfigSchemas( await Promise.all(packageNames.map(name => processItem({ name }))); - return schemas; + const tsSchemas = compileTsSchemas(currentDir, tsSchemaPaths); + + return schemas.concat(tsSchemas); +} + +// This handles the support of TypeScript .d.ts config schema declarations. +// We collect all typescript schema definition and compile them all in one go. +// This is much faster than compiling them separately. +function compileTsSchemas(currentDir: string, paths: string[]) { + if (paths.length === 0) { + return []; + } + + const program = getProgramFromFiles(paths, { + incremental: false, + isolatedModules: true, + lib: ['ES5'], // Skipping most libs speeds processing up a lot, we just need the primitive types anyway + noEmit: true, + noResolve: true, + skipLibCheck: true, // Skipping lib checks speeds things up + skipDefaultLibCheck: true, + strict: true, + typeRoots: [], // Do not include any additional types + types: [], + }); + + const tsSchemas = paths.map(schemaFile => { + const value = generateSchema( + program, + // All schemas should export a `Config` symbol + 'Config', + // This enables usage of @visibility is doc comments + { validationKeywords: ['visibility'] }, + [schemaFile], + ) as JsonObject | null; + + const path = relativePath(currentDir, schemaFile); + + if (!value) { + throw new Error(`Invalid schema in ${path}, missing Config export`); + } + return { path, value }; + }); + + return tsSchemas; } diff --git a/yarn.lock b/yarn.lock index a85e24210f..104aa4a174 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12478,7 +12478,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: +glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@~7.1.6: version "7.1.6" resolved "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== @@ -23239,10 +23239,21 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^4.0.3: - version "4.0.3" - resolved "https://registry.npmjs.org/typescript/-/typescript-4.0.3.tgz#153bbd468ef07725c1df9c77e8b453f8d36abba5" - integrity sha512-tEu6DGxGgRJPb/mVPIZ48e69xCn2yRmCgYmDugAVwmJ6o+0u1RI18eO7E7WBTLYLaEVVOhwQmcdhQHweux/WPg== +typescript-json-schema@^0.43.0: + version "0.43.0" + resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.43.0.tgz#8bd9c832f1f15f006ff933907ce192222fdfd92f" + integrity sha512-4c9IMlIlHYJiQtzL1gh2nIPJEjBgJjDUs50gsnnc+GFyDSK1oFM3uQIBSVosiuA/4t6LSAXDS9vTdqbQC6EcgA== + dependencies: + "@types/json-schema" "^7.0.5" + glob "~7.1.6" + json-stable-stringify "^1.0.1" + typescript "~4.0.2" + yargs "^15.4.1" + +typescript@^4.0.3, typescript@~4.0.2: + version "4.0.5" + resolved "https://registry.npmjs.org/typescript/-/typescript-4.0.5.tgz#ae9dddfd1069f1cb5beb3ef3b2170dd7c1332389" + integrity sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ== ua-parser-js@^0.7.18: version "0.7.21" From 567ce7804a79ce669b5767d0f954e74fdf9d7f2d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:47:04 +0100 Subject: [PATCH 116/259] backend-common: port config schema to d.ts and fill out --- packages/backend-common/config-schema.json | 80 ----------- packages/backend-common/config.d.ts | 148 +++++++++++++++++++++ packages/backend-common/package.json | 5 +- 3 files changed, 151 insertions(+), 82 deletions(-) delete mode 100644 packages/backend-common/config-schema.json create mode 100644 packages/backend-common/config.d.ts diff --git a/packages/backend-common/config-schema.json b/packages/backend-common/config-schema.json deleted file mode 100644 index 2012447a11..0000000000 --- a/packages/backend-common/config-schema.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/backend-common", - "type": "object", - "properties": { - "integrations": { - "type": "object", - "additionalProperties": false, - "properties": { - "github": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "token": { - "type": "string", - "visibility": "secret" - } - } - } - }, - "gitlab": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "token": { - "type": "string", - "visibility": "secret" - } - } - } - }, - "bitbucket": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "username": { - "type": "string" - }, - "appPassword": { - "type": "string", - "visibility": "secret" - } - } - } - }, - "azure": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "token": { - "type": "string", - "visibility": "secret" - } - } - } - } - } - } - } -} diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts new file mode 100644 index 0000000000..64e883aa26 --- /dev/null +++ b/packages/backend-common/config.d.ts @@ -0,0 +1,148 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + frontend: { + baseUrl: string; // defined in core, but repeated here without doc + }; + + backend: { + baseUrl: string; // defined in core, but repeated here without doc + /** Address that the backend should listen to. */ + + listen: + | string + | { + /** Address of the interface that the backend should bind to. */ + address?: string; + /** Port that the backend should listen to. */ + port?: number; + }; + + /** HTTPS configuration for the backend. If omitted the backend will serve HTTP */ + https?: { + /** Certificate configuration or parameters for generating a self-signed certificate */ + certificate?: + | { + /** Algorithm to use to generate a self-signed certificate */ + algorithm: string; + keySize?: number; + days?: number; + } + | { + /** PEM encoded certificate. Use $file to load in a file */ + cert: string; + /** + * PEM encoded certificate key. Use $file to load in a file. + * @visibility secret + */ + key: string; + }; + }; + + /** Database connection configuration, select database type using the `client` field */ + database: + | { + client: 'sqlite3'; + connection: ':memory:' | string; + } + | { + client: 'pg'; + /** + * PostgreSQL connection string or knex configuration object. + * @secret + */ + connection: string | object; + }; + + cors?: { + origin?: string | string[]; + methods?: string | string[]; + allowedHeaders?: string | string[]; + exposedHeaders?: string | string[]; + credentials?: boolean; + maxAge?: number; + preflightContinue?: boolean; + optionsSuccessStatus?: number; + }; + + /** */ + csp?: object; + }; + + /** Configuration for integrations towards various external repository provider systems */ + integrations: { + /** Integration configuration for Azure */ + azure: { + /** The hostname of the given Azure instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + }[]; + + /** Integration configuration for BitBucket */ + bitbucket: { + /** The hostname of the given Bitbucket instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + /** The base url for the BitBucket API, for example https://api.bitbucket.org/2.0 */ + apiBaseUrl?: string; + /** + * The username to use for authenticated requests. + * @visibility secret + */ + username?: string; + /** + * BitBucket app password used to authenticate requests. + * @visibility secret + */ + appPassword?: string; + }[]; + + /** Integration configuration for GitHub */ + github: { + /** The hostname of the given GitHub instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + /** The base url for the GitHub API, for example https://api.github.com */ + apiBaseUrl?: string; + /** The base url for GitHub raw resources, for example https://raw.githubusercontent.com */ + rawBaseUrl?: string; + }[]; + + /** Integration configuration for GitLab */ + gitlab: { + /** The hostname of the given GitLab instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + }[]; + }; +} diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 2f58bd0536..737e9664bf 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -89,7 +89,8 @@ "supertest": "^4.0.2" }, "files": [ - "dist" + "dist", + "config.d.ts" ], - "configSchema": "config-schema.json" + "configSchema": "config.d.ts" } From 8782781f16f87a7bbbc2093b44af70cabef49887 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:53:00 +0100 Subject: [PATCH 117/259] core: ported config schema to d.t.s --- packages/core/config.d.ts | 65 ++++++++++++++++++++++++++++++++++++++ packages/core/package.json | 53 ++----------------------------- 2 files changed, 68 insertions(+), 50 deletions(-) create mode 100644 packages/core/config.d.ts diff --git a/packages/core/config.d.ts b/packages/core/config.d.ts new file mode 100644 index 0000000000..014e4ac934 --- /dev/null +++ b/packages/core/config.d.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** + * Generic frontend configuration. + */ + app: { + /** + * The public absolute root URL that the frontend. + * @visibility frontend + */ + baseUrl: string; + + /** + * The title of the app. + * @visibility frontend + */ + title?: string; + }; + + /** + * Generic backend configuration. + */ + backend: { + /** + * The public absolute root URL that the backend is reachable at. + * @visibility frontend + */ + baseUrl: string; + }; + + /** + * Configuration that provides information about the organization that the app is for. + */ + organization?: { + /** + * The name of the organization that the app belongs to. + * @visibility frontend + */ + name?: string; + }; + + homepage?: { + clocks?: { + /** @visibility frontend */ + label: string; + /** @visibility frontend */ + timezone: string; + }[]; + }; +} diff --git a/packages/core/package.json b/packages/core/package.json index 3d4b53d35b..d0b623ea83 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -79,55 +79,8 @@ "@types/zen-observable": "^0.8.0" }, "files": [ - "dist" + "dist", + "config.d.ts" ], - "configSchema": { - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/core", - "type": "object", - "properties": { - "app": { - "type": "object", - "properties": { - "title": { - "type": "string", - "visibility": "frontend" - }, - "baseUrl": { - "type": "string", - "visibility": "frontend" - } - }, - "required": [ - "title", - "baseUrl" - ] - }, - "backend": { - "type": "object", - "properties": { - "baseUrl": { - "type": "string", - "visibility": "frontend" - } - }, - "required": [ - "baseUrl" - ] - }, - "organization": { - "type": "object", - "properties": { - "name": { - "type": "string", - "visibility": "frontend" - } - } - } - }, - "required": [ - "app", - "backend" - ] - } + "configSchema": "config.d.ts" } From cb1634e818847e629002f5b5cb91b2029264491a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:11:47 +0100 Subject: [PATCH 118/259] cli: added config schema --- packages/cli/package.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/cli/package.json b/packages/cli/package.json index 66c56dd8cc..23c5c5d6af 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -147,5 +147,31 @@ "watch": "./src", "exec": "bin/backstage-cli", "ext": "ts" + }, + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/cli", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "visibility": "frontend" + }, + "title": { + "type": "string", + "visibility": "frontend" + }, + "googleAnalyticsTrackingId": { + "type": "string", + "visibility": "frontend", + "description": "Tracking ID for Google Analytics", + "example": "UA-000000-0" + } + } + } + } } } From f6ed7d5d0d966c14bb48895e094741abde9be7f3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:29:30 +0100 Subject: [PATCH 119/259] config-loader: normalize data paths to ensure consistent behavior --- .../config-loader/src/lib/schema/compile.ts | 6 +++++- .../src/lib/schema/filtering.test.ts | 18 +++++++++--------- .../config-loader/src/lib/schema/filtering.ts | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 5009dccce7..93ab8e978f 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -56,7 +56,11 @@ export function compileConfigSchemas( return false; } if (visibility) { - visibilityByPath.set(dataPath, visibility); + const normalizedPath = dataPath.replace( + /\['?(.*?)'?\]/g, + (_, segment) => `.${segment}`, + ); + visibilityByPath.set(normalizedPath, visibility); } return true; }; diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 8e3fb61ae5..bfad2ca95a 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -40,18 +40,18 @@ const data = { const visiblity = new Map( Object.entries({ - '.arr[0]': 'frontend', - '.arr[1]': 'backend', - '.arr[2]': 'secret', + '.arr.0': 'frontend', + '.arr.1': 'backend', + '.arr.2': 'secret', '.obj.f': 'frontend', '.obj.b': 'backend', '.obj.b.s': 'secret', - '.objArr[0].f': 'frontend', - '.objArr[0].b': 'backend', - '.objArr[0].s': 'secret', - '.objArr[1].f': 'frontend', - '.objArr[1].b': 'backend', - '.objArr[1].s': 'secret', + '.objArr.0.f': 'frontend', + '.objArr.0.b': 'backend', + '.objArr.0.s': 'secret', + '.objArr.1.f': 'frontend', + '.objArr.1.b': 'backend', + '.objArr.1.s': 'secret', '.arrF': 'frontend', '.arrB': 'backend', '.arrS': 'secret', diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 74e75f465d..10a97f9a7f 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -49,7 +49,7 @@ export function filterByVisibility( const arr = new Array(); for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}[${index}]`); + const out = transform(value, `${path}.${index}`); if (out !== undefined) { arr.push(out); } From feb9905a377a5d441c7b7568010b618b8b5e5a37 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:30:49 +0100 Subject: [PATCH 120/259] plugins: add config schema needed by frontend plugins --- plugins/auth-backend/package.json | 35 +------------------- plugins/cost-insights/config.d.ts | 52 ++++++++++++++++++++++++++++++ plugins/cost-insights/package.json | 6 ++-- plugins/lighthouse/package.json | 18 ++++++++++- plugins/rollbar/package.json | 18 ++++++++++- plugins/sentry/package.json | 18 ++++++++++- plugins/techdocs/package.json | 18 ++++++++++- plugins/user-settings/package.json | 21 +++++++++++- 8 files changed, 145 insertions(+), 41 deletions(-) create mode 100644 plugins/cost-insights/config.d.ts diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 2bc00f56a2..af3d056b28 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -66,38 +66,5 @@ "files": [ "dist", "migrations" - ], - "configSchema": { - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/plugin-auth-backend", - "type": "object", - "properties": { - "auth": { - "type": "object", - "properties": { - "providers": { - "type": "object", - "properties": { - "google": { - "type": "object", - "visibility": "frontend", - "additionalProperties": { - "type": "object", - "properties": { - "clientId": { - "type": "string" - }, - "clientSecret": { - "type": "string", - "visibility": "secret" - } - } - } - } - } - } - } - } - } - } + ] } diff --git a/plugins/cost-insights/config.d.ts b/plugins/cost-insights/config.d.ts new file mode 100644 index 0000000000..0f147c2cfe --- /dev/null +++ b/plugins/cost-insights/config.d.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface Config { + costInsights: { + /** + * @visibility frontend + */ + engineerCost: number; + + products: { + [kind: string]: { + /** + * @visibility frontend + */ + name: string; + + /** + * @visibility frontend + */ + icon: 'compute' | 'data' | 'database' | 'storage' | 'search' | 'ml'; + }; + }; + + metrics?: { + [kind: string]: { + /** + * @visibility frontend + */ + name: string; + + /** + * @visibility frontend + */ + default?: boolean; + }; + }; + }; +} diff --git a/plugins/cost-insights/package.json b/plugins/cost-insights/package.json index 39159c03c4..1d7850482c 100644 --- a/plugins/cost-insights/package.json +++ b/plugins/cost-insights/package.json @@ -61,6 +61,8 @@ "msw": "^0.21.2" }, "files": [ - "dist" - ] + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 9389c84856..53bfc0e78b 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -52,5 +52,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/lighthouse", + "type": "object", + "properties": { + "lighthouse": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/rollbar/package.json b/plugins/rollbar/package.json index 722fe35fb6..18e4de3024 100644 --- a/plugins/rollbar/package.json +++ b/plugins/rollbar/package.json @@ -52,5 +52,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/rollbar", + "type": "object", + "properties": { + "rollbar": { + "type": "object", + "properties": { + "organization": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index 7941df5dfd..2757c02db9 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -49,5 +49,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/sentry", + "type": "object", + "properties": { + "sentry": { + "type": "object", + "properties": { + "organization": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 88a38549ad..a513b7aa8d 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -53,5 +53,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/techdocs", + "type": "object", + "properties": { + "techdocs": { + "type": "object", + "properties": { + "requestUrl": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index a48c99cbc6..f82a6b2b6f 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -45,5 +45,24 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/user-settings", + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "providers": { + "type": "object", + "additionalProperties": { + "type": "object", + "visibility": "frontend" + } + } + } + } + } + } } From b8eba8219ede8797aa0a75214be0b8f8b3182788 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:40:50 +0100 Subject: [PATCH 121/259] config-loader: enable validation of required props in d.ts definitions + fix issues --- app-config.yaml | 2 +- packages/backend-common/config.d.ts | 2 +- packages/config-loader/src/lib/schema/collect.ts | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 3d0d093536..a331d72d7c 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -270,7 +270,7 @@ costInsights: homepage: clocks: - label: UTC - timzone: UTC + timezone: UTC - label: NYC timezone: 'America/New_York' - label: STO diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 64e883aa26..b6beff12f7 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -15,7 +15,7 @@ */ export interface Config { - frontend: { + app: { baseUrl: string; // defined in core, but repeated here without doc }; diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index dd0776213c..11a7ce4ec2 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -150,7 +150,10 @@ function compileTsSchemas(currentDir: string, paths: string[]) { // All schemas should export a `Config` symbol 'Config', // This enables usage of @visibility is doc comments - { validationKeywords: ['visibility'] }, + { + required: true, + validationKeywords: ['visibility'], + }, [schemaFile], ) as JsonObject | null; From 13c999c513d1ff4a32841b36f94077473ba4d175 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:41:32 +0100 Subject: [PATCH 122/259] cli: added config:validate --- packages/cli/src/commands/config/validate.ts | 22 ++++++++++++++++++++ packages/cli/src/commands/index.ts | 8 +++++++ 2 files changed, 30 insertions(+) create mode 100644 packages/cli/src/commands/config/validate.ts diff --git a/packages/cli/src/commands/config/validate.ts b/packages/cli/src/commands/config/validate.ts new file mode 100644 index 0000000000..229f9e07e9 --- /dev/null +++ b/packages/cli/src/commands/config/validate.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Command } from 'commander'; +import { loadCliConfig } from '../../lib/config'; + +export default async (cmd: Command) => { + await loadCliConfig(cmd.config); +}; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index bcbde20d84..304b5bae8f 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -146,6 +146,14 @@ export function registerCommands(program: CommanderStatic) { .description('Print the app configuration for the current package') .action(lazy(() => import('./config/print').then(m => m.default))); + program + .command('config:validate') + .option(...configOption) + .description( + 'Validate that the given configuration loads and matches schema', + ) + .action(lazy(() => import('./config/validate').then(m => m.default))); + program .command('prepack') .description('Prepares a package for packaging before publishing') From acb2f2c32e7adafefe0cfb66a983f6984c22936c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:42:17 +0100 Subject: [PATCH 123/259] github/workflows: validate config in CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75c6a13cd2..00ed9ce714 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,9 @@ jobs: - name: prettier run: yarn prettier:check + - name: validate config + run: yarn backstage-cli config:validate + - name: lint run: yarn lerna -- run lint --since origin/master From 1722cb53c281001e5aa96f631b14a6c0582b4c4f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:56:40 +0100 Subject: [PATCH 124/259] changesets: added config schema changesets --- .changeset/add-config-schema-declarations.md | 13 ++++++++++ .changeset/add-config-schema-support.md | 27 ++++++++++++++++++++ .changeset/app-backend-config.md | 7 +++++ 3 files changed, 47 insertions(+) create mode 100644 .changeset/add-config-schema-declarations.md create mode 100644 .changeset/add-config-schema-support.md create mode 100644 .changeset/app-backend-config.md diff --git a/.changeset/add-config-schema-declarations.md b/.changeset/add-config-schema-declarations.md new file mode 100644 index 0000000000..2f52a2ad3f --- /dev/null +++ b/.changeset/add-config-schema-declarations.md @@ -0,0 +1,13 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/core': patch +'@backstage/plugin-cost-insights': patch +'@backstage/plugin-lighthouse': patch +'@backstage/plugin-rollbar': patch +'@backstage/plugin-sentry': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-user-settings': patch +--- + +Added configuration schema diff --git a/.changeset/add-config-schema-support.md b/.changeset/add-config-schema-support.md new file mode 100644 index 0000000000..fd20426b30 --- /dev/null +++ b/.changeset/add-config-schema-support.md @@ -0,0 +1,27 @@ +--- +'@backstage/backend-common': minor +'@backstage/cli': minor +'@backstage/config-loader': minor +--- + +Added support for loading and validating configuration schema, as well as declaring config visibility through schema. + +The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. + +Configuration schema is declared using the following JSONSchema meta schema, which is based on draft07: https://backstage.io/schema/config-v1. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. + +Packages that wish to contribute configuration schema should declare it in a root "configSchema" field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be declared in either `.json` or `.d.ts`. + +TypeScript configuration schema files should export a single `Config` type, for example: + +```ts +export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; +} +``` diff --git a/.changeset/app-backend-config.md b/.changeset/app-backend-config.md new file mode 100644 index 0000000000..9df47ef1b8 --- /dev/null +++ b/.changeset/app-backend-config.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-app-backend': minor +--- + +Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection. + +This removes the confusing behavior where configuration was only injected into the app att build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. From 12f601f830da5d49396b916a0f9baf49d5a2919c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:58:12 +0100 Subject: [PATCH 125/259] config-loader: fix types dependencies --- packages/config-loader/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 9abd3c754e..98583cf833 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -32,8 +32,6 @@ "dependencies": { "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", - "@types/json-schema": "^7.0.6", - "@types/json-schema-merge-allof": "^0.6.0", "ajv": "^6.12.5", "fs-extra": "^9.0.0", "json-schema": "^0.2.5", @@ -44,6 +42,8 @@ }, "devDependencies": { "@types/jest": "^26.0.7", + "@types/json-schema": "^7.0.6", + "@types/json-schema-merge-allof": "^0.6.0", "@types/mock-fs": "^4.10.0", "@types/node": "^12.0.0", "@types/yup": "^0.29.8", From 9f6f39ce5f4c4688e0dd0ab7fd90a09f1dbb870d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 23:02:59 +0100 Subject: [PATCH 126/259] backend-common: integration configs aren't required --- packages/backend-common/config.d.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index b6beff12f7..9e8fb7ee35 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -84,9 +84,9 @@ export interface Config { }; /** Configuration for integrations towards various external repository provider systems */ - integrations: { + integrations?: { /** Integration configuration for Azure */ - azure: { + azure?: { /** The hostname of the given Azure instance */ host: string; /** @@ -97,7 +97,7 @@ export interface Config { }[]; /** Integration configuration for BitBucket */ - bitbucket: { + bitbucket?: { /** The hostname of the given Bitbucket instance */ host: string; /** @@ -120,7 +120,7 @@ export interface Config { }[]; /** Integration configuration for GitHub */ - github: { + github?: { /** The hostname of the given GitHub instance */ host: string; /** @@ -135,7 +135,7 @@ export interface Config { }[]; /** Integration configuration for GitLab */ - gitlab: { + gitlab?: { /** The hostname of the given GitLab instance */ host: string; /** From 82043bfe8bcaedd4717b42767eef35367078a69d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 23:59:15 +0100 Subject: [PATCH 127/259] docs: add docs for configuration schema --- .github/styles/vocab.txt | 1 + docs/conf/defining.md | 103 +++++++++++++++++++++++++++++++++++++-- docs/conf/index.md | 18 ++++++- docs/conf/writing.md | 10 ++-- 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 3c5fa34adc..08e9883560 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -76,6 +76,7 @@ http https img incentivised +inlined inlinehilite interop javascript diff --git a/docs/conf/defining.md b/docs/conf/defining.md index bea03e4e44..83df8c3297 100644 --- a/docs/conf/defining.md +++ b/docs/conf/defining.md @@ -4,16 +4,109 @@ title: Defining Configuration for your Plugin description: Documentation on Defining Configuration for your Plugin --- -There is currently no tooling support or helpers for defining plugin -configuration. But it's on the roadmap. +Configuration in Backstage is organized via a configuration schema, which in +turn is defined using a super set of +[JSON Schema Draft-07](https://json-schema.org/specification-links.html#draft-7). +Each plugin or package within a Backstage app can contribute to the schema, +which during validation is stitched together into a single schema. -Meanwhile, document the config values that you are reading in your plugin -README. +## Schema Collection and Definition -## Format +Schemas are collected from all packages and dependencies in each repo that are a +part of the Backstage ecosystem, including transitive dependencies. The current +definition of "part of the ecosystem" is that a package has at least one +dependency in the `@backstage` namespace, but this is subject to change. + +Each package is search for schema at a single point of entry, a top-level +`"configSchema"` field in `package.json`. The field can either contain an +inlined JSON schema, or a relative path to a schema file. Supported schema file +formats are `.json` or `.d.ts`. + +> When using a schema file, be sure to include the file in your `package.json` > +> `"files"` field as well! + +TypeScript configuration schema files should export a single `Config` type, for +example: + +```ts +export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; +} +``` + +Separate `.json` schema files can use a top-level +`"$schema": "https://backstage.io/schema/config-v1"` declaration in order to +receive schema validation and autocompletion. For example: + +```json +{ + "$schema": "https://backstage.io/schema/config-v1", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "description": "Frontend root URL", + "visibility": "frontend" + } + }, + "required": ["baseUrl"] + }, + "required": ["app"] + } +} +``` + +## Visibility + +The `https://backstage.io/schema/config-v1` meta schema is a super set of JSON +Schema Draft 07. The single addition is a custom `visibility` keyword, which is +used to indicate whether the given config value should be visible in the +frontend or not. The possible values are `frontend`, `backend`, `secret`, where +`backend` is the default. A visibility of `secret` has the same scope at +runtime, but it will be treated with more care in certain contexts, and defining +both `frontend` and `secret` for the same value in two different schemas will +result in an error during schema merging. + +The visibility only applies to the direct parent of where the keyword is placed +in the schema. For example, if you set the visibility to `frontend` for a subset +of the schema with `type: "object"`, but none of the descendants, only an empty +object will be available in the frontend. The full ancestry does not need to +have correctly defined visibilities however, so it is enough to only for example +declare the visibility of a leaf node of `type: "string"`. + +## Validation + +Schema can be validated using the `backstage-cli config:validate` command. If +you want to validate anything else than the default `app-config.yaml`, be sure +to pass in all of the configuration files as `--config ` options as well. + +To validate and examine the frontend configuration, use can use the +`backstage-cli config:print --frontend` command. Just like for validation you +may need to pass in all files using one or multiple `--config ` options. + +## Guidelines + +> Limit static configuration. The first question to ask is whether a particular +> option actually needs to be static configuration, or if it might just as well +> be a TypeScript API. In general options that you want to be able to change for +> different deployment environments should be static configuration, while it +> should otherwise be avoided. When defining configuration for your plugin, keep keys camelCased and stick to existing casing conventions such as `baseUrl`. It is also usually best to prefer objects over arrays, as it makes it possible to override individual values using separate files or environment variables. + +Avoid creating new top-level fields as much as possible. Either place your +configuration within an existing known top-level block, or create a single new +one using e.g. the name of the product that the plugin integrates. diff --git a/docs/conf/index.md b/docs/conf/index.md index 5e4639cc99..96fb40087d 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -33,6 +33,20 @@ values that are common between the two only need to be defined once. Such as the For more details, see [Writing Configuration](./writing.md). +## Configuration Schema + +The configuration is validated using a JSON Schema definitions. Each plugin and +package can provide pieces of the configuration schema, which is stitched +together to form a complete schema during validation. The configuration schema +is also used to select what configuration is available in the frontend using a +custom `visibility` keyword, as configuration is by default only available in +the backend. + +You can validate your configuration against the schema using +`backstage-cli config:validate`, and define schema for your own plugin either +using JSON Schema or TypeScript. For more information, see +[Defining Configuration](./defining.md). + ## Reading Configuration As a plugin developer, you likely end up wanting to define configuration that @@ -49,5 +63,5 @@ More details are provided in dedicated sections of the documentation. plugin. - [Writing Configuration](./writing.md): How to provide configuration for your Backstage deployment. -- [Defining Configuration](./defining.md): How to define configuration for users - of your plugin. +- [Defining Configuration](./defining.md): How to define configuration schema + for users of your plugin or package. diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 6b60c015ae..f28dd5b2d3 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -97,10 +97,10 @@ order: - If no config flags are provided, `app-config.local.yaml` has higher priority than `app-config.yaml`. -## Secrets +## Secrets and Dynamic Data -Secrets are supported via special secret keys that are prefixed with `$`, which -in turn provide a number of different ways to read in secrets. To load a +Secrets are supported via special data loading keys that are prefixed with `$`, +which in turn provide a number of different ways to read in secrets. To load a configuration value as a secret, supply an object with one of the special secret keys, for example `$env` or `$file`. A full list of supported secret keys can be found below. For example, the following will read the config key @@ -117,10 +117,6 @@ will return the value of the environment variable `MY_SECRET_KEY` when the backend started up. All secrets are loaded at startup, so changing the contents of secret files or environment variables will not be reflected at runtime. -Note that secrets will never be included in the frontend bundle or development -builds. When loading configuration you have to explicitly enable reading of -secrets, which is only done for the backend configuration. - As hinted at, secrets can be loaded from a bunch of different sources, and can be extended with more. Below is a list of the currently supported methods for loading secrets. From e2fc08cb15a3e449563d6777a08e033c468fa8ce Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 00:35:09 +0100 Subject: [PATCH 128/259] app-backend: fix tests --- plugins/app-backend/src/lib/config.test.ts | 45 ++++++++++++++----- plugins/app-backend/src/lib/config.ts | 7 ++- .../app-backend/src/service/router.test.ts | 5 ++- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/plugins/app-backend/src/lib/config.test.ts b/plugins/app-backend/src/lib/config.test.ts index 475f72f3b1..95b34c8422 100644 --- a/plugins/app-backend/src/lib/config.test.ts +++ b/plugins/app-backend/src/lib/config.test.ts @@ -43,11 +43,23 @@ describe('injectConfig', () => { jest.resetAllMocks(); }); - it('should not inject without config', async () => { + it('should inject without config', async () => { + fsMock.readdir.mockResolvedValue(['main.js']); + readFileMock.mockImplementation( + async () => '"__APP_INJECTED_RUNTIME_CONFIG__"', + ); await injectConfig(baseOptions); - expect(fsMock.readdir).toHaveBeenCalledTimes(0); - expect(fsMock.readFile).toHaveBeenCalledTimes(0); - expect(fsMock.writeFile).toHaveBeenCalledTimes(0); + expect(fsMock.readdir).toHaveBeenCalledTimes(1); + expect(fsMock.readFile).toHaveBeenCalledTimes(1); + expect(fsMock.writeFile).toHaveBeenCalledTimes(1); + expect(fsMock.writeFile).toHaveBeenCalledWith( + resolvePath(MOCK_DIR, 'main.js'), + '/*__APP_INJECTED_CONFIG_MARKER__*/"[]"/*__INJECTED_END__*/', + 'utf8', + ); + + // eslint-disable-next-line no-eval + expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual([]); }); it('should find the correct file to inject', async () => { @@ -83,14 +95,19 @@ describe('injectConfig', () => { expect(fsMock.writeFile).toHaveBeenCalledTimes(1); expect(fsMock.writeFile).toHaveBeenCalledWith( resolvePath(MOCK_DIR, 'main.js'), - '/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/', + '/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":0},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/', 'utf8', ); // eslint-disable-next-line no-eval - expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual({ - x: 0, - }); + expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual([ + { + data: { + x: 0, + }, + context: 'test', + }, + ]); }); it('should re-inject config', async () => { @@ -107,12 +124,14 @@ describe('injectConfig', () => { expect(fsMock.writeFile).toHaveBeenCalledTimes(1); expect(fsMock.writeFile).toHaveBeenCalledWith( resolvePath(MOCK_DIR, 'main.js'), - 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/)', + 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":0},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/)', 'utf8', ); // eslint-disable-next-line no-eval - expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual({ x: 0 }); + expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual([ + { data: { x: 0 }, context: 'test' }, + ]); readFileMock.mockResolvedValue(fsMock.writeFile.mock.calls[0][1]); @@ -124,11 +143,13 @@ describe('injectConfig', () => { expect(fsMock.writeFile).toHaveBeenCalledTimes(2); expect(fsMock.writeFile).toHaveBeenLastCalledWith( resolvePath(MOCK_DIR, 'main.js'), - 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":1,\\"y\\":2}"/*__INJECTED_END__*/)', + 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":1,\\"y\\":2},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/)', 'utf8', ); // eslint-disable-next-line no-eval - expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual({ x: 1, y: 2 }); + expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual([ + { data: { x: 1, y: 2 }, context: 'test' }, + ]); }); }); diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 82b431c33f..f076967280 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -28,8 +28,7 @@ type InjectOptions = { }; /** - * Injects config from APP_CONFIG_ env vars, replacing existing - * injected config if it has already been injected. + * Injects configs into the app bundle, replacing any existing injected config. */ export async function injectConfig(options: InjectOptions) { const { staticDir, logger, appConfigs } = options; @@ -73,6 +72,10 @@ type ReadOptions = { config: Config; }; +/** + * Read config from environment and process the backend config using the + * schema that is embedded in the frontend build. + */ export async function readConfigs(options: ReadOptions): Promise { const { env, appDistDir, config } = options; diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts index b0cc3c59f7..35dd9287a6 100644 --- a/plugins/app-backend/src/service/router.test.ts +++ b/plugins/app-backend/src/service/router.test.ts @@ -22,7 +22,10 @@ import Router from 'express-promise-router'; import request from 'supertest'; import { createRouter } from './router'; -jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() })); +jest.mock('../lib/config', () => ({ + injectConfig: jest.fn(), + readConfigs: jest.fn(), +})); global.__non_webpack_require__ = { /* eslint-disable-next-line no-restricted-syntax */ From 90fb5c81d06779e9a63bdaa1e59da96f76e6c75a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 01:05:54 +0100 Subject: [PATCH 129/259] cli: update diff to handle package with configSchema files --- packages/cli/src/lib/diff/handlers.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/lib/diff/handlers.ts b/packages/cli/src/lib/diff/handlers.ts index 40e3d4c587..a210986b2a 100644 --- a/packages/cli/src/lib/diff/handlers.ts +++ b/packages/cli/src/lib/diff/handlers.ts @@ -61,7 +61,7 @@ class PackageJsonHandler { await this.syncField('main:src'); } await this.syncField('types'); - await this.syncField('files'); + await this.syncFiles(); await this.syncScripts(); await this.syncPublishConfig(); await this.syncDependencies('dependencies'); @@ -105,6 +105,15 @@ class PackageJsonHandler { } } + private async syncFiles() { + if (typeof this.targetPkg.configSchema === 'string') { + const files = [...this.pkg.files, this.targetPkg.configSchema]; + await this.syncField('files', { files }); + } else { + await this.syncField('files'); + } + } + private async syncScripts() { const pkgScripts = this.pkg.scripts; const targetScripts = (this.targetPkg.scripts = From 9621ef3a9617277cdbfe97ebd06245d9b51975ef Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 10:28:23 +0100 Subject: [PATCH 130/259] techdocs: add storageUrl to config schema Co-authored-by: Sebastian Qvarfordt --- plugins/techdocs/package.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index a513b7aa8d..a2b1349d53 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -65,8 +65,14 @@ "requestUrl": { "type": "string", "visibility": "frontend" + }, + "storageUrl": { + "type": "string" } - } + }, + "required": [ + "requestUrl" + ] } } } From 5357723ea456332b1dfb32f6b2adbe445511bd9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 14:56:08 +0100 Subject: [PATCH 131/259] config-loader: fix .d.ts schema path handling on Windows --- .../config-loader/src/lib/schema/collect.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 11a7ce4ec2..9d8e6e73f9 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -19,6 +19,7 @@ import { resolve as resolvePath, relative as relativePath, dirname, + sep, } from 'path'; import { ConfigSchemaPackageEntry } from './types'; import { getProgramFromFiles, generateSchema } from 'typescript-json-schema'; @@ -43,7 +44,7 @@ export async function collectConfigSchemas( const visitedPackages = new Set(); const schemas = Array(); const tsSchemaPaths = Array(); - const currentDir = process.cwd(); + const currentDir = await fs.realpath(process.cwd()); async function processItem({ name, parentPath }: Item) { // Ensures that we only process each package once. We don't bother with @@ -94,7 +95,12 @@ export async function collectConfigSchemas( ); } if (isDts) { - tsSchemaPaths.push(resolvePath(dirname(pkgPath), pkg.configSchema)); + tsSchemaPaths.push( + relativePath( + currentDir, + resolvePath(dirname(pkgPath), pkg.configSchema), + ), + ); } else { const path = resolvePath(dirname(pkgPath), pkg.configSchema); const value = await fs.readJson(path); @@ -118,7 +124,7 @@ export async function collectConfigSchemas( await Promise.all(packageNames.map(name => processItem({ name }))); - const tsSchemas = compileTsSchemas(currentDir, tsSchemaPaths); + const tsSchemas = compileTsSchemas(tsSchemaPaths); return schemas.concat(tsSchemas); } @@ -126,7 +132,7 @@ export async function collectConfigSchemas( // This handles the support of TypeScript .d.ts config schema declarations. // We collect all typescript schema definition and compile them all in one go. // This is much faster than compiling them separately. -function compileTsSchemas(currentDir: string, paths: string[]) { +function compileTsSchemas(paths: string[]) { if (paths.length === 0) { return []; } @@ -144,7 +150,7 @@ function compileTsSchemas(currentDir: string, paths: string[]) { types: [], }); - const tsSchemas = paths.map(schemaFile => { + const tsSchemas = paths.map(path => { const value = generateSchema( program, // All schemas should export a `Config` symbol @@ -154,11 +160,9 @@ function compileTsSchemas(currentDir: string, paths: string[]) { required: true, validationKeywords: ['visibility'], }, - [schemaFile], + [path.split(sep).join('/')], // Unix paths are expected for all OSes here ) as JsonObject | null; - const path = relativePath(currentDir, schemaFile); - if (!value) { throw new Error(`Invalid schema in ${path}, missing Config export`); } From 72116243287db45f35f3b94cd655632a0902fe35 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 19:20:16 +0100 Subject: [PATCH 132/259] config-loader: added tests for schema collection --- .../src/lib/schema/collect.test.ts | 229 ++++++++++++++++++ .../config-loader/src/lib/schema/collect.ts | 29 ++- 2 files changed, 247 insertions(+), 11 deletions(-) create mode 100644 packages/config-loader/src/lib/schema/collect.test.ts diff --git a/packages/config-loader/src/lib/schema/collect.test.ts b/packages/config-loader/src/lib/schema/collect.test.ts new file mode 100644 index 0000000000..7aca4f7c0c --- /dev/null +++ b/packages/config-loader/src/lib/schema/collect.test.ts @@ -0,0 +1,229 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { collectConfigSchemas } from './collect'; + +const mockSchema = { + type: 'object', + properties: { + key: { + type: 'string', + visibility: 'frontend', + }, + }, +}; + +describe('collectConfigSchemas', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should not find any schemas without packages', async () => { + mockFs({ + 'lerna.json': JSON.stringify({ + packages: ['packages/*'], + }), + }); + + await expect(collectConfigSchemas([])).resolves.toEqual([]); + }); + + it('should find schema in a local package', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: mockSchema, + }), + }, + }, + }); + + await expect(collectConfigSchemas(['a'])).resolves.toEqual([ + { + path: 'node_modules/a/package.json', + value: mockSchema, + }, + ]); + }); + + it('should find schema in transitive dependencies', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + dependencies: { b: '0.0.0', '@backstage/mock': '0.0.0' }, + }), + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + dependencies: { + c1: '0.0.0', + c2: '0.0.0', + '@backstage/mock': '0.0.0', + }, + configSchema: { ...mockSchema, title: 'b' }, + }), + }, + c1: { + 'package.json': JSON.stringify({ + name: 'c1', + dependencies: { d1: '0.0.0' }, + configSchema: { ...mockSchema, title: 'c1' }, + }), + }, + c2: { + 'package.json': JSON.stringify({ + name: 'c2', + dependencies: { d2: '0.0.0' }, + }), + }, + d1: { + 'package.json': JSON.stringify({ + name: 'd1', + dependencies: {}, + configSchema: { ...mockSchema, title: 'd1' }, + }), + }, + d2: { + 'package.json': JSON.stringify({ + name: 'd2', + dependencies: {}, + configSchema: { ...mockSchema, title: 'd2' }, + }), + }, + }, + }); + + await expect(collectConfigSchemas(['a'])).resolves.toEqual([ + { + path: 'node_modules/b/package.json', + value: { ...mockSchema, title: 'b' }, + }, + { + path: 'node_modules/c1/package.json', + value: { ...mockSchema, title: 'c1' }, + }, + { + path: 'node_modules/d1/package.json', + value: { ...mockSchema, title: 'd1' }, + }, + ]); + }); + + it('should schema of different types', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: { ...mockSchema, title: 'inline' }, + }), + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + configSchema: 'schema.json', + }), + 'schema.json': JSON.stringify({ ...mockSchema, title: 'external' }), + }, + c: { + 'package.json': JSON.stringify({ + name: 'c', + configSchema: 'schema.d.ts', + }), + 'schema.d.ts': `export interface Config { + /** @visibility secret */ + tsKey: string + }`, + }, + }, + // TypeScript compilation needs to load some real files inside the typescript dir + '../../node_modules/typescript': (mockFs as any).load( + '../../node_modules/typescript', + ), + }); + + await expect(collectConfigSchemas(['a', 'b', 'c'])).resolves.toEqual([ + { + path: 'node_modules/a/package.json', + value: { ...mockSchema, title: 'inline' }, + }, + { + path: 'node_modules/b/schema.json', + value: { ...mockSchema, title: 'external' }, + }, + { + path: 'node_modules/c/schema.d.ts', + value: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { + tsKey: { + type: 'string', + visibility: 'secret', + }, + }, + required: ['tsKey'], + }, + }, + ]); + }); + + it('should not allow unknown schema file types', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: 'schema.yaml', + }), + 'schema.yaml': mockSchema, + }, + }, + }); + + await expect(collectConfigSchemas(['a'])).rejects.toThrow( + 'Config schema files must be .json or .d.ts, got schema.yaml', + ); + }); + + it('should reject typescript config declaration without a Config type', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: 'schema.d.ts', + }), + 'schema.d.ts': `export interface NotConfig {}`, + }, + }, + // TypeScript compilation needs to load some real files inside the typescript dir + '../../node_modules/typescript': (mockFs as any).load( + '../../node_modules/typescript', + ), + }); + + await expect(collectConfigSchemas(['a'])).rejects.toThrow( + 'Invalid schema in node_modules/a/schema.d.ts, missing Config export', + ); + }); +}); diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 9d8e6e73f9..0e53562875 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -151,17 +151,24 @@ function compileTsSchemas(paths: string[]) { }); const tsSchemas = paths.map(path => { - const value = generateSchema( - program, - // All schemas should export a `Config` symbol - 'Config', - // This enables usage of @visibility is doc comments - { - required: true, - validationKeywords: ['visibility'], - }, - [path.split(sep).join('/')], // Unix paths are expected for all OSes here - ) as JsonObject | null; + let value; + try { + value = generateSchema( + program, + // All schemas should export a `Config` symbol + 'Config', + // This enables usage of @visibility is doc comments + { + required: true, + validationKeywords: ['visibility'], + }, + [path.split(sep).join('/')], // Unix paths are expected for all OSes here + ) as JsonObject | null; + } catch (error) { + if (error.message !== 'type Config not found') { + throw error; + } + } if (!value) { throw new Error(`Invalid schema in ${path}, missing Config export`); From eda9f91515b6024a79e273d24044f2f643ee8d40 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 19:32:25 +0100 Subject: [PATCH 133/259] config-loader: added test for config schema load --- .../config-loader/src/lib/schema/load.test.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 packages/config-loader/src/lib/schema/load.test.ts diff --git a/packages/config-loader/src/lib/schema/load.test.ts b/packages/config-loader/src/lib/schema/load.test.ts new file mode 100644 index 0000000000..a13f36be20 --- /dev/null +++ b/packages/config-loader/src/lib/schema/load.test.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { loadConfigSchema } from './load'; + +describe('loadConfigSchema', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should load schema from packages or data', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: { + type: 'object', + properties: { + key1: { type: 'string', visibility: 'frontend' }, + }, + }, + }), + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + configSchema: 'schema.json', + }), + 'schema.json': JSON.stringify({ + name: 'a', + configSchema: { + type: 'object', + properties: { + key2: { type: 'number' }, + }, + }, + }), + }, + }, + }); + + const schema = await loadConfigSchema({ + dependencies: ['a'], + }); + + const configs = [{ data: { key1: 'a', key2: 2 }, context: 'test' }]; + + expect(schema.process(configs)).toEqual(configs); + expect(schema.process(configs, { visiblity: ['frontend'] })).toEqual([ + { data: { key1: 'a' }, context: 'test' }, + ]); + expect( + schema.process(configs, { + visiblity: ['frontend'], + valueTransform: () => 'X', + }), + ).toEqual([{ data: { key1: 'X' }, context: 'test' }]); + expect( + schema.process(configs, { + valueTransform: () => 'X', + }), + ).toEqual([{ data: { key1: 'X', key2: 'X' }, context: 'test' }]); + + const serialized = schema.serialize(); + + const schema2 = await loadConfigSchema({ serialized }); + expect(schema2.process(configs, { visiblity: ['frontend'] })).toEqual([ + { data: { key1: 'a' }, context: 'test' }, + ]); + expect(() => + schema2.process([...configs, { data: { key1: 3 }, context: 'test2' }]), + ).toThrow( + 'Config validation failed, Config should be string { type=string } at .key1', + ); + + await expect( + loadConfigSchema({ + serialized: { ...serialized, backstageConfigSchemaVersion: 2 }, + }), + ).rejects.toThrow( + 'Serialized configuration schema is invalid or has an invalid version number', + ); + }); +}); From 39591369bacda81c315a6e3f01ce07af004f4692 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 19:42:52 +0100 Subject: [PATCH 134/259] config-loader: add tests for schema compile and skip adding backend visibility paths --- .../src/lib/schema/compile.test.ts | 114 ++++++++++++++++++ .../config-loader/src/lib/schema/compile.ts | 2 +- 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 packages/config-loader/src/lib/schema/compile.test.ts diff --git a/packages/config-loader/src/lib/schema/compile.test.ts b/packages/config-loader/src/lib/schema/compile.test.ts new file mode 100644 index 0000000000..91e7aa687e --- /dev/null +++ b/packages/config-loader/src/lib/schema/compile.test.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { compileConfigSchemas } from './compile'; + +describe('compileConfigSchemas', () => { + it('should merge schemas', () => { + const validate = compileConfigSchemas([ + { + path: 'a', + value: { type: 'object', properties: { a: { type: 'string' } } }, + }, + { + path: 'b', + value: { type: 'object', properties: { b: { type: 'number' } } }, + }, + ]); + expect(validate([{ data: { a: 1 }, context: 'test' }])).toEqual({ + errors: ['Config should be string { type=string } at .a'], + visibilityByPath: new Map(), + }); + expect(validate([{ data: { b: 'b' }, context: 'test' }])).toEqual({ + errors: ['Config should be number { type=number } at .b'], + visibilityByPath: new Map(), + }); + }); + + it('should discover visibilities', () => { + const validate = compileConfigSchemas([ + { + path: 'a1', + value: { + type: 'object', + properties: { + a: { type: 'string', visibility: 'frontend' }, + b: { type: 'string', visibility: 'backend' }, + c: { type: 'string' }, + d: { + type: 'array', + visibility: 'secret', + items: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + }, + { + path: 'a2', + value: { + type: 'object', + properties: { + a: { type: 'string' }, + b: { type: 'string', visibility: 'secret' }, + c: { type: 'string', visibility: 'backend' }, + d: { + type: 'array', + visibility: 'secret', + items: { type: 'string' }, + }, + }, + }, + }, + ]); + expect( + validate([ + { data: { a: 'a', b: 'b', c: 'c', d: ['d'] }, context: 'test' }, + ]), + ).toEqual({ + visibilityByPath: new Map( + Object.entries({ + '.a': 'frontend', + '.b': 'secret', + '.d': 'secret', + '.d.0': 'frontend', + }), + ), + }); + }); + + it('should reject visiblity conflicts', () => { + expect(() => + compileConfigSchemas([ + { + path: 'a1', + value: { + type: 'object', + properties: { a: { type: 'string', visibility: 'frontend' } }, + }, + }, + { + path: 'a2', + value: { + type: 'object', + properties: { a: { type: 'string', visibility: 'secret' } }, + }, + }, + ]), + ).toThrow( + "Config schema visibility is both 'frontend' and 'secret' for properties/a/visibility", + ); + }); +}); diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 93ab8e978f..f01a4f640a 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -55,7 +55,7 @@ export function compileConfigSchemas( if (!dataPath) { return false; } - if (visibility) { + if (visibility && visibility !== 'backend') { const normalizedPath = dataPath.replace( /\['?(.*?)'?\]/g, (_, segment) => `.${segment}`, From 0b97a89f55b9bc9174ab1475456593400a21f300 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Nov 2020 18:25:55 +0100 Subject: [PATCH 135/259] address config schema review comments --- .changeset/add-config-schema-support.md | 6 ++-- .changeset/app-backend-config.md | 2 +- .github/styles/vocab.txt | 1 + .github/workflows/master.yml | 3 ++ docs/conf/defining.md | 30 +++++++++---------- docs/conf/index.md | 8 ++--- packages/backend-common/config.d.ts | 18 +++++------ .../config-loader/src/lib/schema/types.ts | 6 ++-- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/.changeset/add-config-schema-support.md b/.changeset/add-config-schema-support.md index fd20426b30..300c75f651 100644 --- a/.changeset/add-config-schema-support.md +++ b/.changeset/add-config-schema-support.md @@ -4,13 +4,13 @@ '@backstage/config-loader': minor --- -Added support for loading and validating configuration schema, as well as declaring config visibility through schema. +Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas. The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. -Configuration schema is declared using the following JSONSchema meta schema, which is based on draft07: https://backstage.io/schema/config-v1. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. +A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. -Packages that wish to contribute configuration schema should declare it in a root "configSchema" field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be declared in either `.json` or `.d.ts`. +Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format. TypeScript configuration schema files should export a single `Config` type, for example: diff --git a/.changeset/app-backend-config.md b/.changeset/app-backend-config.md index 9df47ef1b8..39b0b956ad 100644 --- a/.changeset/app-backend-config.md +++ b/.changeset/app-backend-config.md @@ -4,4 +4,4 @@ Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection. -This removes the confusing behavior where configuration was only injected into the app att build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. +This removes the confusing behavior where configuration was only injected into the app at build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 08e9883560..740cf5b07f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -177,6 +177,7 @@ src subkey superfences Superfences +superset talkdesk Talkdesk tasklist diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f5029bf836..4c20194f32 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -47,6 +47,9 @@ jobs: run: yarn install --frozen-lockfile # End of yarn setup + - name: validate config + run: yarn backstage-cli config:validate + - name: lint run: yarn lerna -- run lint diff --git a/docs/conf/defining.md b/docs/conf/defining.md index 83df8c3297..d231320e93 100644 --- a/docs/conf/defining.md +++ b/docs/conf/defining.md @@ -5,7 +5,7 @@ description: Documentation on Defining Configuration for your Plugin --- Configuration in Backstage is organized via a configuration schema, which in -turn is defined using a super set of +turn is defined using a superset of [JSON Schema Draft-07](https://json-schema.org/specification-links.html#draft-7). Each plugin or package within a Backstage app can contribute to the schema, which during validation is stitched together into a single schema. @@ -17,13 +17,13 @@ part of the Backstage ecosystem, including transitive dependencies. The current definition of "part of the ecosystem" is that a package has at least one dependency in the `@backstage` namespace, but this is subject to change. -Each package is search for schema at a single point of entry, a top-level +Each package is searched for a schema at a single point of entry, a top-level `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Supported schema file formats are `.json` or `.d.ts`. -> When using a schema file, be sure to include the file in your `package.json` > -> `"files"` field as well! +> When defining a schema file, be sure to include the file in your +> `package.json` > `"files"` field as well! TypeScript configuration schema files should export a single `Config` type, for example: @@ -67,11 +67,11 @@ receive schema validation and autocompletion. For example: ## Visibility -The `https://backstage.io/schema/config-v1` meta schema is a super set of JSON +The `https://backstage.io/schema/config-v1` meta schema is a superset of JSON Schema Draft 07. The single addition is a custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the -frontend or not. The possible values are `frontend`, `backend`, `secret`, where -`backend` is the default. A visibility of `secret` has the same scope at +frontend or not. The possible values are `frontend`, `backend`, and `secret`, +where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. @@ -85,24 +85,24 @@ declare the visibility of a leaf node of `type: "string"`. ## Validation -Schema can be validated using the `backstage-cli config:validate` command. If +Schemas can be validated using the `backstage-cli config:validate` command. If you want to validate anything else than the default `app-config.yaml`, be sure to pass in all of the configuration files as `--config ` options as well. -To validate and examine the frontend configuration, use can use the +To validate and examine the frontend configuration, use the `backstage-cli config:print --frontend` command. Just like for validation you may need to pass in all files using one or multiple `--config ` options. ## Guidelines -> Limit static configuration. The first question to ask is whether a particular -> option actually needs to be static configuration, or if it might just as well -> be a TypeScript API. In general options that you want to be able to change for -> different deployment environments should be static configuration, while it -> should otherwise be avoided. +> Make limited use of static configuration. The first question to ask is whether +> a particular option actually needs to be static configuration, or if it might +> just as well be a TypeScript API. In general, options that you want to be able +> to change for different deployment environments should be static +> configuration, while it should otherwise be avoided. When defining configuration for your plugin, keep keys camelCased and stick to -existing casing conventions such as `baseUrl`. +existing casing conventions such as `baseUrl` rather than `baseURL`. It is also usually best to prefer objects over arrays, as it makes it possible to override individual values using separate files or environment variables. diff --git a/docs/conf/index.md b/docs/conf/index.md index 96fb40087d..48da4eb74e 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -35,15 +35,15 @@ For more details, see [Writing Configuration](./writing.md). ## Configuration Schema -The configuration is validated using a JSON Schema definitions. Each plugin and -package can provide pieces of the configuration schema, which is stitched +The configuration is validated using JSON Schema definitions. Each plugin and +package can provide pieces of the configuration schema, which are stitched together to form a complete schema during validation. The configuration schema is also used to select what configuration is available in the frontend using a custom `visibility` keyword, as configuration is by default only available in the backend. You can validate your configuration against the schema using -`backstage-cli config:validate`, and define schema for your own plugin either +`backstage-cli config:validate`, and define a schema for your own plugin either using JSON Schema or TypeScript. For more information, see [Defining Configuration](./defining.md). @@ -63,5 +63,5 @@ More details are provided in dedicated sections of the documentation. plugin. - [Writing Configuration](./writing.md): How to provide configuration for your Backstage deployment. -- [Defining Configuration](./defining.md): How to define configuration schema +- [Defining Configuration](./defining.md): How to define a configuration schema for users of your plugin or package. diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 9e8fb7ee35..7ebcd01e8c 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -21,8 +21,8 @@ export interface Config { backend: { baseUrl: string; // defined in core, but repeated here without doc - /** Address that the backend should listen to. */ + /** Address that the backend should listen to. */ listen: | string | { @@ -86,7 +86,7 @@ export interface Config { /** Configuration for integrations towards various external repository provider systems */ integrations?: { /** Integration configuration for Azure */ - azure?: { + azure?: Array<{ /** The hostname of the given Azure instance */ host: string; /** @@ -94,10 +94,10 @@ export interface Config { * @visibility secret */ token?: string; - }[]; + }>; /** Integration configuration for BitBucket */ - bitbucket?: { + bitbucket?: Array<{ /** The hostname of the given Bitbucket instance */ host: string; /** @@ -117,10 +117,10 @@ export interface Config { * @visibility secret */ appPassword?: string; - }[]; + }>; /** Integration configuration for GitHub */ - github?: { + github?: Array<{ /** The hostname of the given GitHub instance */ host: string; /** @@ -132,10 +132,10 @@ export interface Config { apiBaseUrl?: string; /** The base url for GitHub raw resources, for example https://raw.githubusercontent.com */ rawBaseUrl?: string; - }[]; + }>; /** Integration configuration for GitLab */ - gitlab?: { + gitlab?: Array<{ /** The hostname of the given GitLab instance */ host: string; /** @@ -143,6 +143,6 @@ export interface Config { * @visibility secret */ token?: string; - }[]; + }>; }; } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index c82036bbf4..7705242b31 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -21,11 +21,11 @@ import { AppConfig, JsonObject } from '@backstage/config'; */ export type ConfigSchemaPackageEntry = { /** - * The configuration schema itself, as JSONSchema draft-07 + * The configuration schema itself. */ value: JsonObject; /** - * The path that the configuration schema was discovered at. + * The relative path that the configuration schema was discovered at. */ path: string; }; @@ -59,7 +59,7 @@ type ValidationResult = { */ errors?: ValidationError[]; /** - * The configuration visibilities the where discovered during validation. + * The configuration visibilities that were discovered during validation. * * The path in the key uses the form `////` */ From 80bf8c7f4fb97f2af0dbefce0fabb42980de50c8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Nov 2020 18:26:40 +0100 Subject: [PATCH 136/259] cli: rename config:validate to config:check --- .github/workflows/ci.yml | 2 +- .github/workflows/master.yml | 2 +- docs/conf/defining.md | 6 +++--- docs/conf/index.md | 2 +- packages/cli/src/commands/index.ts | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ed9ce714..c35e2a88ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: run: yarn prettier:check - name: validate config - run: yarn backstage-cli config:validate + run: yarn backstage-cli config:check - name: lint run: yarn lerna -- run lint --since origin/master diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4c20194f32..ca63e627e9 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -48,7 +48,7 @@ jobs: # End of yarn setup - name: validate config - run: yarn backstage-cli config:validate + run: yarn backstage-cli config:check - name: lint run: yarn lerna -- run lint diff --git a/docs/conf/defining.md b/docs/conf/defining.md index d231320e93..ead5c4ff67 100644 --- a/docs/conf/defining.md +++ b/docs/conf/defining.md @@ -85,9 +85,9 @@ declare the visibility of a leaf node of `type: "string"`. ## Validation -Schemas can be validated using the `backstage-cli config:validate` command. If -you want to validate anything else than the default `app-config.yaml`, be sure -to pass in all of the configuration files as `--config ` options as well. +Schemas can be validated using the `backstage-cli config:check` command. If you +want to validate anything else than the default `app-config.yaml`, be sure to +pass in all of the configuration files as `--config ` options as well. To validate and examine the frontend configuration, use the `backstage-cli config:print --frontend` command. Just like for validation you diff --git a/docs/conf/index.md b/docs/conf/index.md index 48da4eb74e..a6f1d1f6f7 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -43,7 +43,7 @@ custom `visibility` keyword, as configuration is by default only available in the backend. You can validate your configuration against the schema using -`backstage-cli config:validate`, and define a schema for your own plugin either +`backstage-cli config:check`, and define a schema for your own plugin either using JSON Schema or TypeScript. For more information, see [Defining Configuration](./defining.md). diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 304b5bae8f..1ebe518cac 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -147,7 +147,7 @@ export function registerCommands(program: CommanderStatic) { .action(lazy(() => import('./config/print').then(m => m.default))); program - .command('config:validate') + .command('config:check') .option(...configOption) .description( 'Validate that the given configuration loads and matches schema', From c7d6fefe9c9751d667208ae3bc6c9c879f22773c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 10:18:10 +0100 Subject: [PATCH 137/259] cli: add config schema for frontend listening config + fix usage --- packages/cli/package.json | 16 ++++++++++++++++ packages/cli/src/lib/bundler/server.ts | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 23c5c5d6af..516e354012 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -169,6 +169,22 @@ "visibility": "frontend", "description": "Tracking ID for Google Analytics", "example": "UA-000000-0" + }, + "listen": { + "type": "object", + "description": "Listening configuration for local development", + "properties": { + "host": { + "type": "number", + "visibility": "frontend", + "description": "The host that the frontend should be bound to. Only used for local development." + }, + "post": { + "type": "number", + "visibility": "frontend", + "description": "The port that the frontend should be bound to. Only used for local development." + } + } } } } diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index f540e01084..d198f5507c 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -26,9 +26,9 @@ export async function serveBundle(options: ServeOptions) { const url = resolveBaseUrl(options.frontendConfig); const host = - options.config.getOptionalString('app.listen.host') || url.hostname; + options.frontendConfig.getOptionalString('app.listen.host') || url.hostname; const port = - options.config.getOptionalNumber('app.listen.port') || + options.frontendConfig.getOptionalNumber('app.listen.port') || Number(url.port) || (url.protocol === 'https:' ? 443 : 80); From 102f0b394687a2464ac72e2bf87047f41ca9e2bd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 11:28:17 +0100 Subject: [PATCH 138/259] docs/stability-index: trim down list of plugins but open up for contributions + set techdocs-backend to 0 --- .github/styles/vocab.txt | 1 + docs/overview/stability-index.md | 101 ++----------------------------- 2 files changed, 6 insertions(+), 96 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index bd89163f1f..b9d9e53dfa 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -196,6 +196,7 @@ toolsets tooltip touchpoints ui +untracked upvote url utils diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index 1ec90e106d..f5a769c601 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -225,6 +225,10 @@ Many backend plugins are split into "REST API" and "TypeScript Interface" sections. The "TypeScript Interface" refers to the API used to integrate the plugin into the backend. +Any plugin that is not listed below is untracked and can generally be considered +unstable with a score of `0`. Open a Pull Request if you want your plugin to be +added! + ### [`api-docs`](https://github.com/backstage/backstage/tree/master/plugins/api-docs/) Components to discover and display API entities as an extension to the catalog @@ -280,25 +284,6 @@ Provides the catalog schema and resolvers for the graphql backend. Stability: `0`. Under heavy development and subject to change. -### [`circleci`](https://github.com/backstage/backstage/tree/master/plugins/circleci/) - -Automate your development process with CI hosted in the cloud or on a private -server. - -Stability: `0` - -### [`cloudbuild`](https://github.com/backstage/backstage/tree/master/plugins/cloudbuild/) - -Visualize Google Cloud Build flows. - -Stability: `0` - -### [`cost-insights`](https://github.com/backstage/backstage/tree/master/plugins/cost-insights/) - -Visualize, understand and optimize your team's cloud costs. - -Stability: `0` - ### [`explore`](https://github.com/backstage/backstage/tree/master/plugins/explore/) A frontend plugin that introduces the concept of exploring internal and external @@ -306,27 +291,6 @@ tooling in an organization. Stability: `0`. Only an example at the moment and not customizable. -### [`gcp-projects`](https://github.com/backstage/backstage/tree/master/plugins/gcp-projects/) - -Create, list and manage your Google Cloud Projects. - -Stability: `0` - -### [`github-actions`](https://github.com/backstage/backstage/tree/master/plugins/github-actions/) - -GitHub Actions makes it easy to automate all your software workflows, now with -world-class CI/CD. Build, test, and deploy your code right from GitHub. - -Stability: `0` - -### [`gitops-profiles`](https://github.com/backstage/backstage/tree/master/plugins/gitops-profiles/) - -A frontend plugin with a separate backend that can be used to provision EKS -clusters. - -Stability: `0`. This is an early plugin that now has quite a lot of overlap with -the scaffolder plugin. - ### [`graphiql`](https://github.com/backstage/backstage/tree/master/plugins/graphiql/) Integrates GraphiQL as a tool to browse GraphQL API endpoints inside Backstage. @@ -339,13 +303,6 @@ A backend plugin that provides Stability: `0`. Under heavy development and subject to change. -### [`jenkins`](https://github.com/backstage/backstage/tree/master/plugins/jenkins/) - -A plugin that visualizes Jenkins workflows for entities. Jenkins offers a simple -way to set up a continuous integration and continuous delivery environment. - -Stability: `0` - ### [`kubernetes`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes/) The frontend component of the Kubernetes plugin, used to browse and visualize @@ -360,20 +317,6 @@ resources from clusters and associate them with entities in the Catalog. Stability: `1`. -### [`lighthouse`](https://github.com/backstage/backstage/tree/master/plugins/lighthouse/) - -Google's Lighthouse tool is a great resource for benchmarking and improving the -accessibility, performance, SEO, and best practices of your website. - -Stability: `0` - -### [`newrelic`](https://github.com/backstage/backstage/tree/master/plugins/newrelic/) - -Observability platform built to help engineers create and monitor their -software. - -Stability: `0` - ### [`proxy-backend`](https://github.com/backstage/backstage/tree/master/plugins/proxy-backend/) A backend plugin used to set up proxying to other endpoints based on static @@ -389,20 +332,6 @@ catalog. Stability: `0`. This plugin is likely to be replaced by a generic entity import plugin instead. -### [`rollbar`](https://github.com/backstage/backstage/tree/master/plugins/rollbar/) - -The frontend component of the rollbar plugin, which can be used to view Rollbar -errors for your services in Backstage. - -Stability: `0` - -### [`rollbar-backend`](https://github.com/backstage/backstage/tree/master/plugins/rollbar-backend/) - -The backend component of the rollbar plugin, which can be used to view Rollbar -errors for your services in Backstage. - -Stability: `0` - ### [`scaffolder`](https://github.com/backstage/backstage/tree/master/plugins/scaffolder/) The frontend scaffolder plugin where one can browse templates and initiate @@ -418,26 +347,6 @@ the catalog. Stability: `1`. There is planned work to rework the scaffolder in https://github.com/backstage/backstage/issues/2771. -### [`sentry`](https://github.com/backstage/backstage/tree/master/plugins/sentry/) - -The frontend component of the sentry plugin, which can be used to view Sentry -issues in Backstage. - -Stability: `0` - -### [`sentry-backend`](https://github.com/backstage/backstage/tree/master/plugins/sentry-backend/) - -The backend component of the sentry plugin, which can be used to view Sentry -issues in Backstage. - -Stability: `0` - -### [`sonarqube`](https://github.com/backstage/backstage/tree/master/plugins/sonarqube/) - -Components to display code quality metrics from SonarCloud and SonarQube. - -Stability: `0` - ### [`tech-radar`](https://github.com/backstage/backstage/tree/master/plugins/tech-radar/) Visualize the your company's official guidelines of different areas of software @@ -457,7 +366,7 @@ Stability: `1` The backend component of the TechDocs plugin, used to transform and serve TechDocs. -Stability: `1` +Stability: `0` ### [`user-settings`](https://github.com/backstage/backstage/tree/master/plugins/user-settings/) From acc226dd6335066db22ad480762a0ad8ed937ce1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 17:20:13 +0100 Subject: [PATCH 139/259] Update docs/overview/stability-index.md Co-authored-by: Adam Harvey --- docs/overview/stability-index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index f5a769c601..416cdcdd4a 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -41,14 +41,14 @@ TL;DR: ### [`example-app`](https://github.com/backstage/backstage/tree/master/packages/app/) -This is the `packages/app` package, and it's serves as an example as well as +This is the `packages/app` package, and it serves as an example as well as utility for local development in the main Backstage repo. Stability: `N/A` ### [`example-backend`](https://github.com/backstage/backstage/tree/master/packages/backend/) -This is the `packages/backend` package, and it's serves as an example as well as +This is the `packages/backend` package, and it serves as an example as well as utility for local development in the main Backstage repo. Stability: `N/A` From 717e43de146b7c18798bcef4ebd0d5bd740578f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 18 Nov 2020 19:58:36 +0100 Subject: [PATCH 140/259] catalog-client: change entities interface, add fields support (#3296) --- .changeset/shaggy-turkeys-warn.md | 5 + .../catalog-client/src/CatalogClient.test.ts | 45 ++++++--- packages/catalog-client/src/CatalogClient.ts | 96 +++++++++++-------- packages/catalog-client/src/types.ts | 13 ++- .../ApiExplorerPage/ApiExplorerPage.test.tsx | 34 +++---- .../ApiExplorerPage/ApiExplorerPage.tsx | 6 +- .../CatalogFilter/CatalogFilter.test.tsx | 46 ++++----- .../CatalogPage/CatalogPage.test.tsx | 50 +++++----- .../ResultsFilter/ResultsFilter.test.tsx | 74 +++++++------- .../UnregisterEntityDialog.tsx | 10 +- .../src/filter/EntityFilterGroupsProvider.tsx | 9 +- .../src/filter/useEntityFilterGroup.test.tsx | 18 ++-- .../RollbarHome/RollbarHome.test.tsx | 2 +- .../rollbar/src/hooks/useRollbarEntities.ts | 4 +- .../ScaffolderPage/ScaffolderPage.tsx | 10 +- .../TemplatePage/TemplatePage.test.tsx | 60 ++++++------ .../components/TemplatePage/TemplatePage.tsx | 30 +++--- .../reader/components/TechDocsHome.test.tsx | 3 +- .../src/reader/components/TechDocsHome.tsx | 18 ++-- 19 files changed, 298 insertions(+), 235 deletions(-) create mode 100644 .changeset/shaggy-turkeys-warn.md diff --git a/.changeset/shaggy-turkeys-warn.md b/.changeset/shaggy-turkeys-warn.md new file mode 100644 index 0000000000..442cc788f2 --- /dev/null +++ b/.changeset/shaggy-turkeys-warn.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +Changed the getEntities interface to (1) nest parameters in an object, (2) support field selection, and (3) return an object with an items field for future extension diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 65625c11b8..6369f95b76 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { Entity } from '@backstage/catalog-model'; -import { DiscoveryApi } from './types'; +import { CatalogListResponse, DiscoveryApi } from './types'; const server = setupServer(); const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base'; @@ -40,7 +40,7 @@ describe('CatalogClient', () => { }); describe('getEntities', () => { - const defaultResponse: Entity[] = [ + const defaultServiceResponse: Entity[] = [ { apiVersion: '1', kind: 'Component', @@ -58,22 +58,26 @@ describe('CatalogClient', () => { }, }, ]; + const defaultResponse: CatalogListResponse = { + items: defaultServiceResponse, + }; beforeEach(() => { server.use( rest.get(`${mockBaseUrl}/entities`, (_, res, ctx) => { - return res(ctx.json(defaultResponse)); + return res(ctx.json(defaultServiceResponse)); }), ); }); it('should entities from correct endpoint', async () => { - const entities = await client.getEntities(); - expect(entities).toEqual(defaultResponse); + const response = await client.getEntities(); + expect(response).toEqual(defaultResponse); }); it('builds entity search filters properly', async () => { expect.assertions(2); + server.use( rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D'); @@ -81,13 +85,32 @@ describe('CatalogClient', () => { }), ); - const entities = await client.getEntities({ - a: '1', - b: ['2', '3'], - ö: '=', + const response = await client.getEntities({ + filter: { + a: '1', + b: ['2', '3'], + ö: '=', + }, }); - expect(entities).toEqual([]); + expect(response.items).toEqual([]); + }); + + it('builds entity field selectors properly', async () => { + expect.assertions(2); + + server.use( + rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { + expect(req.url.search).toBe('?fields=a.b,%C3%B6'); + return res(ctx.json([])); + }), + ); + + const response = await client.getEntities({ + fields: ['a.b', 'ö'], + }); + + expect(response.items).toEqual([]); }); }); }); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index e39759a88d..362b1e71da 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -25,6 +25,8 @@ import { AddLocationRequest, AddLocationResponse, CatalogApi, + CatalogEntitiesRequest, + CatalogListResponse, DiscoveryApi, } from './types'; @@ -35,55 +37,33 @@ export class CatalogClient implements CatalogApi { this.discoveryApi = options.discoveryApi; } - private async getRequired(path: string): Promise { - const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; - const response = await fetch(url); - - if (!response.ok) { - const payload = await response.text(); - const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; - throw new Error(message); - } - - return await response.json(); - } - - private async getOptional(path: string): Promise { - const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; - const response = await fetch(url); - - if (!response.ok) { - if (response.status === 404) { - return undefined; - } - - const payload = await response.text(); - const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; - throw new Error(message); - } - - return await response.json(); - } - async getLocationById(id: String): Promise { return await this.getOptional(`/locations/${id}`); } async getEntities( - filter?: Record, - ): Promise { - let path = `/entities`; - if (filter) { - const parts: string[] = []; - for (const [key, value] of Object.entries(filter)) { - for (const v of [value].flat()) { - parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(v)}`); - } + request?: CatalogEntitiesRequest, + ): Promise> { + const { filter = {}, fields = [] } = request ?? {}; + const params: string[] = []; + + const filterParts: string[] = []; + for (const [key, value] of Object.entries(filter)) { + for (const v of [value].flat()) { + filterParts.push(`${encodeURIComponent(key)}=${encodeURIComponent(v)}`); } - path += `?filter=${parts.join(',')}`; + } + if (filterParts.length) { + params.push(`filter=${filterParts.join(',')}`); } - return await this.getRequired(path); + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + + const query = params.length ? `?${params.join('&')}` : ''; + const entities: Entity[] = await this.getRequired(`/entities${query}`); + return { items: entities }; } async getEntityByName(compoundName: EntityName): Promise { @@ -153,4 +133,38 @@ export class CatalogClient implements CatalogApi { } return undefined; } + + // + // Private methods + // + + private async getRequired(path: string): Promise { + const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; + const response = await fetch(url); + + if (!response.ok) { + const payload = await response.text(); + const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; + throw new Error(message); + } + + return await response.json(); + } + + private async getOptional(path: string): Promise { + const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; + const response = await fetch(url); + + if (!response.ok) { + if (response.status === 404) { + return undefined; + } + + const payload = await response.text(); + const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; + throw new Error(message); + } + + return await response.json(); + } } diff --git a/packages/catalog-client/src/types.ts b/packages/catalog-client/src/types.ts index 575b8d6ca2..e72317d444 100644 --- a/packages/catalog-client/src/types.ts +++ b/packages/catalog-client/src/types.ts @@ -16,10 +16,21 @@ import { Entity, EntityName, Location } from '@backstage/catalog-model'; +export type CatalogEntitiesRequest = { + filter?: Record | undefined; + fields?: string[] | undefined; +}; + +export type CatalogListResponse = { + items: T[]; +}; + export interface CatalogApi { getLocationById(id: String): Promise; getEntityByName(name: EntityName): Promise; - getEntities(filter?: Record): Promise; + getEntities( + request?: CatalogEntitiesRequest, + ): Promise>; addLocation(location: AddLocationRequest): Promise; getLocationByEntity(entity: Entity): Promise; removeEntityByUid(uid: string): Promise; diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx index 46cf014134..10495bb957 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx @@ -26,24 +26,26 @@ import { ApiExplorerPage } from './ApiExplorerPage'; describe('ApiCatalogPage', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { - name: 'Entity1', + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { + name: 'Entity1', + }, + spec: { type: 'openapi' }, }, - spec: { type: 'openapi' }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { - name: 'Entity2', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { + name: 'Entity2', + }, + spec: { type: 'openapi' }, }, - spec: { type: 'openapi' }, - }, - ] as Entity[]), + ] as Entity[], + }), getLocationByEntity: () => Promise.resolve({ id: 'id', type: 'github', target: 'url' }), }; diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx index c6a8d62b58..398268caf2 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx @@ -25,8 +25,8 @@ import { ApiExplorerLayout } from './ApiExplorerLayout'; export const ApiExplorerPage = () => { const catalogApi = useApi(catalogApiRef); - const { loading, error, value: matchingEntities } = useAsync(() => { - return catalogApi.getEntities({ kind: 'API' }); + const { loading, error, value: catalogResponse } = useAsync(() => { + return catalogApi.getEntities({ filter: { kind: 'API' } }); }, [catalogApi]); return ( @@ -44,7 +44,7 @@ export const ApiExplorerPage = () => { All your APIs diff --git a/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx b/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx index a4ede7c2f2..b92fc3d4bc 100644 --- a/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx +++ b/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx @@ -33,30 +33,32 @@ import { ButtonGroup, CatalogFilter } from './CatalogFilter'; describe('Catalog Filter', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity1', + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity1', + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'tools@example.com', - type: 'service', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity2', + }, + spec: { + owner: 'not-tools@example.com', + type: 'service', + }, }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity2', - }, - spec: { - owner: 'not-tools@example.com', - type: 'service', - }, - }, - ] as Entity[]), + ] as Entity[], + }), }; const identityApi: Partial = { diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx index 0d645f0f35..83bf4a6f6e 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx @@ -34,37 +34,39 @@ import { CatalogPage } from './CatalogPage'; describe('CatalogPage', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity1', + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity1', + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'tools@example.com', - type: 'service', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity2', + }, + spec: { + owner: 'not-tools@example.com', + type: 'service', + }, }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity2', - }, - spec: { - owner: 'not-tools@example.com', - type: 'service', - }, - }, - ] as Entity[]), + ] as Entity[], + }), getLocationByEntity: () => Promise.resolve({ id: 'id', type: 'github', target: 'url' }), }; const testProfile: Partial = { displayName: 'Display Name', }; - const indentityApi: Partial = { + const identityApi: Partial = { getUserId: () => 'tools@example.com', getProfile: () => testProfile, }; @@ -75,7 +77,7 @@ describe('CatalogPage', () => { diff --git a/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx b/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx index 39837d2ca1..d105979600 100644 --- a/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx +++ b/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx @@ -33,46 +33,48 @@ import { ResultsFilter } from './ResultsFilter'; describe('Results Filter', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity1', - tags: ['java'], + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity1', + tags: ['java'], + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'tools@example.com', - type: 'service', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity2', + }, + spec: { + owner: 'not-tools@example.com', + type: 'service', + }, }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity2', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity3', + tags: ['java', 'test'], + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'not-tools@example.com', - type: 'service', - }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity3', - tags: ['java', 'test'], - }, - spec: { - owner: 'tools@example.com', - type: 'service', - }, - }, - ] as Entity[]), + ] as Entity[], + }), }; - const indentityApi: Partial = { + const identityApi: Partial = { getUserId: () => 'tools@example.com', }; @@ -82,7 +84,7 @@ describe('Results Filter', () => { diff --git a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx b/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx index 218cc45e78..96b6880bb9 100644 --- a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx +++ b/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx @@ -44,9 +44,13 @@ function useColocatedEntities(entity: Entity): AsyncState { const catalogApi = useApi(catalogApiRef); return useAsync(async () => { const myLocation = entity.metadata.annotations?.[LOCATION_ANNOTATION]; - return myLocation - ? await catalogApi.getEntities({ [LOCATION_ANNOTATION]: myLocation }) - : []; + if (!myLocation) { + return []; + } + const response = await catalogApi.getEntities({ + filter: { [LOCATION_ANNOTATION]: myLocation }, + }); + return response.items; }, [catalogApi, entity]); } diff --git a/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx b/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx index 36cc0c56df..edda8eecb5 100644 --- a/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx +++ b/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx @@ -46,9 +46,12 @@ export const EntityFilterGroupsProvider = ({ // The hook that implements the actual context building function useProvideEntityFilters(): FilterGroupsContext { const catalogApi = useApi(catalogApiRef); - const [{ value: entities, error }, doReload] = useAsyncFn(() => - catalogApi.getEntities({ kind: 'Component' }), - ); + const [{ value: entities, error }, doReload] = useAsyncFn(async () => { + const response = await catalogApi.getEntities({ + filter: { kind: 'Component' }, + }); + return response.items; + }); const filterGroups = useRef<{ [filterGroupId: string]: FilterGroup; diff --git a/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx b/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx index 6b573e3721..e9af063160 100644 --- a/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx +++ b/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx @@ -49,7 +49,7 @@ describe('useEntityFilterGroup', () => { }); it('works for an empty set of filters', async () => { - catalogApi.getEntities.mockResolvedValue([]); + catalogApi.getEntities.mockResolvedValue({ items: [] }); const group: FilterGroup = { filters: {} }; const { result, waitFor } = renderHook( () => useEntityFilterGroup('g1', group), @@ -60,13 +60,15 @@ describe('useEntityFilterGroup', () => { }); it('works for a single group', async () => { - catalogApi.getEntities.mockResolvedValue([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { name: 'n' }, - }, - ]); + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'n' }, + }, + ], + }); const group: FilterGroup = { filters: { f1: e => e.metadata.name === 'n', diff --git a/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx b/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx index c6ee052c0e..ff3e909bf7 100644 --- a/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx +++ b/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx @@ -52,7 +52,7 @@ describe('RollbarHome component', () => { catalogApiRef, ({ async getEntities() { - return []; + return { items: [] }; }, } as Partial) as CatalogApi, ], diff --git a/plugins/rollbar/src/hooks/useRollbarEntities.ts b/plugins/rollbar/src/hooks/useRollbarEntities.ts index 3edf34d1f3..5979c5832a 100644 --- a/plugins/rollbar/src/hooks/useRollbarEntities.ts +++ b/plugins/rollbar/src/hooks/useRollbarEntities.ts @@ -28,8 +28,8 @@ export function useRollbarEntities() { configApi.getString('organization.name'); const { value, loading, error } = useAsync(async () => { - const entities = await catalogApi.getEntities(); - return entities.filter(entity => { + const response = await catalogApi.getEntities(); + return response.items.filter(entity => { return !!entity.metadata.annotations?.[ROLLBAR_ANNOTATION]; }); }, [catalogApi]); diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index d0cb25f254..abdb411ad9 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -53,10 +53,12 @@ export const ScaffolderPage = () => { const { data: templates, isValidating, error } = useStaleWhileRevalidate( 'templates/all', - async () => - catalogApi.getEntities({ kind: 'Template' }) as Promise< - TemplateEntityV1alpha1[] - >, + async () => { + const response = await catalogApi.getEntities({ + filter: { kind: 'Template' }, + }); + return response.items as TemplateEntityV1alpha1[]; + }, ); useEffect(() => { diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx index 6229a88fc3..62963b58b8 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx @@ -13,18 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React from 'react'; -import { TemplatePage } from './TemplatePage'; -import { wrapInTestApp, renderWithEffects } from '@backstage/test-utils'; -import { ApiRegistry, errorApiRef, ApiProvider } from '@backstage/core'; -import { scaffolderApiRef, ScaffolderApi } from '../../api'; -import { catalogApiRef, CatalogApi } from '@backstage/plugin-catalog'; -import { mutate } from 'swr'; -import { act } from 'react-dom/test-utils'; -import { Route, MemoryRouter } from 'react-router'; -import { rootRoute } from '../../routes'; -import { ThemeProvider } from '@material-ui/core'; +import { ApiProvider, ApiRegistry, errorApiRef } from '@backstage/core'; +import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog'; +import { renderInTestApp, renderWithEffects } from '@backstage/test-utils'; import { lightTheme } from '@backstage/theme'; +import { ThemeProvider } from '@material-ui/core'; +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { MemoryRouter, Route } from 'react-router'; +import { ScaffolderApi, scaffolderApiRef } from '../../api'; +import { rootRoute } from '../../routes'; +import { TemplatePage } from './TemplatePage'; const templateMock = { apiVersion: 'backstage.io/v1alpha1', @@ -90,48 +89,43 @@ const apis = ApiRegistry.from([ ]); describe('TemplatePage', () => { - afterEach(async () => { - // Cleaning up swr's cache - await act(async () => { - await mutate('templates/test'); - }); - }); + beforeEach(() => jest.resetAllMocks()); + it('renders correctly', async () => { - catalogApiMock.getEntities.mockResolvedValueOnce([templateMock]); - const rendered = await renderWithEffects( - wrapInTestApp( - - - , - ), + catalogApiMock.getEntities.mockResolvedValueOnce({ items: [templateMock] }); + const rendered = await renderInTestApp( + + + , ); expect(rendered.queryByText('Create a new component')).toBeInTheDocument(); expect(rendered.queryByText('React SSR Template')).toBeInTheDocument(); // await act(async () => await mutate('templates/test')); }); + it('renders spinner while loading', async () => { let resolve: Function; const promise = new Promise(res => { resolve = res; }); - catalogApiMock.getEntities.mockResolvedValueOnce(promise); - const rendered = await renderWithEffects( - wrapInTestApp( - - - , - ), + catalogApiMock.getEntities.mockReturnValueOnce(promise); + const rendered = await renderInTestApp( + + + , ); expect(rendered.queryByText('Create a new component')).toBeInTheDocument(); expect(rendered.queryByTestId('loading-progress')).toBeInTheDocument(); // Need to cleanup the promise or will timeout - resolve!(); + act(() => { + resolve!({ items: [] }); + }); }); it('navigates away if no template was loaded', async () => { - catalogApiMock.getEntities.mockResolvedValueOnce([]); + catalogApiMock.getEntities.mockResolvedValueOnce({ items: [] }); const rendered = await renderWithEffects( diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index cb6b38903e..4836cdab56 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -27,28 +27,26 @@ import { catalogApiRef } from '@backstage/plugin-catalog'; import { LinearProgress } from '@material-ui/core'; import { IChangeEvent } from '@rjsf/core'; import React, { useState } from 'react'; -import { useParams } from 'react-router-dom'; -import useStaleWhileRevalidate from 'swr'; -import { scaffolderApiRef } from '../../api'; -import { JobStatusModal } from '../JobStatusModal'; -import { Job } from '../../types'; -import { MultistepJsonForm } from '../MultistepJsonForm'; import { Navigate } from 'react-router'; +import { useParams } from 'react-router-dom'; +import { useAsync } from 'react-use'; +import { scaffolderApiRef } from '../../api'; import { rootRoute } from '../../routes'; +import { Job } from '../../types'; +import { JobStatusModal } from '../JobStatusModal'; +import { MultistepJsonForm } from '../MultistepJsonForm'; const useTemplate = ( templateName: string, catalogApi: typeof catalogApiRef.T, ) => { - const { data, error } = useStaleWhileRevalidate( - `templates/${templateName}`, - async () => - catalogApi.getEntities({ - kind: 'Template', - 'metadata.name': templateName, - }) as Promise, - ); - return { template: data?.[0], loading: !error && !data, error }; + const { value, loading, error } = useAsync(async () => { + const response = await catalogApi.getEntities({ + filter: { kind: 'Template', 'metadata.name': templateName }, + }); + return response.items as TemplateEntityV1alpha1[]; + }); + return { template: value?.[0], loading, error }; }; const OWNER_REPO_SCHEMA = { @@ -110,7 +108,7 @@ export const TemplatePage = () => { const handleCreateComplete = async (job: Job) => { const target = job.metadata.remoteUrl?.replace( /\.git$/, - // TODO(Rugvip): This is not the location we want. As part of scaffodler v2 we + // TODO(Rugvip): This is not the location we want. As part of scaffolder v2 we // want this to be more flexible, but before that we might want // to update all templates to use catalog-info.yaml instead. '/blob/master/component-info.yaml', diff --git a/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx b/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx index d5d0e19e08..0924b90fc1 100644 --- a/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; import { ApiProvider, ApiRegistry } from '@backstage/core-api'; import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog'; import { wrapInTestApp } from '@backstage/test-utils'; @@ -24,7 +23,7 @@ import { TechDocsHome } from './TechDocsHome'; describe('TechDocs Home', () => { const catalogApi: Partial = { - getEntities: () => Promise.resolve([] as Entity[]), + getEntities: () => Promise.resolve({ items: [] }), }; const apiRegistry = ApiRegistry.from([[catalogApiRef, catalogApi]]); diff --git a/plugins/techdocs/src/reader/components/TechDocsHome.tsx b/plugins/techdocs/src/reader/components/TechDocsHome.tsx index 0408a540ca..9f569dd167 100644 --- a/plugins/techdocs/src/reader/components/TechDocsHome.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsHome.tsx @@ -14,19 +14,19 @@ * limitations under the License. */ -import React from 'react'; -import { useAsync } from 'react-use'; -import { useNavigate, generatePath } from 'react-router-dom'; -import { Grid } from '@material-ui/core'; import { + Content, + Header, ItemCard, + Page, Progress, useApi, - Content, - Page, - Header, } from '@backstage/core'; import { catalogApiRef } from '@backstage/plugin-catalog'; +import { Grid } from '@material-ui/core'; +import React from 'react'; +import { generatePath, useNavigate } from 'react-router-dom'; +import { useAsync } from 'react-use'; import { rootDocsRouteRef } from '../../plugin'; export const TechDocsHome = () => { @@ -34,8 +34,8 @@ export const TechDocsHome = () => { const navigate = useNavigate(); const { value, loading, error } = useAsync(async () => { - const entities = await catalogApi.getEntities(); - return entities.filter(entity => { + const response = await catalogApi.getEntities(); + return response.items.filter(entity => { return !!entity.metadata.annotations?.['backstage.io/techdocs-ref']; }); }); From f8cb1a976d82e8e54cd70dd31dc0957506f092b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 18 Nov 2020 20:09:38 +0100 Subject: [PATCH 141/259] fix: unbreak search after merging catalog client change --- plugins/search/src/apis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index 1783b91d2a..f3a050b2a2 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -35,7 +35,7 @@ class SearchApi { private async entities() { const entities = await this.catalogApi.getEntities(); - return entities.map((result: any) => ({ + return entities.items.map((result: any) => ({ name: result.metadata.name, description: result.metadata.description, owner: result.spec.owner, From 66daa47c6a1265e8ff052d9dbcf17f9bb0893650 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 18 Nov 2020 20:22:09 +0100 Subject: [PATCH 142/259] 1. Remove DiscoveryApi (frontend) and use PluginEndpointDiscovery(SingleHostDiscovery) (backend) 2. Mock class for testing --- plugins/techdocs-backend/package.json | 1 - .../src/techdocs/stages/publish/local.test.ts | 25 +++++++------------ .../src/techdocs/stages/publish/local.ts | 14 ++++++----- yarn.lock | 16 ++++++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 236f53b9d1..c1d1f6ad27 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -23,7 +23,6 @@ "@backstage/backend-common": "^0.2.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", - "@backstage/core-api": "^0.2.1", "@types/dockerode": "^2.5.34", "@types/express": "^4.17.6", "command-exists-promise": "^2.0.2", diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts index 4b6503f620..18344489e1 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts @@ -17,8 +17,10 @@ /* eslint-disable no-restricted-syntax */ import fs from 'fs-extra'; import path from 'path'; -import { getVoidLogger, SingleHostDiscovery } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; +import { + getVoidLogger, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; import { LocalPublish } from './local'; const createMockEntity = (annotations = {}) => { @@ -38,20 +40,11 @@ const logger = getVoidLogger(); describe('local publisher', () => { it('should publish generated documentation dir', async () => { - const testConfig = ConfigReader.fromConfigs([ - { - context: '', - data: { - backend: { - baseUrl: 'http://localhost:7000', - listen: { - port: 7000, - }, - }, - }, - }, - ]); - const testDiscovery = SingleHostDiscovery.fromConfig(testConfig); + const testDiscovery: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValueOnce('http://localhost:7000'), + getExternalBaseUrl: jest.fn(), + }; + const publisher = new LocalPublish(logger, testDiscovery); const mockEntity = createMockEntity(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts index 9f066a919c..e13c66b7f0 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts @@ -17,16 +17,18 @@ import fs from 'fs-extra'; import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; import { PublisherBase } from './types'; -import { resolvePackagePath } from '@backstage/backend-common'; -import { DiscoveryApi } from '@backstage/core-api'; +import { + resolvePackagePath, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; export class LocalPublish implements PublisherBase { private readonly logger: Logger; - private readonly discoveryApi: DiscoveryApi; + private readonly discovery: PluginEndpointDiscovery; - constructor(logger: Logger, discoveryApi: DiscoveryApi) { + constructor(logger: Logger, discovery: PluginEndpointDiscovery) { this.logger = logger; - this.discoveryApi = discoveryApi; + this.discovery = discovery; } publish({ @@ -66,7 +68,7 @@ export class LocalPublish implements PublisherBase { reject(err); } - this.discoveryApi.getBaseUrl('techdocs').then(techdocsApiUrl => { + this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { resolve({ remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, }); diff --git a/yarn.lock b/yarn.lock index 9d4984e152..10e70f52fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1290,34 +1290,40 @@ to-fast-properties "^2.0.0" "@backstage/core@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@backstage/core/-/core-0.2.0.tgz#543246b2d87563c9aa4d9fb96e40fdfc7e827520" - integrity sha512-75m2u3FoUngBOvt9l65xZcYTzzB+49OXpY1A9VNFUR1+jMs3cL/0HDfByQV2H0xXaHzMngQ8C5u/sWhkQsij1w== + version "0.3.0" dependencies: "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.0" - "@backstage/theme" "^0.2.0" + "@backstage/core-api" "^0.2.1" + "@backstage/theme" "^0.2.1" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" + "@types/dagre" "^0.7.44" "@types/react" "^16.9" "@types/react-sparklines" "^1.7.0" classnames "^2.2.6" clsx "^1.1.0" + d3-selection "^2.0.0" + d3-shape "^2.0.0" + d3-zoom "^2.0.0" + dagre "^0.8.5" immer "^7.0.9" lodash "^4.17.15" material-table "^1.69.1" prop-types "^15.7.2" + qs "^6.9.4" rc-progress "^3.0.0" react "^16.12.0" react-dom "^16.12.0" react-helmet "6.1.0" react-hook-form "^6.6.0" + react-markdown "^5.0.2" react-router "6.0.0-beta.0" react-router-dom "6.0.0-beta.0" react-sparklines "^1.7.0" react-syntax-highlighter "^13.5.1" react-use "^15.3.3" + remark-gfm "^1.0.0" "@bcoe/v8-coverage@^0.2.3": version "0.2.3" From f8877545c54cf3852f5f414fffe687c9fbe464fd Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 18 Nov 2020 20:35:22 +0100 Subject: [PATCH 143/259] TechDocs: Reject promise if call to getBaseUrl fails --- .../src/techdocs/stages/publish/local.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts index e13c66b7f0..07cef2f4d2 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts @@ -68,11 +68,16 @@ export class LocalPublish implements PublisherBase { reject(err); } - this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { - resolve({ - remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + this.discovery + .getBaseUrl('techdocs') + .then(techdocsApiUrl => { + resolve({ + remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + }); + }) + .catch(reason => { + reject(reason); }); - }); }); }); } From 9d792a435b09a38cc4a4ea6437f7ac5219af50c4 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Wed, 18 Nov 2020 15:05:21 -0500 Subject: [PATCH 144/259] pretty print subdollar values for sku costs --- .../components/ProductInsightsCard/ProductEntityDialog.tsx | 4 ++-- plugins/cost-insights/src/utils/formatters.ts | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx index 764a18d305..dfa33d15f6 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx @@ -20,7 +20,7 @@ import { Table, TableColumn } from '@backstage/core'; import { Dialog, IconButton, Typography } from '@material-ui/core'; import { default as CloseButton } from '@material-ui/icons/Close'; import { CostGrowthIndicator } from '../CostGrowth'; -import { currencyFormatter, formatPercent } from '../../utils/formatters'; +import { costFormatter, formatPercent } from '../../utils/formatters'; import { useEntityDialogStyles as useStyles } from '../../utils/styles'; import { BarChartOptions, Entity } from '../../types'; @@ -38,7 +38,7 @@ function createRenderer(col: keyof RowData, classes: Record) { case 'current': return ( - {currencyFormatter.format(row[col])} + {costFormatter.format(row[col])} ); case 'ratio': diff --git a/plugins/cost-insights/src/utils/formatters.ts b/plugins/cost-insights/src/utils/formatters.ts index e81fc1f7c9..42505e7072 100644 --- a/plugins/cost-insights/src/utils/formatters.ts +++ b/plugins/cost-insights/src/utils/formatters.ts @@ -24,6 +24,11 @@ export type Period = { periodEnd: string; }; +export const costFormatter = new Intl.NumberFormat('en-US', { + style: 'currency', + currency: 'USD', +}); + export const currencyFormatter = new Intl.NumberFormat('en-US', { style: 'currency', currency: 'USD', From 4a25ee5cf546525dbbc4646b0aec78b5cd15349e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 18 Nov 2020 21:10:04 +0100 Subject: [PATCH 145/259] auth-backend: use the catalog client (#3298) --- plugins/auth-backend/package.json | 1 + .../lib/catalog/CatalogIdentityClient.test.ts | 48 +++++++++++++++++++ .../src/lib/catalog/CatalogIdentityClient.ts | 42 +++++----------- .../src/providers/google/provider.ts | 34 ++++++------- plugins/auth-backend/src/providers/types.ts | 6 ++- plugins/auth-backend/src/service/router.ts | 19 ++++---- .../src/service/standaloneServer.ts | 10 ++-- 7 files changed, 99 insertions(+), 61 deletions(-) create mode 100644 plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index af3d056b28..28ac51034e 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.2.1", + "@backstage/catalog-client": "^0.2.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts new file mode 100644 index 0000000000..ec6aa1b6ba --- /dev/null +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogApi } from '@backstage/catalog-client'; +import { UserEntity } from '@backstage/catalog-model'; +import { CatalogIdentityClient } from './CatalogIdentityClient'; + +describe('CatalogIdentityClient', () => { + const catalogApi: jest.Mocked = { + getLocationById: jest.fn(), + getEntityByName: jest.fn(), + getEntities: jest.fn(), + addLocation: jest.fn(), + getLocationByEntity: jest.fn(), + removeEntityByUid: jest.fn(), + }; + + afterEach(() => jest.resetAllMocks()); + + it('passes through the correct search params', async () => { + catalogApi.getEntities.mockResolvedValueOnce({ items: [{} as UserEntity] }); + const client = new CatalogIdentityClient({ + catalogApi: catalogApi as CatalogApi, + }); + + client.findUser({ annotations: { key: 'value' } }); + + expect(catalogApi.getEntities).toBeCalledWith({ + filter: { + kind: 'user', + 'metadata.annotations.key': 'value', + }, + }); + }); +}); diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts index e56763f656..5bc4e5de21 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts @@ -14,13 +14,9 @@ * limitations under the License. */ -import { - ConflictError, - NotFoundError, - PluginEndpointDiscovery, -} from '@backstage/backend-common'; +import { ConflictError, NotFoundError } from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; import { UserEntity } from '@backstage/catalog-model'; -import fetch from 'cross-fetch'; type UserQuery = { annotations: Record; @@ -30,10 +26,10 @@ type UserQuery = { * A catalog client tailored for reading out identity data from the catalog. */ export class CatalogIdentityClient { - private readonly discovery: PluginEndpointDiscovery; + private readonly catalogApi: CatalogApi; - constructor(options: { discovery: PluginEndpointDiscovery }) { - this.discovery = options.discovery; + constructor(options: { catalogApi: CatalogApi }) { + this.catalogApi = options.catalogApi; } /** @@ -42,35 +38,23 @@ export class CatalogIdentityClient { * Throws a NotFoundError or ConflictError if 0 or multiple users are found. */ async findUser(query: UserQuery): Promise { - const conditions = ['kind=user']; + const filter: Record = { + kind: 'user', + }; for (const [key, value] of Object.entries(query.annotations)) { - const uk = encodeURIComponent(key); - const uv = encodeURIComponent(value); - conditions.push(`metadata.annotations.${uk}=${uv}`); + filter[`metadata.annotations.${key}`] = value; } - const baseUrl = await this.discovery.getBaseUrl('catalog'); - const response = await fetch( - `${baseUrl}/entities?filter=${conditions.join(',')}`, - ); + const { items } = await this.catalogApi.getEntities({ filter }); - if (!response.ok) { - const text = await response.text(); - throw new Error( - `Request failed with ${response.status} ${response.statusText}, ${text}`, - ); - } - - const users: UserEntity[] = await response.json(); - - if (users.length !== 1) { - if (users.length > 1) { + if (items.length !== 1) { + if (items.length > 1) { throw new ConflictError('User lookup resulted in multiple matches'); } else { throw new NotFoundError('User not found'); } } - return users[0]; + return items[0] as UserEntity; } } diff --git a/plugins/auth-backend/src/providers/google/provider.ts b/plugins/auth-backend/src/providers/google/provider.ts index 80143ca201..dd5b51eb01 100644 --- a/plugins/auth-backend/src/providers/google/provider.ts +++ b/plugins/auth-backend/src/providers/google/provider.ts @@ -15,29 +15,29 @@ */ import express from 'express'; -import { Logger } from 'winston'; +import passport from 'passport'; import { Strategy as GoogleStrategy } from 'passport-google-oauth20'; +import { Logger } from 'winston'; +import { CatalogIdentityClient } from '../../lib/catalog'; import { + encodeState, + OAuthAdapter, + OAuthEnvironmentHandler, + OAuthHandlers, + OAuthProviderOptions, + OAuthRefreshRequest, + OAuthResponse, + OAuthStartRequest, +} from '../../lib/oauth'; +import { + executeFetchUserProfileStrategy, executeFrameHandlerStrategy, executeRedirectStrategy, executeRefreshTokenStrategy, makeProfileInfo, - executeFetchUserProfileStrategy, PassportDoneCallback, } from '../../lib/passport'; -import { RedirectInfo, AuthProviderFactory } from '../types'; -import { - OAuthAdapter, - OAuthHandlers, - OAuthProviderOptions, - OAuthResponse, - OAuthEnvironmentHandler, - OAuthStartRequest, - encodeState, - OAuthRefreshRequest, -} from '../../lib/oauth'; -import passport from 'passport'; -import { CatalogIdentityClient } from '../../lib/catalog'; +import { AuthProviderFactory, RedirectInfo } from '../types'; type PrivateInfo = { refreshToken: string; @@ -179,7 +179,7 @@ export const createGoogleProvider: AuthProviderFactory = ({ config, logger, tokenIssuer, - discovery, + catalogApi, }) => OAuthEnvironmentHandler.mapConfig(config, envConfig => { const providerId = 'google'; @@ -192,7 +192,7 @@ export const createGoogleProvider: AuthProviderFactory = ({ clientSecret, callbackUrl, logger, - identityClient: new CatalogIdentityClient({ discovery }), + identityClient: new CatalogIdentityClient({ catalogApi }), }); return OAuthAdapter.fromConfig(globalConfig, provider, { diff --git a/plugins/auth-backend/src/providers/types.ts b/plugins/auth-backend/src/providers/types.ts index 6776095f99..0e500e027e 100644 --- a/plugins/auth-backend/src/providers/types.ts +++ b/plugins/auth-backend/src/providers/types.ts @@ -14,11 +14,12 @@ * limitations under the License. */ +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; +import { Config } from '@backstage/config'; import express from 'express'; import { Logger } from 'winston'; import { TokenIssuer } from '../identity'; -import { Config } from '@backstage/config'; -import { PluginEndpointDiscovery } from '@backstage/backend-common'; export type AuthProviderConfig = { /** @@ -117,6 +118,7 @@ export type AuthProviderFactoryOptions = { logger: Logger; tokenIssuer: TokenIssuer; discovery: PluginEndpointDiscovery; + catalogApi: CatalogApi; }; export type AuthProviderFactory = ( diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 6bb100de6b..0d19e67fc4 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -14,18 +14,19 @@ * limitations under the License. */ -import express from 'express'; -import Router from 'express-promise-router'; -import cookieParser from 'cookie-parser'; -import { Logger } from 'winston'; -import { createAuthProvider } from '../providers'; -import { Config } from '@backstage/config'; -import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity'; import { NotFoundError, - PluginEndpointDiscovery, PluginDatabaseManager, + PluginEndpointDiscovery, } from '@backstage/backend-common'; +import { CatalogClient } from '@backstage/catalog-client'; +import { Config } from '@backstage/config'; +import cookieParser from 'cookie-parser'; +import express from 'express'; +import Router from 'express-promise-router'; +import { Logger } from 'winston'; +import { createOidcRouter, DatabaseKeyStore, TokenFactory } from '../identity'; +import { createAuthProvider } from '../providers'; export interface RouterOptions { logger: Logger; @@ -56,6 +57,7 @@ export async function createRouter({ keyDurationSeconds, logger: logger.child({ component: 'token-factory' }), }); + const catalogApi = new CatalogClient({ discoveryApi: discovery }); router.use(cookieParser()); router.use(express.urlencoded({ extended: false })); @@ -73,6 +75,7 @@ export async function createRouter({ logger, tokenIssuer, discovery, + catalogApi, }); const r = Router(); diff --git a/plugins/auth-backend/src/service/standaloneServer.ts b/plugins/auth-backend/src/service/standaloneServer.ts index a4f2377507..15a3347d5f 100644 --- a/plugins/auth-backend/src/service/standaloneServer.ts +++ b/plugins/auth-backend/src/service/standaloneServer.ts @@ -14,16 +14,16 @@ * limitations under the License. */ -import Knex from 'knex'; -import { Server } from 'http'; -import { Logger } from 'winston'; -import { createRouter } from './router'; import { createServiceBuilder, - useHotMemoize, loadBackendConfig, SingleHostDiscovery, + useHotMemoize, } from '@backstage/backend-common'; +import { Server } from 'http'; +import Knex from 'knex'; +import { Logger } from 'winston'; +import { createRouter } from './router'; export interface ServerOptions { logger: Logger; From 86017bc2fbc5d34f540b6530a864ba699a897ab9 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 18 Nov 2020 21:19:06 +0100 Subject: [PATCH 146/259] docs: Add catalog-client to project structure --- docs/support/project-structure.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/support/project-structure.md b/docs/support/project-structure.md index 034fba4cb8..deb8c47777 100644 --- a/docs/support/project-structure.md +++ b/docs/support/project-structure.md @@ -89,6 +89,10 @@ are separated out into their own folder, see further down. There are no "core" packages in the backend. Instead we have `backend-common` which contains helper middleware and other utils. +- [`catalog-client`](https://github.com/backstage/backstage/tree/master/packages/catalog-client) - + A client to communicate with Backstage software catalog. Recommended for backend plugins, compatible + but not recommended for frontend plugins (use `@backstage/plugin-catalog` package instead). + - [`catalog-model/`](https://github.com/backstage/backstage/tree/master/packages/catalog-model) - You can consider this to be a library for working with the catalog of sorts. It contains the definition of an From c645ea174ac2e1a624c90e51bca6f57da3de9635 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Wed, 18 Nov 2020 15:21:09 -0500 Subject: [PATCH 147/259] widen first column of entity dialog table --- .../src/components/ProductInsightsCard/ProductEntityDialog.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx b/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx index dfa33d15f6..4cccd638cb 100644 --- a/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx +++ b/plugins/cost-insights/src/components/ProductInsightsCard/ProductEntityDialog.tsx @@ -122,6 +122,7 @@ export const ProductEntityDialog = ({ title: SKU, render: createRenderer('sku', classes), customSort: createSorter('sku'), + width: '33.33%', }, { field: 'previous', From f360395d00eaf6d0c9318fa1ed9e39d391d024f7 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Wed, 18 Nov 2020 15:41:46 -0500 Subject: [PATCH 148/259] changeset --- .changeset/cost-insights-wet-plants-glow.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/cost-insights-wet-plants-glow.md diff --git a/.changeset/cost-insights-wet-plants-glow.md b/.changeset/cost-insights-wet-plants-glow.md new file mode 100644 index 0000000000..3d2be289d4 --- /dev/null +++ b/.changeset/cost-insights-wet-plants-glow.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +UI improvements: Increase width of first column in product entity dialog table +UI improvement: Display full cost amount in product entity dialog table From 4787cd04deaadcc531ce27b7ce686b3940bb4b47 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 18 Nov 2020 22:53:19 +0100 Subject: [PATCH 149/259] docs: Rephrase catalog-client and run prettier --- docs/support/project-structure.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/support/project-structure.md b/docs/support/project-structure.md index deb8c47777..c25e087e57 100644 --- a/docs/support/project-structure.md +++ b/docs/support/project-structure.md @@ -90,8 +90,10 @@ are separated out into their own folder, see further down. which contains helper middleware and other utils. - [`catalog-client`](https://github.com/backstage/backstage/tree/master/packages/catalog-client) - - A client to communicate with Backstage software catalog. Recommended for backend plugins, compatible - but not recommended for frontend plugins (use `@backstage/plugin-catalog` package instead). + An isomorphic client to interact with the Software Catalog. Backend plugins + can use the package directly. Frontend plugins can use the client by using + `@backstage/plugin-catalog` in combination with `useApi` and the + `catalogApiRef`. - [`catalog-model/`](https://github.com/backstage/backstage/tree/master/packages/catalog-model) - You can consider this to be a library for working with the catalog of sorts. From 7d7abd50c8befe18f86dab1ba5cb6982539ca5aa Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 19:45:32 +0100 Subject: [PATCH 150/259] create-app: add app-backend plugin to template --- .changeset/add-app-backend.md | 104 ++++++++++++++++++ packages/create-app/package.json | 1 + packages/create-app/src/lib/versions.ts | 2 + .../default-app/app-config.production.yaml | 8 ++ .../templates/default-app/package.json.hbs | 1 + .../default-app/packages/backend/Dockerfile | 2 +- .../packages/backend/package.json.hbs | 4 +- .../default-app/packages/backend/src/index.ts | 5 +- .../packages/backend/src/plugins/app.ts | 13 +++ 9 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 .changeset/add-app-backend.md create mode 100644 packages/create-app/templates/default-app/app-config.production.yaml create mode 100644 packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts diff --git a/.changeset/add-app-backend.md b/.changeset/add-app-backend.md new file mode 100644 index 0000000000..e44a56c885 --- /dev/null +++ b/.changeset/add-app-backend.md @@ -0,0 +1,104 @@ +--- +'@backstage/create-app': patch +--- + +Add `app-backend` as a backend plugin, and make a single docker build of the backend the default way to deploy backstage. + +Note that the `app-backend` currently only is a solution for deployments of the app, it's not a dev server and is not intended for local development. + +## Template changes + +As a part of installing the `app-backend` plugin, the below changes where made. The changes are grouped into two steps, installing the plugin, and updating the Docker build and configuration. + +### Installing the `app-backend` plugin in the backend + +First, install the `@backstage/plugin-app-backend` plugin package in your backend. These changes where made for `v0.3.0` of the plugin, and the installation process might change in the future. Run the following from the root of the repo: + +```bash +cd packages/backend +yarn add @backstage/plugin-app-backend +``` + +For the `app-backend` to get access to the static content in the frontend we also need to add the local `app` package as a dependency. Add the following to your `"dependencies"` in `packages/backend/package.json`, assuming your app package is still named `app` and on version `0.0.0`: + +```json +"app": "0.0.0", +``` + +Don't worry, this will not cause your entire frontend dependency tree to be added to the app, just double check that `packages/app/package.json` has a `"bundled": true` field at top-level. This signals to the backend build process that the package is bundled and that no transitive dependencies should be included. + +Next, create `packages/backend/src/plugins/app.ts` with the following: + +```ts +import { createRouter } from '@backstage/plugin-app-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + return await createRouter({ + logger, + config, + appPackageName: 'app', + }); +} +``` + +In `packages/backend/src/index.ts`, make the following changes: + +Add an import for the newly created plugin setup file: + +```ts +import app from './plugins/app'; +``` + +Setup the following plugin env. + +```ts +const appEnv = useHotMemoize(module, () => createEnv('app')); +``` + +Change service builder setup to include the `app` plugin as follows. Note that the `app` plugin is not installed on the `/api` route with most other plugins. + +```ts +const service = createServiceBuilder(module) + .loadConfig(config) + .addRouter('/api', apiRouter) + .addRouter('', await app(appEnv)); +``` + +You should now have the `app-backend` plugin installed in your backend, ready to serve the frontend bundle! + +### Docker build setup + +Since the backend image is now the only one needed for a simple Backstage deployment, the image tag name in the `build-image` script inside `packages/backend/package.json` was changed to the following: + +```json +"build-image": "backstage-cli backend:build-image --build --tag backstage", +``` + +For convenience, a `build-image` script was also added to the root `package.json` with the following: + +```json +"build-image": "yarn workspace backend build-image", +``` + +In the root of the repo, a new `app-config.production.yaml` file was added. This is used to set the appropriate `app.baseUrl` now that the frontend is served directly by the backend in the production deployment. It has the following contents: + +```yaml +app: + # Should be the same as backend.baseUrl when using the `app-backend` plugin + baseUrl: http://localhost:7000 + +backend: + baseUrl: http://localhost:7000 + listen: + port: 7000 +``` + +In order to load in the new configuration at runtime, the command in the `Dockerfile` at the repo root was changed to the following: + +```dockerfile +CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] +``` diff --git a/packages/create-app/package.json b/packages/create-app/package.json index bd1d8d1bd1..55cdcb1a2a 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -43,6 +43,7 @@ "@backstage/config": "^0.1.1", "@backstage/core": "^0.3.0", "@backstage/plugin-api-docs": "^0.2.1", + "@backstage/plugin-app-backend": "^0.2.0", "@backstage/plugin-auth-backend": "^0.2.1", "@backstage/plugin-catalog": "^0.2.1", "@backstage/plugin-catalog-backend": "^0.2.0", diff --git a/packages/create-app/src/lib/versions.ts b/packages/create-app/src/lib/versions.ts index 196e0004c1..84f015509c 100644 --- a/packages/create-app/src/lib/versions.ts +++ b/packages/create-app/src/lib/versions.ts @@ -35,6 +35,7 @@ import { version as cli } from '@backstage/cli/package.json'; import { version as config } from '@backstage/config/package.json'; import { version as core } from '@backstage/core/package.json'; import { version as pluginApiDocs } from '@backstage/plugin-api-docs/package.json'; +import { version as pluginAppBackend } from '@backstage/plugin-app-backend/package.json'; import { version as pluginAuthBackend } from '@backstage/plugin-auth-backend/package.json'; import { version as pluginCatalog } from '@backstage/plugin-catalog/package.json'; import { version as pluginCatalogBackend } from '@backstage/plugin-catalog-backend/package.json'; @@ -61,6 +62,7 @@ export const packageVersions = { '@backstage/config': config, '@backstage/core': core, '@backstage/plugin-api-docs': pluginApiDocs, + '@backstage/plugin-app-backend': pluginAppBackend, '@backstage/plugin-auth-backend': pluginAuthBackend, '@backstage/plugin-catalog': pluginCatalog, '@backstage/plugin-catalog-backend': pluginCatalogBackend, diff --git a/packages/create-app/templates/default-app/app-config.production.yaml b/packages/create-app/templates/default-app/app-config.production.yaml new file mode 100644 index 0000000000..92f4574fd1 --- /dev/null +++ b/packages/create-app/templates/default-app/app-config.production.yaml @@ -0,0 +1,8 @@ +app: + # Should be the same as backend.baseUrl when using the `app-backend` plugin + baseUrl: http://localhost:7000 + +backend: + baseUrl: http://localhost:7000 + listen: + port: 7000 diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index b9fb0f3744..0116287a46 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -8,6 +8,7 @@ "scripts": { "start": "yarn workspace app start", "build": "lerna run build", + "build-image": "yarn workspace backend build-image", "tsc": "tsc", "tsc:full": "tsc --skipLibCheck false --incremental false", "clean": "backstage-cli clean && lerna run clean", diff --git a/packages/create-app/templates/default-app/packages/backend/Dockerfile b/packages/create-app/templates/default-app/packages/backend/Dockerfile index 93929ceb86..0fcd23f0e1 100644 --- a/packages/create-app/templates/default-app/packages/backend/Dockerfile +++ b/packages/create-app/templates/default-app/packages/backend/Dockerfile @@ -13,4 +13,4 @@ RUN yarn install --frozen-lockfile --production # Do not use this Dockerfile outside of that command, as it will copy in the source code instead. COPY . . -CMD ["node", "packages/backend"] +CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index df9ac28739..179d604670 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -9,7 +9,7 @@ }, "scripts": { "build": "backstage-cli backend:build", - "build-image": "backstage-cli backend:build-image --build --tag example-backend", + "build-image": "backstage-cli backend:build-image --build --tag backstage", "start": "backstage-cli backend:dev", "lint": "backstage-cli lint", "test": "backstage-cli test", @@ -17,9 +17,11 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { + "app": "0.0.0", "@backstage/backend-common": "^{{version '@backstage/backend-common'}}", "@backstage/catalog-model": "^{{version '@backstage/catalog-model'}}", "@backstage/config": "^{{version '@backstage/config'}}", + "@backstage/plugin-app-backend": "^{{version '@backstage/plugin-app-backend'}}", "@backstage/plugin-auth-backend": "^{{version '@backstage/plugin-auth-backend'}}", "@backstage/plugin-catalog-backend": "^{{version '@backstage/plugin-catalog-backend'}}", "@backstage/plugin-proxy-backend": "^{{version '@backstage/plugin-proxy-backend'}}", diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 51de1ad6b3..80dc6230a6 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -18,6 +18,7 @@ import { UrlReaders, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; +import app from './plugins/app'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import scaffolder from './plugins/scaffolder'; @@ -53,6 +54,7 @@ async function main() { const authEnv = useHotMemoize(module, () => createEnv('auth')); const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); + const appEnv = useHotMemoize(module, () => createEnv('app')); const apiRouter = Router(); apiRouter.use('/catalog', await catalog(catalogEnv)); @@ -64,7 +66,8 @@ async function main() { const service = createServiceBuilder(module) .loadConfig(config) - .addRouter('/api', apiRouter); + .addRouter('/api', apiRouter) + .addRouter('', await app(appEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts new file mode 100644 index 0000000000..51316949cb --- /dev/null +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts @@ -0,0 +1,13 @@ +import { createRouter } from '@backstage/plugin-app-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + return await createRouter({ + logger, + config, + appPackageName: 'app', + }); +} From 3525e6f687bd5fc9948e58fb2066a3864b39176a Mon Sep 17 00:00:00 2001 From: Stefano Vuerich Date: Thu, 19 Nov 2020 08:40:14 +0100 Subject: [PATCH 151/259] fix: conditional getuid and getgid for Windows environment --- .../stages/templater/helpers.test.ts | 15 +++++++------- .../scaffolder/stages/templater/helpers.ts | 20 ++++++++++++++----- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts index a4441ca23f..ca838fbfaa 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts @@ -17,15 +17,9 @@ import Stream, { PassThrough } from 'stream'; import os from 'os'; import fs from 'fs'; import Docker from 'dockerode'; -import { runDockerContainer } from './helpers'; +import { UserOptions, runDockerContainer} from './helpers'; describe('helpers', () => { - if (process.platform === 'win32') { - // eslint-disable-next-line jest/no-focused-tests - it.only('should skip tests on windows', () => { - expect('test').not.toBe('run'); - }); - } const mockDocker = new Docker() as jest.Mocked; @@ -142,12 +136,17 @@ describe('helpers', () => { dockerClient: mockDocker, }); + const userOptions: UserOptions = {}; + if (process.getuid && process.getgid) { + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + expect(mockDocker.run).toHaveBeenCalledWith( imageName, args, expect.any(Stream), expect.objectContaining({ - User: `${process.getuid()}:${process.getgid()}`, + ...userOptions, Env: ['HOME=/tmp'], }), ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index e71b63bfb0..42bdbc8060 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -36,6 +36,10 @@ export type RunCommandOptions = { logStream?: Writable; }; +export type UserOptions = { + User?: string; +} + /** * Gets the templater key to use for templating from the entity * @param entity Template entity @@ -115,6 +119,16 @@ export const runDockerContainer = async ({ }); }); + const userOptions: UserOptions = {}; + if (process.getuid && process.getgid) { + // Files that are created inside the Docker container will be owned by + // root on the host system on non Mac systems, because of reasons. Mainly the fact that + // volume sharing is done using NFS on Mac and actual mounts in Linux world. + // So we set the user in the container as the same user and group id as the host. + // On Windows we don't have process.getuid nor process.getgid + userOptions.User = `${process.getuid()}:${process.getgid()}`; + } + const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( imageName, args, @@ -129,11 +143,7 @@ export const runDockerContainer = async ({ `${await fs.promises.realpath(templateDir)}:/template`, ], }, - // Files that are created inside the Docker container will be owned by - // root on the host system on non Mac systems, because of reasons. Mainly the fact that - // volume sharing is done using NFS on Mac and actual mounts in Linux world. - // So we set the user in the container as the same user and group id as the host. - User: `${process.getuid()}:${process.getgid()}`, + ...userOptions, // Set the home directory inside the container as something that applications can // write to, otherwise they will just flop and fail trying to write to / Env: ['HOME=/tmp'], From 00b60af58ae758b0e3d0ba7f4c625a413dea7a12 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 19 Nov 2020 09:26:31 +0100 Subject: [PATCH 152/259] docs: Correct URL of techdocs-container --- docs/features/techdocs/getting-started.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index cbbbc1e6cd..e51c6a122e 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -95,7 +95,7 @@ environment is compatible with techdocs. You will have to install the `mkdocs` and `mkdocs-techdocs-core` package from pip, as well as `graphviz` and `plantuml` from your OS package manager (e.g. apt). See our -[Dockerfile](https://github.com/spotify/backstage/blob/master/packages/techdocs-container/Dockerfile) +[Dockerfile](https://github.com/backstage/techdocs-container/blob/main/Dockerfile) for the latest requirements. You should be trying to match your Dockerfile with this one. @@ -104,7 +104,7 @@ Note: We recommend Python version 3.7 or higher. Caveat: Please install the `mkdocs-techdocs-core` package after all other Python packages. The order is important to make sure we get correct version of some of the dependencies. For example, we want `Markdown` version to be -[3.2.2](https://github.com/spotify/backstage/blob/f9f70c225548017b6a14daea75b00fbd399c11eb/packages/techdocs-container/techdocs-core/requirements.txt#L11). +[3.2.2](https://github.com/backstage/backstage/blob/f9f70c225548017b6a14daea75b00fbd399c11eb/packages/techdocs-container/techdocs-core/requirements.txt#L11). You can also explicitly install `Markdown==3.2.2` after installing all other Python packages. From 49554d43e37084979c6e3f56f7a3e3f2b9c0c995 Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak <36006588+lewtakm@users.noreply.github.com> Date: Thu, 19 Nov 2020 10:30:34 +0100 Subject: [PATCH 153/259] Feat: update Lighthouse plugin to use new MarkdownComponent (#3328) --- plugins/lighthouse/package.json | 1 - .../lighthouse/src/components/Intro/index.tsx | 7 +- yarn.lock | 156 +----------------- 3 files changed, 7 insertions(+), 157 deletions(-) diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 53bfc0e78b..8172847120 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -34,7 +34,6 @@ "@types/react": "^16.9", "react": "^16.13.1", "react-dom": "^16.13.1", - "react-markdown": "^4.3.1", "react-router-dom": "6.0.0-beta.0", "react-use": "^15.3.3" }, diff --git a/plugins/lighthouse/src/components/Intro/index.tsx b/plugins/lighthouse/src/components/Intro/index.tsx index dc2a8ea283..3e969a3c74 100644 --- a/plugins/lighthouse/src/components/Intro/index.tsx +++ b/plugins/lighthouse/src/components/Intro/index.tsx @@ -15,8 +15,7 @@ */ import React, { useState } from 'react'; import { useLocalStorage } from 'react-use'; -import Markdown from 'react-markdown'; -import { ContentHeader, InfoCard } from '@backstage/core'; +import { ContentHeader, InfoCard, MarkdownContent } from '@backstage/core'; import { makeStyles, Button, Grid, Tabs, Tab } from '@material-ui/core'; import CloseIcon from '@material-ui/icons/Close'; @@ -116,8 +115,8 @@ function GettingStartedCard() { } > - {value === 0 && } - {value === 1 && } + {value === 0 && } + {value === 1 && } ); } diff --git a/yarn.lock b/yarn.lock index feacd32d97..bb59924b80 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8750,11 +8750,6 @@ codeowners-utils@^1.0.2: ignore "^5.1.4" locate-path "^5.0.0" -collapse-white-space@^1.0.2: - version "1.0.6" - resolved "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-1.0.6.tgz#e63629c0016665792060dbbeb79c42239d2c5287" - integrity sha512-jEovNnrhMuqyCcjfEJA56v0Xq8SkIoPKDyaHahwo3POf4qcSXqMYuwNcOTzp74vTsR9Tn08z4MxWqAhcekogkQ== - collect-v8-coverage@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.0.tgz#150ee634ac3650b71d9c985eb7f608942334feb1" @@ -13624,7 +13619,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.0, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -13851,7 +13846,7 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-buffer@^1.0.2, is-buffer@^1.1.4, is-buffer@^1.1.5: +is-buffer@^1.0.2, is-buffer@^1.1.5: version "1.1.6" resolved "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -14263,11 +14258,6 @@ is-utf8@^0.2.0: resolved "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= -is-whitespace-character@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/is-whitespace-character/-/is-whitespace-character-1.0.4.tgz#0858edd94a95594c7c9dd0b5c174ec6e45ee4aa7" - integrity sha512-SDweEzfIZM0SJV0EUga669UTKlmL0Pq8Lno0QDQsPnvECB3IM2aP0gdx5TrU0A01MAPfViaZiI2V1QMZLaKK5w== - is-window@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/is-window/-/is-window-1.0.2.tgz#2c896ca53db97de45d3c33133a65d8c9f563480d" @@ -14278,11 +14268,6 @@ is-windows@^1.0.0, is-windows@^1.0.1, is-windows@^1.0.2: resolved "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== -is-word-character@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/is-word-character/-/is-word-character-1.0.4.tgz#ce0e73216f98599060592f62ff31354ddbeb0230" - integrity sha512-5SMO8RVennx3nZrqtKwCGyyetPE9VDba5ugvKLaD4KopPG5kR4mQ7tNt/r7feL5yt5h3lpuBbIUmCOG2eSzXHA== - is-wsl@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz#1f16e4aa22b04d1336b66188a66af3c600c3a66d" @@ -16134,11 +16119,6 @@ map-visit@^1.0.0: dependencies: object-visit "^1.0.0" -markdown-escapes@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/markdown-escapes/-/markdown-escapes-1.0.4.tgz#c95415ef451499d7602b91095f3c8e8975f78535" - integrity sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg== - markdown-it@^10.0.0: version "10.0.0" resolved "https://registry.npmjs.org/markdown-it/-/markdown-it-10.0.0.tgz#abfc64f141b1722d663402044e43927f1f50a8dc" @@ -17996,7 +17976,7 @@ parse-asn1@^5.0.0: pbkdf2 "^3.0.3" safe-buffer "^5.1.1" -parse-entities@^1.1.0, parse-entities@^1.1.2: +parse-entities@^1.1.2: version "1.2.2" resolved "https://registry.npmjs.org/parse-entities/-/parse-entities-1.2.2.tgz#c31bf0f653b6661354f8973559cb86dd1d5edf50" integrity sha512-NzfpbxW/NPrzZ/yYSoQxyqUZMZXIdCfE0OIN4ESsnptHJECoUk3FZktxNuzQf4tjt5UEopnxpYJbvYuxIFDdsg== @@ -19781,20 +19761,6 @@ react-lifecycles-compat@^3.0.0, react-lifecycles-compat@^3.0.4: resolved "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" integrity sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA== -react-markdown@^4.3.1: - version "4.3.1" - resolved "https://registry.npmjs.org/react-markdown/-/react-markdown-4.3.1.tgz#39f0633b94a027445b86c9811142d05381300f2f" - integrity sha512-HQlWFTbDxTtNY6bjgp3C3uv1h2xcjCSi1zAEzfBW9OwJJvENSYiLXWNXN5hHLsoqai7RnZiiHzcnWdXk2Splzw== - dependencies: - html-to-react "^1.3.4" - mdast-add-list-metadata "1.0.1" - prop-types "^15.7.2" - react-is "^16.8.6" - remark-parse "^5.0.0" - unified "^6.1.5" - unist-util-visit "^1.3.0" - xtend "^4.0.1" - react-markdown@^5.0.0, react-markdown@^5.0.2: version "5.0.2" resolved "https://registry.npmjs.org/react-markdown/-/react-markdown-5.0.2.tgz#d15a8beb37b4ec34fc23dd892e7755eb7040b8db" @@ -20529,27 +20495,6 @@ remark-gfm@^1.0.0: mdast-util-gfm "^0.1.0" micromark-extension-gfm "^0.3.0" -remark-parse@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/remark-parse/-/remark-parse-5.0.0.tgz#4c077f9e499044d1d5c13f80d7a98cf7b9285d95" - integrity sha512-b3iXszZLH1TLoyUzrATcTQUZrwNl1rE70rVdSruJFlDaJ9z5aMkhrG43Pp68OgfHndL/ADz6V69Zow8cTQu+JA== - dependencies: - collapse-white-space "^1.0.2" - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - is-word-character "^1.0.0" - markdown-escapes "^1.0.0" - parse-entities "^1.1.0" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - trim "0.0.1" - trim-trailing-lines "^1.0.0" - unherit "^1.0.4" - unist-util-remove-position "^1.0.0" - vfile-location "^2.0.0" - xtend "^4.0.1" - remark-parse@^9.0.0: version "9.0.0" resolved "https://registry.npmjs.org/remark-parse/-/remark-parse-9.0.0.tgz#4d20a299665880e4f4af5d90b7c7b8a935853640" @@ -20596,7 +20541,7 @@ repeat-element@^1.1.2: resolved "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz#782e0d825c0c5a3bb39731f84efee6b742e6b1ce" integrity sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g== -repeat-string@^1.0.0, repeat-string@^1.5.2, repeat-string@^1.5.4, repeat-string@^1.6.1: +repeat-string@^1.0.0, repeat-string@^1.5.2, repeat-string@^1.6.1: version "1.6.1" resolved "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= @@ -21853,11 +21798,6 @@ start-server-webpack-plugin@^2.2.5: resolved "https://registry.npmjs.org/start-server-webpack-plugin/-/start-server-webpack-plugin-2.2.5.tgz#4a2838759b0f36acd11b0b2f5f196f289ae29d31" integrity sha512-DRCkciwCJoCFZ+wt3wWMkR1M2mpVhJbUKFXqhK3FWyIUKYb42NnocH5sMwqgo+nPNHupqNwK/v8lgfBbr2NKdg== -state-toggle@^1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe" - integrity sha512-d/5Z4/2iiCnHw6Xzghyhb+GcmF89bxwgXG60wjIiZaxnymbyOmI8Hk4VqHXiVVp6u2ysaskFfXg3ekCj4WNftQ== - static-extend@^0.1.1: version "0.1.2" resolved "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" @@ -22969,16 +22909,6 @@ trim-off-newlines@^1.0.0: resolved "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz#9f9ba9d9efa8764c387698bcbfeb2c848f11adb3" integrity sha1-n5up2e+odkw4dpi8v+sshI8RrbM= -trim-trailing-lines@^1.0.0: - version "1.1.3" - resolved "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-1.1.3.tgz#7f0739881ff76657b7776e10874128004b625a94" - integrity sha512-4ku0mmjXifQcTVfYDfR5lpgV7zVqPg6zV9rdZmwOPqq0+Zq19xDqEgagqVbc4pOOShbncuAOIs59R3+3gcF3ZA== - -trim@0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" - integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= - triple-beam@^1.2.0, triple-beam@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" @@ -23306,14 +23236,6 @@ unfetch@^4.1.0: resolved "https://registry.npmjs.org/unfetch/-/unfetch-4.1.0.tgz#6ec2dd0de887e58a4dee83a050ded80ffc4137db" integrity sha512-crP/n3eAPUJxZXM9T80/yv0YhkTEx2K1D3h7D1AJM6fzsWZrxdyRuLN0JH/dkZh1LNH8LxCnBzoPFCPbb2iGpg== -unherit@^1.0.4: - version "1.1.3" - resolved "https://registry.npmjs.org/unherit/-/unherit-1.1.3.tgz#6c9b503f2b41b262330c80e91c8614abdaa69c22" - integrity sha512-Ft16BJcnapDKp0+J/rqFC3Rrk6Y/Ng4nzsC028k2jdDII/rdZ7Wd3pPT/6+vIIxRagwRc9K0IUX0Ra4fKvw+WQ== - dependencies: - inherits "^2.0.0" - xtend "^4.0.0" - unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -23337,18 +23259,6 @@ unicode-property-aliases-ecmascript@^1.0.4: resolved "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz#dd57a99f6207bedff4628abefb94c50db941c8f4" integrity sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg== -unified@^6.1.5: - version "6.2.0" - resolved "https://registry.npmjs.org/unified/-/unified-6.2.0.tgz#7fbd630f719126d67d40c644b7e3f617035f6dba" - integrity sha512-1k+KPhlVtqmG99RaTbAv/usu85fcSRu3wY8X+vnsEhIxNP5VbVIDiXnLqyKIG+UMdyTg0ZX9EI6k2AfjJkHPtA== - dependencies: - bail "^1.0.0" - extend "^3.0.0" - is-plain-obj "^1.1.0" - trough "^1.0.0" - vfile "^2.0.0" - x-is-string "^0.1.0" - unified@^9.0.0: version "9.2.0" resolved "https://registry.npmjs.org/unified/-/unified-9.2.0.tgz#67a62c627c40589edebbf60f53edfd4d822027f8" @@ -23402,28 +23312,11 @@ unique-string@^2.0.0: dependencies: crypto-random-string "^2.0.0" -unist-util-is@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/unist-util-is/-/unist-util-is-3.0.0.tgz#d9e84381c2468e82629e4a5be9d7d05a2dd324cd" - integrity sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A== - unist-util-is@^4.0.0: version "4.0.3" resolved "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.0.3.tgz#e8b44db55fc20c43752b3346c116344d45d7c91d" integrity sha512-bTofCFVx0iQM8Jqb1TBDVRIQW03YkD3p66JOd/aCWuqzlLyUtx1ZAGw/u+Zw+SttKvSVcvTiKYbfrtLoLefykw== -unist-util-remove-position@^1.0.0: - version "1.1.4" - resolved "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz#ec037348b6102c897703eee6d0294ca4755a2020" - integrity sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A== - dependencies: - unist-util-visit "^1.1.0" - -unist-util-stringify-position@^1.0.0, unist-util-stringify-position@^1.1.1: - version "1.1.2" - resolved "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-1.1.2.tgz#3f37fcf351279dcbca7480ab5889bb8a832ee1c6" - integrity sha512-pNCVrk64LZv1kElr0N1wPiHEUoXNVFERp+mlTg/s9R5Lwg87f9bM/3sQB99w+N9D/qnM9ar3+AKDBwo/gm/iQQ== - unist-util-stringify-position@^2.0.0: version "2.0.3" resolved "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz#cce3bfa1cdf85ba7375d1d5b17bdc4cada9bd9da" @@ -23436,13 +23329,6 @@ unist-util-visit-parents@1.1.2: resolved "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-1.1.2.tgz#f6e3afee8bdbf961c0e6f028ea3c0480028c3d06" integrity sha512-yvo+MMLjEwdc3RhhPYSximset7rwjMrdt9E41Smmvg25UQIenzrN83cRnF1JMzoMi9zZOQeYXHSDf7p+IQkW3Q== -unist-util-visit-parents@^2.0.0: - version "2.1.2" - resolved "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz#25e43e55312166f3348cae6743588781d112c1e9" - integrity sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g== - dependencies: - unist-util-is "^3.0.0" - unist-util-visit-parents@^3.0.0: version "3.1.1" resolved "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz#65a6ce698f78a6b0f56aa0e88f13801886cdaef6" @@ -23451,13 +23337,6 @@ unist-util-visit-parents@^3.0.0: "@types/unist" "^2.0.0" unist-util-is "^4.0.0" -unist-util-visit@^1.1.0, unist-util-visit@^1.3.0: - version "1.4.1" - resolved "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz#4724aaa8486e6ee6e26d7ff3c8685960d560b1e3" - integrity sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw== - dependencies: - unist-util-visit-parents "^2.0.0" - unist-util-visit@^2.0.0: version "2.0.3" resolved "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz#c3703893146df47203bb8a9795af47d7b971208c" @@ -23813,18 +23692,6 @@ verror@1.10.0, verror@^1.8.1: core-util-is "1.0.2" extsprintf "^1.2.0" -vfile-location@^2.0.0: - version "2.0.6" - resolved "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz#8a274f39411b8719ea5728802e10d9e0dff1519e" - integrity sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA== - -vfile-message@^1.0.0: - version "1.1.1" - resolved "https://registry.npmjs.org/vfile-message/-/vfile-message-1.1.1.tgz#5833ae078a1dfa2d96e9647886cd32993ab313e1" - integrity sha512-1WmsopSGhWt5laNir+633LszXvZ+Z/lxveBf6yhGsqnQIhlhzooZae7zV6YVM1Sdkw68dtAW3ow0pOdPANugvA== - dependencies: - unist-util-stringify-position "^1.1.1" - vfile-message@^2.0.0: version "2.0.4" resolved "https://registry.npmjs.org/vfile-message/-/vfile-message-2.0.4.tgz#5b43b88171d409eae58477d13f23dd41d52c371a" @@ -23833,16 +23700,6 @@ vfile-message@^2.0.0: "@types/unist" "^2.0.0" unist-util-stringify-position "^2.0.0" -vfile@^2.0.0: - version "2.3.0" - resolved "https://registry.npmjs.org/vfile/-/vfile-2.3.0.tgz#e62d8e72b20e83c324bc6c67278ee272488bf84a" - integrity sha512-ASt4mBUHcTpMKD/l5Q+WJXNtshlWxOogYyGYYrg4lt/vuRjC1EFQtlAofL5VmtVNIZJzWYFJjzGWZ0Gw8pzW1w== - dependencies: - is-buffer "^1.1.4" - replace-ext "1.0.0" - unist-util-stringify-position "^1.0.0" - vfile-message "^1.0.0" - vfile@^4.0.0: version "4.2.0" resolved "https://registry.npmjs.org/vfile/-/vfile-4.2.0.tgz#26c78ac92eb70816b01d4565e003b7e65a2a0e01" @@ -24419,11 +24276,6 @@ ws@^7.3.1: resolved "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8" integrity sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA== -x-is-string@^0.1.0: - version "0.1.0" - resolved "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz#474b50865af3a49a9c4657f05acd145458f77d82" - integrity sha1-R0tQhlrzpJqcRlfwWs0UVFj3fYI= - xcase@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/xcase/-/xcase-2.0.1.tgz#c7fa72caa0f440db78fd5673432038ac984450b9" From ef8dff2cbe4e6388c661678bde23cd11cbdc7eff Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 19 Nov 2020 11:38:36 +0100 Subject: [PATCH 154/259] Replace broken icon for the api-docs plugin Closes #3330 I would guess that https://github.com/vscode-icons/vscode-icons is a more stable location for an icon. --- microsite/data/plugins/api-docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microsite/data/plugins/api-docs.yaml b/microsite/data/plugins/api-docs.yaml index 3343a46693..03ec818445 100644 --- a/microsite/data/plugins/api-docs.yaml +++ b/microsite/data/plugins/api-docs.yaml @@ -5,5 +5,5 @@ authorUrl: https://sda.se/ category: Discovery description: Components to discover and display API entities as an extension to the catalog plugin. documentation: https://github.com/backstage/backstage/blob/master/plugins/api-docs/README.md -iconUrl: https://thecoders.io/wp-content/uploads/2019/11/tech-swagger.svg +iconUrl: https://raw.githubusercontent.com/vscode-icons/vscode-icons/master/icons/file_type_swagger.svg npmPackageName: '@backstage/plugin-api-docs' From 09d90484ea25b34b28b3f61eef95139385458e83 Mon Sep 17 00:00:00 2001 From: Stefano Vuerich Date: Thu, 19 Nov 2020 11:46:34 +0100 Subject: [PATCH 155/259] fix: runned prettier --- .../src/scaffolder/stages/templater/helpers.test.ts | 3 +-- .../src/scaffolder/stages/templater/helpers.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts index ca838fbfaa..57e47f4c27 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts @@ -17,10 +17,9 @@ import Stream, { PassThrough } from 'stream'; import os from 'os'; import fs from 'fs'; import Docker from 'dockerode'; -import { UserOptions, runDockerContainer} from './helpers'; +import { UserOptions, runDockerContainer } from './helpers'; describe('helpers', () => { - const mockDocker = new Docker() as jest.Mocked; beforeEach(() => { diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index 42bdbc8060..e79f2cb066 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -38,7 +38,7 @@ export type RunCommandOptions = { export type UserOptions = { User?: string; -} +}; /** * Gets the templater key to use for templating from the entity From 019c67a02280e39e25f07c92745c36bf849846ea Mon Sep 17 00:00:00 2001 From: Askar Date: Thu, 19 Nov 2020 13:39:22 +0100 Subject: [PATCH 156/259] fix(search): return EmptyState/Progress bar/Alert depending on result of api call (#3350) * fix(search): return EmptyState/Progress bar/Alert depending on result of api request * Update plugins/search/src/components/SearchResult/SearchResult.tsx Co-authored-by: Emma Indal Co-authored-by: Emma Indal --- .../components/SearchResult/SearchResult.tsx | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index a4985c20fb..e4163c0de7 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -17,7 +17,14 @@ import React, { useState, useEffect } from 'react'; import { useAsync } from 'react-use'; import { makeStyles, Typography, Grid, Divider } from '@material-ui/core'; -import { Table, TableColumn, useApi } from '@backstage/core'; +import { Alert } from '@material-ui/lab'; +import { + EmptyState, + Progress, + Table, + TableColumn, + useApi, +} from '@backstage/core'; import { catalogApiRef } from '@backstage/plugin-catalog'; import { FiltersButton, Filters, FiltersState } from '../Filters'; @@ -159,8 +166,19 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => { setFilteredResults(withFilters); } }, [filters, searchQuery, results]); - - if (loading || error || !results) return null; + if (loading) { + return ; + } + if (error) { + return ( + + Error encountered while fetching search results. {error.toString()} + + ); + } + if (!results || results.length === 0) { + return ; + } const resetFilters = () => { setFilters({ From 76306d3e4b2cae48606d1172b9490cb612f17c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Wed, 18 Nov 2020 07:38:18 +0100 Subject: [PATCH 157/259] Extract types --- .../src/scaffolder/stages/publish/types.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts index 6e9c45ba43..6dfc29dbde 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts @@ -29,11 +29,18 @@ export type PublisherBase = { * catalog, plus the values from the form and the directory that has * been templated */ - publish(opts: { - entity: TemplateEntityV1alpha1; - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }>; + publish(opts: PublisherOptions): Promise; +}; + +export type PublisherOptions = { + entity: TemplateEntityV1alpha1; + values: RequiredTemplateValues & Record; + directory: string; +}; + +export type PublisherResult = { + remoteUrl: string; + catalogInfoUrl?: string; }; export type PublisherBuilder = { From e0a910bc6dcaeef3a0790a7324893562dbe956f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 12:54:31 +0100 Subject: [PATCH 158/259] Start using the extracted types --- .../src/scaffolder/stages/publish/azure.ts | 7 ++----- .../src/scaffolder/stages/publish/github.ts | 7 ++----- .../src/scaffolder/stages/publish/gitlab.ts | 7 ++----- .../src/scaffolder/stages/publish/types.ts | 1 - plugins/scaffolder-backend/src/service/router.ts | 1 - 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 33f7f89199..8f3ed33423 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PublisherBase } from './types'; +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { GitApi } from 'azure-devops-node-api/GitApi'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; import { pushToRemoteUserPass } from './helpers'; @@ -33,10 +33,7 @@ export class AzurePublisher implements PublisherBase { async publish({ values, directory, - }: { - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }> { + }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index d5542e8800..2e835ba3d4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PublisherBase } from './types'; +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Octokit } from '@octokit/rest'; import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; @@ -46,10 +46,7 @@ export class GithubPublisher implements PublisherBase { async publish({ values, directory, - }: { - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }> { + }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass( directory, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts index e748dc53dc..4fae8e8a5c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PublisherBase } from './types'; +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Gitlab } from '@gitbeaker/core'; import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; @@ -32,10 +32,7 @@ export class GitlabPublisher implements PublisherBase { async publish({ values, directory, - }: { - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }> { + }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass(directory, remoteUrl, 'oauth2', this.token); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts index 6dfc29dbde..f3bc59e19d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts @@ -33,7 +33,6 @@ export type PublisherBase = { }; export type PublisherOptions = { - entity: TemplateEntityV1alpha1; values: RequiredTemplateValues & Record; directory: string; }; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 2dc6a794fb..fd25f2d845 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -157,7 +157,6 @@ export async function createRouter( const publisher = publishers.get(ctx.entity); ctx.logger.info('Will now store the template'); const { remoteUrl } = await publisher.publish({ - entity: ctx.entity, values: ctx.values, directory: ctx.resultDir, }); From 67b6320e839e504f7c997e19dd3b9171ecc55188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 13:51:40 +0100 Subject: [PATCH 159/259] Generate catatalog-info.yaml URL in the publishers --- .../scaffolder/stages/publish/azure.test.ts | 10 ++-- .../src/scaffolder/stages/publish/azure.ts | 3 +- .../scaffolder/stages/publish/github.test.ts | 50 +++++++++++++------ .../src/scaffolder/stages/publish/github.ts | 6 ++- .../scaffolder-backend/src/service/router.ts | 4 +- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts index 7ab6f81ae5..9ea7d2de5f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts @@ -41,7 +41,7 @@ describe('Azure Publisher', () => { describe('publish: createRemoteInAzure', () => { it('should use azure-devops-node-api to create a repo in the given project', async () => { mockGitApi.createRepository.mockResolvedValue({ - remoteUrl: 'mockclone', + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', } as { remoteUrl: string }); const result = await publisher.publish({ @@ -52,7 +52,11 @@ describe('Azure Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + catalogInfoUrl: + 'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml', + }); expect(mockGitApi.createRepository).toHaveBeenCalledWith( { name: 'repo', @@ -61,7 +65,7 @@ describe('Azure Publisher', () => { ); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://dev.azure.com/organization/project/_git/repo', 'notempty', 'fake-token', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 8f3ed33423..1e962bf223 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -36,8 +36,9 @@ export class AzurePublisher implements PublisherBase { }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token); + const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`; - return { remoteUrl }; + return { remoteUrl, catalogInfoUrl }; } private async createRemote( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts index c85b4acba3..cda64faf25 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts @@ -53,7 +53,7 @@ describe('GitHub Publisher', () => { it('should use octokit to create a repo in an organisation if the organisation property is set', async () => { mockGithubClient.repos.createInOrg.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -71,7 +71,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ org: 'blam', name: 'test', @@ -89,7 +93,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -98,7 +102,7 @@ describe('GitHub Publisher', () => { it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -116,7 +120,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -126,7 +134,7 @@ describe('GitHub Publisher', () => { expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled(); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -136,7 +144,7 @@ describe('GitHub Publisher', () => { it('should invite other user in the authed user', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -155,7 +163,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -171,7 +183,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -188,7 +200,7 @@ describe('GitHub Publisher', () => { it('creates a private repository in the organization with visibility set to internal', async () => { mockGithubClient.repos.createInOrg.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -206,7 +218,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ org: 'blam', name: 'test', @@ -215,7 +231,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -232,7 +248,7 @@ describe('GitHub Publisher', () => { it('creates a private repository', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -249,7 +265,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -258,7 +278,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 2e835ba3d4..64976b8e41 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -54,8 +54,12 @@ export class GithubPublisher implements PublisherBase { this.token, 'x-oauth-basic', ); + const catalogInfoUrl = remoteUrl.replace( + /\.git$/, + '/blob/master/catalog-info.yaml', + ); - return { remoteUrl }; + return { remoteUrl, catalogInfoUrl }; } private async createRemote( diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fd25f2d845..ac3b1c80ec 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -156,11 +156,11 @@ export async function createRouter( handler: async (ctx: StageContext<{ resultDir: string }>) => { const publisher = publishers.get(ctx.entity); ctx.logger.info('Will now store the template'); - const { remoteUrl } = await publisher.publish({ + const result = await publisher.publish({ values: ctx.values, directory: ctx.resultDir, }); - return { remoteUrl }; + return result; }, }, ], From a13a090de99a08cb5e2f4a5f1473bcc3bb237f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 13:52:21 +0100 Subject: [PATCH 160/259] Use the new catalogInfoUrl in TemplatePage --- .../src/components/TemplatePage/TemplatePage.tsx | 14 +++----------- plugins/scaffolder/src/types.ts | 1 + 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 4836cdab56..ffa2bfcec7 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -106,18 +106,10 @@ export const TemplatePage = () => { ); const handleCreateComplete = async (job: Job) => { - const target = job.metadata.remoteUrl?.replace( - /\.git$/, - // TODO(Rugvip): This is not the location we want. As part of scaffolder v2 we - // want this to be more flexible, but before that we might want - // to update all templates to use catalog-info.yaml instead. - '/blob/master/component-info.yaml', - ); - - if (!target) { + if (!job.metadata.catalogInfoUrl) { errorApi.post( new Error( - `Failed to find component-info.yaml file in ${job.metadata.remoteUrl}.`, + `Failed to find catalog-info.yaml file in ${job.metadata.remoteUrl}.`, ), ); return; @@ -125,7 +117,7 @@ export const TemplatePage = () => { const { entities: [createdEntity], - } = await catalogApi.addLocation({ target }); + } = await catalogApi.addLocation({ target: job.metadata.catalogInfoUrl }); setEntity((createdEntity as any) as TemplateEntityV1alpha1); }; diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 7106444904..e07581ad28 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -19,6 +19,7 @@ export type Job = { entity: any; values: any; remoteUrl?: string; + catalogInfoUrl?: string; }; status: 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; stages: Stage[]; From ef2831dde3567a2c02f78e7bc8d31c62bc212ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 15:00:24 +0100 Subject: [PATCH 161/259] Add changeset --- .changeset/dull-pans-sip.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/dull-pans-sip.md diff --git a/.changeset/dull-pans-sip.md b/.changeset/dull-pans-sip.md new file mode 100644 index 0000000000..13616c1f8d --- /dev/null +++ b/.changeset/dull-pans-sip.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Move constructing the catalog-info.yaml URL for scaffolded components to the publishers From bc0f47b7b795a09fa2dd955d4aba844b529edcf4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Nov 2020 15:13:57 +0000 Subject: [PATCH 162/259] Version Packages --- .changeset/add-app-backend.md | 104 ----------------- .changeset/add-config-schema-declarations.md | 13 --- .changeset/add-config-schema-support.md | 27 ----- .changeset/app-backend-config.md | 7 -- .../cost-insights-strange-rings-smile.md | 5 - .../cost-insights-ten-carpets-return.md | 5 - .changeset/cost-insights-wet-plants-glow.md | 6 - .changeset/cost-insights-wild-cars-wait.md | 5 - .changeset/friendly-dodos-remember.md | 5 - .changeset/hip-experts-dance.md | 5 - .changeset/loud-pets-clap.md | 5 - .changeset/modern-boats-knock.md | 5 - .changeset/ninety-gifts-guess.md | 6 - .changeset/odd-camels-begin.md | 5 - .changeset/shaggy-turkeys-warn.md | 5 - .changeset/silent-suns-bathe.md | 5 - packages/app/CHANGELOG.md | 24 ++++ packages/app/package.json | 22 ++-- packages/backend-common/CHANGELOG.md | 38 +++++++ packages/backend-common/package.json | 10 +- packages/backend/CHANGELOG.md | 25 +++++ packages/backend/package.json | 28 ++--- packages/catalog-client/CHANGELOG.md | 7 ++ packages/catalog-client/package.json | 4 +- packages/catalog-model/package.json | 2 +- packages/cli/CHANGELOG.md | 33 ++++++ packages/cli/package.json | 12 +- packages/config-loader/CHANGELOG.md | 26 +++++ packages/config-loader/package.json | 2 +- packages/core-api/package.json | 4 +- packages/core/CHANGELOG.md | 6 + packages/core/package.json | 6 +- packages/create-app/CHANGELOG.md | 105 ++++++++++++++++++ packages/create-app/package.json | 32 +++--- packages/dev-utils/CHANGELOG.md | 11 ++ packages/dev-utils/package.json | 8 +- packages/integration/CHANGELOG.md | 7 ++ packages/integration/package.json | 4 +- packages/test-utils/CHANGELOG.md | 9 ++ packages/test-utils/package.json | 4 +- packages/theme/package.json | 2 +- plugins/api-docs/package.json | 10 +- plugins/app-backend/CHANGELOG.md | 17 +++ plugins/app-backend/package.json | 8 +- plugins/auth-backend/CHANGELOG.md | 12 ++ plugins/auth-backend/package.json | 8 +- plugins/catalog-backend/CHANGELOG.md | 11 ++ plugins/catalog-backend/package.json | 8 +- plugins/catalog-graphql/CHANGELOG.md | 10 ++ plugins/catalog-graphql/package.json | 8 +- plugins/catalog/CHANGELOG.md | 11 ++ plugins/catalog/package.json | 14 +-- plugins/circleci/package.json | 10 +- plugins/cloudbuild/package.json | 10 +- plugins/cost-insights/CHANGELOG.md | 17 +++ plugins/cost-insights/package.json | 12 +- plugins/explore/package.json | 8 +- plugins/gcp-projects/package.json | 8 +- plugins/github-actions/package.json | 10 +- plugins/gitops-profiles/package.json | 8 +- plugins/graphiql/package.json | 8 +- plugins/graphql/CHANGELOG.md | 11 ++ plugins/graphql/package.json | 8 +- plugins/jenkins/package.json | 10 +- plugins/kubernetes-backend/CHANGELOG.md | 10 ++ plugins/kubernetes-backend/package.json | 6 +- plugins/kubernetes/package.json | 10 +- plugins/lighthouse/CHANGELOG.md | 10 ++ plugins/lighthouse/package.json | 12 +- plugins/newrelic/package.json | 8 +- plugins/proxy-backend/CHANGELOG.md | 10 ++ plugins/proxy-backend/package.json | 6 +- plugins/register-component/package.json | 10 +- plugins/rollbar-backend/CHANGELOG.md | 10 ++ plugins/rollbar-backend/package.json | 6 +- plugins/rollbar/CHANGELOG.md | 10 ++ plugins/rollbar/package.json | 12 +- plugins/scaffolder-backend/CHANGELOG.md | 11 ++ plugins/scaffolder-backend/package.json | 6 +- plugins/scaffolder/package.json | 10 +- plugins/search/package.json | 10 +- plugins/sentry-backend/CHANGELOG.md | 10 ++ plugins/sentry-backend/package.json | 6 +- plugins/sentry/CHANGELOG.md | 8 ++ plugins/sentry/package.json | 10 +- plugins/sonarqube/package.json | 8 +- plugins/tech-radar/package.json | 10 +- plugins/techdocs-backend/CHANGELOG.md | 10 ++ plugins/techdocs-backend/package.json | 6 +- plugins/techdocs/CHANGELOG.md | 11 ++ plugins/techdocs/package.json | 14 +-- plugins/user-settings/CHANGELOG.md | 8 ++ plugins/user-settings/package.json | 10 +- plugins/welcome/package.json | 8 +- 94 files changed, 716 insertions(+), 441 deletions(-) delete mode 100644 .changeset/add-app-backend.md delete mode 100644 .changeset/add-config-schema-declarations.md delete mode 100644 .changeset/add-config-schema-support.md delete mode 100644 .changeset/app-backend-config.md delete mode 100644 .changeset/cost-insights-strange-rings-smile.md delete mode 100644 .changeset/cost-insights-ten-carpets-return.md delete mode 100644 .changeset/cost-insights-wet-plants-glow.md delete mode 100644 .changeset/cost-insights-wild-cars-wait.md delete mode 100644 .changeset/friendly-dodos-remember.md delete mode 100644 .changeset/hip-experts-dance.md delete mode 100644 .changeset/loud-pets-clap.md delete mode 100644 .changeset/modern-boats-knock.md delete mode 100644 .changeset/ninety-gifts-guess.md delete mode 100644 .changeset/odd-camels-begin.md delete mode 100644 .changeset/shaggy-turkeys-warn.md delete mode 100644 .changeset/silent-suns-bathe.md create mode 100644 packages/catalog-client/CHANGELOG.md create mode 100644 packages/integration/CHANGELOG.md diff --git a/.changeset/add-app-backend.md b/.changeset/add-app-backend.md deleted file mode 100644 index e44a56c885..0000000000 --- a/.changeset/add-app-backend.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Add `app-backend` as a backend plugin, and make a single docker build of the backend the default way to deploy backstage. - -Note that the `app-backend` currently only is a solution for deployments of the app, it's not a dev server and is not intended for local development. - -## Template changes - -As a part of installing the `app-backend` plugin, the below changes where made. The changes are grouped into two steps, installing the plugin, and updating the Docker build and configuration. - -### Installing the `app-backend` plugin in the backend - -First, install the `@backstage/plugin-app-backend` plugin package in your backend. These changes where made for `v0.3.0` of the plugin, and the installation process might change in the future. Run the following from the root of the repo: - -```bash -cd packages/backend -yarn add @backstage/plugin-app-backend -``` - -For the `app-backend` to get access to the static content in the frontend we also need to add the local `app` package as a dependency. Add the following to your `"dependencies"` in `packages/backend/package.json`, assuming your app package is still named `app` and on version `0.0.0`: - -```json -"app": "0.0.0", -``` - -Don't worry, this will not cause your entire frontend dependency tree to be added to the app, just double check that `packages/app/package.json` has a `"bundled": true` field at top-level. This signals to the backend build process that the package is bundled and that no transitive dependencies should be included. - -Next, create `packages/backend/src/plugins/app.ts` with the following: - -```ts -import { createRouter } from '@backstage/plugin-app-backend'; -import { PluginEnvironment } from '../types'; - -export default async function createPlugin({ - logger, - config, -}: PluginEnvironment) { - return await createRouter({ - logger, - config, - appPackageName: 'app', - }); -} -``` - -In `packages/backend/src/index.ts`, make the following changes: - -Add an import for the newly created plugin setup file: - -```ts -import app from './plugins/app'; -``` - -Setup the following plugin env. - -```ts -const appEnv = useHotMemoize(module, () => createEnv('app')); -``` - -Change service builder setup to include the `app` plugin as follows. Note that the `app` plugin is not installed on the `/api` route with most other plugins. - -```ts -const service = createServiceBuilder(module) - .loadConfig(config) - .addRouter('/api', apiRouter) - .addRouter('', await app(appEnv)); -``` - -You should now have the `app-backend` plugin installed in your backend, ready to serve the frontend bundle! - -### Docker build setup - -Since the backend image is now the only one needed for a simple Backstage deployment, the image tag name in the `build-image` script inside `packages/backend/package.json` was changed to the following: - -```json -"build-image": "backstage-cli backend:build-image --build --tag backstage", -``` - -For convenience, a `build-image` script was also added to the root `package.json` with the following: - -```json -"build-image": "yarn workspace backend build-image", -``` - -In the root of the repo, a new `app-config.production.yaml` file was added. This is used to set the appropriate `app.baseUrl` now that the frontend is served directly by the backend in the production deployment. It has the following contents: - -```yaml -app: - # Should be the same as backend.baseUrl when using the `app-backend` plugin - baseUrl: http://localhost:7000 - -backend: - baseUrl: http://localhost:7000 - listen: - port: 7000 -``` - -In order to load in the new configuration at runtime, the command in the `Dockerfile` at the repo root was changed to the following: - -```dockerfile -CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] -``` diff --git a/.changeset/add-config-schema-declarations.md b/.changeset/add-config-schema-declarations.md deleted file mode 100644 index 2f52a2ad3f..0000000000 --- a/.changeset/add-config-schema-declarations.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -'@backstage/backend-common': patch -'@backstage/cli': patch -'@backstage/core': patch -'@backstage/plugin-cost-insights': patch -'@backstage/plugin-lighthouse': patch -'@backstage/plugin-rollbar': patch -'@backstage/plugin-sentry': patch -'@backstage/plugin-techdocs': patch -'@backstage/plugin-user-settings': patch ---- - -Added configuration schema diff --git a/.changeset/add-config-schema-support.md b/.changeset/add-config-schema-support.md deleted file mode 100644 index 300c75f651..0000000000 --- a/.changeset/add-config-schema-support.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -'@backstage/backend-common': minor -'@backstage/cli': minor -'@backstage/config-loader': minor ---- - -Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas. - -The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. - -A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. - -Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format. - -TypeScript configuration schema files should export a single `Config` type, for example: - -```ts -export interface Config { - app: { - /** - * Frontend root URL - * @visibility frontend - */ - baseUrl: string; - }; -} -``` diff --git a/.changeset/app-backend-config.md b/.changeset/app-backend-config.md deleted file mode 100644 index 39b0b956ad..0000000000 --- a/.changeset/app-backend-config.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/plugin-app-backend': minor ---- - -Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection. - -This removes the confusing behavior where configuration was only injected into the app at build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. diff --git a/.changeset/cost-insights-strange-rings-smile.md b/.changeset/cost-insights-strange-rings-smile.md deleted file mode 100644 index c5f4ff9b73..0000000000 --- a/.changeset/cost-insights-strange-rings-smile.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-cost-insights': patch ---- - -remove excessive margin from cost overview banner diff --git a/.changeset/cost-insights-ten-carpets-return.md b/.changeset/cost-insights-ten-carpets-return.md deleted file mode 100644 index f378a20eab..0000000000 --- a/.changeset/cost-insights-ten-carpets-return.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-cost-insights': minor ---- - -remove cost insights currency feature flag diff --git a/.changeset/cost-insights-wet-plants-glow.md b/.changeset/cost-insights-wet-plants-glow.md deleted file mode 100644 index 3d2be289d4..0000000000 --- a/.changeset/cost-insights-wet-plants-glow.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-cost-insights': patch ---- - -UI improvements: Increase width of first column in product entity dialog table -UI improvement: Display full cost amount in product entity dialog table diff --git a/.changeset/cost-insights-wild-cars-wait.md b/.changeset/cost-insights-wild-cars-wait.md deleted file mode 100644 index 24f84dc56b..0000000000 --- a/.changeset/cost-insights-wild-cars-wait.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-cost-insights': patch ---- - -Fix savings/excess display calculation diff --git a/.changeset/friendly-dodos-remember.md b/.changeset/friendly-dodos-remember.md deleted file mode 100644 index 463d71f806..0000000000 --- a/.changeset/friendly-dodos-remember.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog': patch ---- - -Add About Card tooltips diff --git a/.changeset/hip-experts-dance.md b/.changeset/hip-experts-dance.md deleted file mode 100644 index f88fe62cd4..0000000000 --- a/.changeset/hip-experts-dance.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -An entity A, that exists in the catalog, can no longer be overwritten by registering a different location that also tries to supply an entity with the same kind+namespace+name. Writes of that new entity will instead be rejected with a log message similar to `Rejecting write of entity Component:default/artist-lookup from file:/Users/freben/dev/github/backstage/packages/catalog-model/examples/components/artist-lookup-component.yaml because entity existed from github:https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml` diff --git a/.changeset/loud-pets-clap.md b/.changeset/loud-pets-clap.md deleted file mode 100644 index 0d50c066f9..0000000000 --- a/.changeset/loud-pets-clap.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'example-app': patch ---- - -Removed obsolete CircleCI proxy config from example-app diff --git a/.changeset/modern-boats-knock.md b/.changeset/modern-boats-knock.md deleted file mode 100644 index 16f80fe041..0000000000 --- a/.changeset/modern-boats-knock.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/cli': patch ---- - -Support specifying listen host/port for frontend diff --git a/.changeset/ninety-gifts-guess.md b/.changeset/ninety-gifts-guess.md deleted file mode 100644 index bdd20322ec..0000000000 --- a/.changeset/ninety-gifts-guess.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/backend-common': patch -'@backstage/integration': patch ---- - -Added the integration package diff --git a/.changeset/odd-camels-begin.md b/.changeset/odd-camels-begin.md deleted file mode 100644 index d28d729609..0000000000 --- a/.changeset/odd-camels-begin.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-common': minor ---- - -Refactored UrlReader.readTree to be required and accept (url, options) diff --git a/.changeset/shaggy-turkeys-warn.md b/.changeset/shaggy-turkeys-warn.md deleted file mode 100644 index 442cc788f2..0000000000 --- a/.changeset/shaggy-turkeys-warn.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/catalog-client': minor ---- - -Changed the getEntities interface to (1) nest parameters in an object, (2) support field selection, and (3) return an object with an items field for future extension diff --git a/.changeset/silent-suns-bathe.md b/.changeset/silent-suns-bathe.md deleted file mode 100644 index 96e32ea125..0000000000 --- a/.changeset/silent-suns-bathe.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-scaffolder-backend': patch ---- - -Extracted pushToRemote function for reuse between publishers diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index e24e7393ec..ab00924a26 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,29 @@ # example-app +## 0.2.2 + +### Patch Changes + +- 3efd03c0e: Removed obsolete CircleCI proxy config from example-app +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [17a9f48f6] +- Updated dependencies [4040d4fcb] +- Updated dependencies [f360395d0] +- Updated dependencies [259d848ee] +- Updated dependencies [8b7737d0b] +- Updated dependencies [902340451] + - @backstage/cli@0.3.0 + - @backstage/core@0.3.1 + - @backstage/plugin-cost-insights@0.4.0 + - @backstage/plugin-lighthouse@0.2.2 + - @backstage/plugin-rollbar@0.2.2 + - @backstage/plugin-sentry@0.2.2 + - @backstage/plugin-techdocs@0.2.2 + - @backstage/plugin-user-settings@0.2.2 + - @backstage/plugin-catalog@0.2.2 + - @backstage/test-utils@0.1.3 + ## 0.2.1 ### Patch Changes diff --git a/packages/app/package.json b/packages/app/package.json index 61b4f7a264..529c5d4f63 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,17 +1,17 @@ { "name": "example-app", - "version": "0.2.1", + "version": "0.2.2", "private": true, "bundled": true, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/cli": "^0.2.0", - "@backstage/core": "^0.3.0", + "@backstage/cli": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/plugin-api-docs": "^0.2.1", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/plugin-circleci": "^0.2.1", "@backstage/plugin-cloudbuild": "^0.2.1", - "@backstage/plugin-cost-insights": "^0.3.0", + "@backstage/plugin-cost-insights": "^0.4.0", "@backstage/plugin-explore": "^0.2.1", "@backstage/plugin-gcp-projects": "^0.2.1", "@backstage/plugin-github-actions": "^0.2.1", @@ -19,18 +19,18 @@ "@backstage/plugin-graphiql": "^0.2.1", "@backstage/plugin-jenkins": "^0.3.0", "@backstage/plugin-kubernetes": "^0.2.1", - "@backstage/plugin-lighthouse": "^0.2.1", + "@backstage/plugin-lighthouse": "^0.2.2", "@backstage/plugin-newrelic": "^0.2.1", "@backstage/plugin-register-component": "^0.2.1", - "@backstage/plugin-rollbar": "^0.2.1", + "@backstage/plugin-rollbar": "^0.2.2", "@backstage/plugin-scaffolder": "^0.3.0", - "@backstage/plugin-sentry": "^0.2.1", + "@backstage/plugin-sentry": "^0.2.2", "@backstage/plugin-search": "^0.2.0", "@backstage/plugin-tech-radar": "^0.3.0", - "@backstage/plugin-techdocs": "^0.2.1", - "@backstage/plugin-user-settings": "^0.2.1", + "@backstage/plugin-techdocs": "^0.2.2", + "@backstage/plugin-user-settings": "^0.2.2", "@backstage/plugin-welcome": "^0.2.1", - "@backstage/test-utils": "^0.1.2", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/packages/backend-common/CHANGELOG.md b/packages/backend-common/CHANGELOG.md index 0f18908932..4822032c7f 100644 --- a/packages/backend-common/CHANGELOG.md +++ b/packages/backend-common/CHANGELOG.md @@ -1,5 +1,43 @@ # @backstage/backend-common +## 0.3.0 + +### Minor Changes + +- 1722cb53c: Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas. + + The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. + + A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. + + Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format. + + TypeScript configuration schema files should export a single `Config` type, for example: + + ```ts + export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; + } + ``` + +- 8e2effb53: Refactored UrlReader.readTree to be required and accept (url, options) + +### Patch Changes + +- 1722cb53c: Added configuration schema +- 7b37e6834: Added the integration package +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] + - @backstage/config-loader@0.3.0 + - @backstage/integration@0.1.1 + - @backstage/test-utils@0.1.3 + ## 0.2.1 ### Patch Changes diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 737e9664bf..89150302b5 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.2.1", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -31,9 +31,9 @@ "dependencies": { "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", - "@backstage/config-loader": "^0.2.0", - "@backstage/integration": "^0.1.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/config-loader": "^0.3.0", + "@backstage/integration": "^0.1.1", + "@backstage/test-utils": "^0.1.3", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "compression": "^1.7.4", @@ -66,7 +66,7 @@ } }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/compression": "^1.7.0", "@types/concat-stream": "^1.6.0", "@types/fs-extra": "^9.0.3", diff --git a/packages/backend/CHANGELOG.md b/packages/backend/CHANGELOG.md index 2fe72cb101..a8bc1fef7d 100644 --- a/packages/backend/CHANGELOG.md +++ b/packages/backend/CHANGELOG.md @@ -1,5 +1,30 @@ # example-backend +## 0.2.2 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [f531d307c] +- Updated dependencies [3efd03c0e] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] +- Updated dependencies [d33f5157c] + - @backstage/backend-common@0.3.0 + - @backstage/plugin-app-backend@0.3.0 + - @backstage/plugin-catalog-backend@0.2.1 + - example-app@0.2.2 + - @backstage/plugin-scaffolder-backend@0.3.1 + - @backstage/plugin-auth-backend@0.2.2 + - @backstage/plugin-graphql-backend@0.1.3 + - @backstage/plugin-kubernetes-backend@0.1.3 + - @backstage/plugin-proxy-backend@0.2.1 + - @backstage/plugin-rollbar-backend@0.1.3 + - @backstage/plugin-sentry-backend@0.1.3 + - @backstage/plugin-techdocs-backend@0.2.1 + ## 0.2.1 ### Patch Changes diff --git a/packages/backend/package.json b/packages/backend/package.json index e67cadb44e..014a3f78d5 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "example-backend", - "version": "0.2.1", + "version": "0.2.2", "main": "dist/index.cjs.js", "types": "src/index.ts", "private": true, @@ -18,24 +18,24 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { - "@backstage/backend-common": "^0.2.1", + "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", - "@backstage/plugin-app-backend": "^0.2.0", - "@backstage/plugin-auth-backend": "^0.2.1", - "@backstage/plugin-catalog-backend": "^0.2.0", - "@backstage/plugin-graphql-backend": "^0.1.2", - "@backstage/plugin-kubernetes-backend": "^0.1.2", - "@backstage/plugin-proxy-backend": "^0.2.0", - "@backstage/plugin-rollbar-backend": "^0.1.2", - "@backstage/plugin-scaffolder-backend": "^0.3.0", - "@backstage/plugin-sentry-backend": "^0.1.2", - "@backstage/plugin-techdocs-backend": "^0.2.0", + "@backstage/plugin-app-backend": "^0.3.0", + "@backstage/plugin-auth-backend": "^0.2.2", + "@backstage/plugin-catalog-backend": "^0.2.1", + "@backstage/plugin-graphql-backend": "^0.1.3", + "@backstage/plugin-kubernetes-backend": "^0.1.3", + "@backstage/plugin-proxy-backend": "^0.2.1", + "@backstage/plugin-rollbar-backend": "^0.1.3", + "@backstage/plugin-scaffolder-backend": "^0.3.1", + "@backstage/plugin-sentry-backend": "^0.1.3", + "@backstage/plugin-techdocs-backend": "^0.2.1", "@gitbeaker/node": "^25.2.0", "@octokit/rest": "^18.0.0", "azure-devops-node-api": "^10.1.1", "dockerode": "^3.2.0", - "example-app": "^0.2.1", + "example-app": "^0.2.2", "express": "^4.17.1", "express-promise-router": "^3.0.3", "knex": "^0.21.6", @@ -45,7 +45,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/dockerode": "^2.5.32", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5", diff --git a/packages/catalog-client/CHANGELOG.md b/packages/catalog-client/CHANGELOG.md new file mode 100644 index 0000000000..574cd301bf --- /dev/null +++ b/packages/catalog-client/CHANGELOG.md @@ -0,0 +1,7 @@ +# @backstage/catalog-client + +## 0.3.0 + +### Minor Changes + +- 717e43de1: Changed the getEntities interface to (1) nest parameters in an object, (2) support field selection, and (3) return an object with an items field for future extension diff --git a/packages/catalog-client/package.json b/packages/catalog-client/package.json index c4aa632ca9..2482f9579d 100644 --- a/packages/catalog-client/package.json +++ b/packages/catalog-client/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/catalog-client", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -25,7 +25,7 @@ "cross-fetch": "^3.0.6" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/jest": "^26.0.7", "msw": "^0.21.2" }, diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 47d047fe9a..e712e66f58 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -29,7 +29,7 @@ "yup": "^0.29.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/express": "^4.17.6", "@types/jest": "^26.0.7", "@types/lodash": "^4.14.151", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 387f5e416f..a5bcc6bcfd 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,38 @@ # @backstage/cli +## 0.3.0 + +### Minor Changes + +- 1722cb53c: Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas. + + The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. + + A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. + + Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format. + + TypeScript configuration schema files should export a single `Config` type, for example: + + ```ts + export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; + } + ``` + +### Patch Changes + +- 1722cb53c: Added configuration schema +- 902340451: Support specifying listen host/port for frontend +- Updated dependencies [1722cb53c] + - @backstage/config-loader@0.3.0 + ## 0.2.0 ### Minor Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 516e354012..ec3aaacdee 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/cli", "description": "CLI for developing Backstage plugins and apps", - "version": "0.2.0", + "version": "0.3.0", "private": false, "publishConfig": { "access": "public" @@ -30,7 +30,7 @@ "dependencies": { "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", - "@backstage/config-loader": "^0.2.0", + "@backstage/config-loader": "^0.3.0", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^3.18.5", "@lerna/project": "^3.18.0", @@ -109,11 +109,11 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.2.1", + "@backstage/backend-common": "^0.3.0", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.3.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/core": "^0.3.1", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@types/diff": "^4.0.2", "@types/fs-extra": "^9.0.1", diff --git a/packages/config-loader/CHANGELOG.md b/packages/config-loader/CHANGELOG.md index 9c6bee077f..729dc81206 100644 --- a/packages/config-loader/CHANGELOG.md +++ b/packages/config-loader/CHANGELOG.md @@ -1,5 +1,31 @@ # @backstage/config-loader +## 0.3.0 + +### Minor Changes + +- 1722cb53c: Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas. + + The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. + + A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. + + Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format. + + TypeScript configuration schema files should export a single `Config` type, for example: + + ```ts + export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; + } + ``` + ## 0.2.0 ### Minor Changes diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 98583cf833..1584a61df5 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config-loader", "description": "Config loading functionality used by Backstage backend, and CLI", - "version": "0.2.0", + "version": "0.3.0", "private": false, "publishConfig": { "access": "public", diff --git a/packages/core-api/package.json b/packages/core-api/package.json index 98eaa0fa61..212d4c9046 100644 --- a/packages/core-api/package.json +++ b/packages/core-api/package.json @@ -30,7 +30,7 @@ }, "dependencies": { "@backstage/config": "^0.1.1", - "@backstage/test-utils": "^0.1.2", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -42,7 +42,7 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@backstage/test-utils-core": "^0.1.1", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 435b344a6a..213e2b5246 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/core +## 0.3.1 + +### Patch Changes + +- 1722cb53c: Added configuration schema + ## 0.3.0 ### Minor Changes diff --git a/packages/core/package.json b/packages/core/package.json index d0b623ea83..70c1a9e647 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core", "description": "Core API used by Backstage plugins and apps", - "version": "0.3.0", + "version": "0.3.1", "private": false, "publishConfig": { "access": "public", @@ -63,8 +63,8 @@ "remark-gfm": "^1.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/packages/create-app/CHANGELOG.md b/packages/create-app/CHANGELOG.md index 94b0071e86..4526bd4733 100644 --- a/packages/create-app/CHANGELOG.md +++ b/packages/create-app/CHANGELOG.md @@ -1,5 +1,110 @@ # @backstage/create-app +## 0.2.2 + +### Patch Changes + +- 7d7abd50c: Add `app-backend` as a backend plugin, and make a single docker build of the backend the default way to deploy backstage. + + Note that the `app-backend` currently only is a solution for deployments of the app, it's not a dev server and is not intended for local development. + + ## Template changes + + As a part of installing the `app-backend` plugin, the below changes where made. The changes are grouped into two steps, installing the plugin, and updating the Docker build and configuration. + + ### Installing the `app-backend` plugin in the backend + + First, install the `@backstage/plugin-app-backend` plugin package in your backend. These changes where made for `v0.3.0` of the plugin, and the installation process might change in the future. Run the following from the root of the repo: + + ```bash + cd packages/backend + yarn add @backstage/plugin-app-backend + ``` + + For the `app-backend` to get access to the static content in the frontend we also need to add the local `app` package as a dependency. Add the following to your `"dependencies"` in `packages/backend/package.json`, assuming your app package is still named `app` and on version `0.0.0`: + + ```json + "app": "0.0.0", + ``` + + Don't worry, this will not cause your entire frontend dependency tree to be added to the app, just double check that `packages/app/package.json` has a `"bundled": true` field at top-level. This signals to the backend build process that the package is bundled and that no transitive dependencies should be included. + + Next, create `packages/backend/src/plugins/app.ts` with the following: + + ```ts + import { createRouter } from '@backstage/plugin-app-backend'; + import { PluginEnvironment } from '../types'; + + export default async function createPlugin({ + logger, + config, + }: PluginEnvironment) { + return await createRouter({ + logger, + config, + appPackageName: 'app', + }); + } + ``` + + In `packages/backend/src/index.ts`, make the following changes: + + Add an import for the newly created plugin setup file: + + ```ts + import app from './plugins/app'; + ``` + + Setup the following plugin env. + + ```ts + const appEnv = useHotMemoize(module, () => createEnv('app')); + ``` + + Change service builder setup to include the `app` plugin as follows. Note that the `app` plugin is not installed on the `/api` route with most other plugins. + + ```ts + const service = createServiceBuilder(module) + .loadConfig(config) + .addRouter('/api', apiRouter) + .addRouter('', await app(appEnv)); + ``` + + You should now have the `app-backend` plugin installed in your backend, ready to serve the frontend bundle! + + ### Docker build setup + + Since the backend image is now the only one needed for a simple Backstage deployment, the image tag name in the `build-image` script inside `packages/backend/package.json` was changed to the following: + + ```json + "build-image": "backstage-cli backend:build-image --build --tag backstage", + ``` + + For convenience, a `build-image` script was also added to the root `package.json` with the following: + + ```json + "build-image": "yarn workspace backend build-image", + ``` + + In the root of the repo, a new `app-config.production.yaml` file was added. This is used to set the appropriate `app.baseUrl` now that the frontend is served directly by the backend in the production deployment. It has the following contents: + + ```yaml + app: + # Should be the same as backend.baseUrl when using the `app-backend` plugin + baseUrl: http://localhost:7000 + + backend: + baseUrl: http://localhost:7000 + listen: + port: 7000 + ``` + + In order to load in the new configuration at runtime, the command in the `Dockerfile` at the repo root was changed to the following: + + ```dockerfile + CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] + ``` + ## 0.2.1 ### Patch Changes diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 55cdcb1a2a..c01d4d3dbb 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/create-app", "description": "Create app package for Backstage", - "version": "0.2.1", + "version": "0.2.2", "private": false, "publishConfig": { "access": "public" @@ -37,30 +37,30 @@ "recursive-readdir": "^2.2.2" }, "devDependencies": { - "@backstage/backend-common": "^0.2.1", + "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/plugin-api-docs": "^0.2.1", - "@backstage/plugin-app-backend": "^0.2.0", - "@backstage/plugin-auth-backend": "^0.2.1", - "@backstage/plugin-catalog": "^0.2.1", - "@backstage/plugin-catalog-backend": "^0.2.0", + "@backstage/plugin-app-backend": "^0.3.0", + "@backstage/plugin-auth-backend": "^0.2.2", + "@backstage/plugin-catalog": "^0.2.2", + "@backstage/plugin-catalog-backend": "^0.2.1", "@backstage/plugin-circleci": "^0.2.1", "@backstage/plugin-explore": "^0.2.1", "@backstage/plugin-github-actions": "^0.2.1", - "@backstage/plugin-lighthouse": "^0.2.1", - "@backstage/plugin-proxy-backend": "^0.2.0", + "@backstage/plugin-lighthouse": "^0.2.2", + "@backstage/plugin-proxy-backend": "^0.2.1", "@backstage/plugin-register-component": "^0.2.1", - "@backstage/plugin-rollbar-backend": "^0.1.2", + "@backstage/plugin-rollbar-backend": "^0.1.3", "@backstage/plugin-scaffolder": "^0.3.0", - "@backstage/plugin-scaffolder-backend": "^0.3.0", + "@backstage/plugin-scaffolder-backend": "^0.3.1", "@backstage/plugin-tech-radar": "^0.3.0", - "@backstage/plugin-techdocs": "^0.2.1", - "@backstage/plugin-techdocs-backend": "^0.2.0", - "@backstage/plugin-user-settings": "^0.2.1", - "@backstage/test-utils": "^0.1.2", + "@backstage/plugin-techdocs": "^0.2.2", + "@backstage/plugin-techdocs-backend": "^0.2.1", + "@backstage/plugin-user-settings": "^0.2.2", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@types/fs-extra": "^9.0.1", "@types/inquirer": "^7.3.1", diff --git a/packages/dev-utils/CHANGELOG.md b/packages/dev-utils/CHANGELOG.md index 274a61bacb..fd8b11bada 100644 --- a/packages/dev-utils/CHANGELOG.md +++ b/packages/dev-utils/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/dev-utils +## 0.1.4 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [902340451] + - @backstage/cli@0.3.0 + - @backstage/core@0.3.1 + - @backstage/test-utils@0.1.3 + ## 0.1.3 ### Patch Changes diff --git a/packages/dev-utils/package.json b/packages/dev-utils/package.json index cf25d2e296..bf2d230ba7 100644 --- a/packages/dev-utils/package.json +++ b/packages/dev-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/dev-utils", "description": "Utilities for developing Backstage plugins.", - "version": "0.1.3", + "version": "0.1.4", "private": false, "publishConfig": { "access": "public", @@ -29,9 +29,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/core": "^0.3.1", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/packages/integration/CHANGELOG.md b/packages/integration/CHANGELOG.md new file mode 100644 index 0000000000..fde914fac7 --- /dev/null +++ b/packages/integration/CHANGELOG.md @@ -0,0 +1,7 @@ +# @backstage/integration + +## 0.1.1 + +### Patch Changes + +- 7b37e6834: Added the integration package diff --git a/packages/integration/package.json b/packages/integration/package.json index cfbdcbe57c..2bc65c46a3 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/integration", - "version": "0.1.0", + "version": "0.1.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -24,7 +24,7 @@ "git-url-parse": "^11.4.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/jest": "^26.0.7" }, "files": [ diff --git a/packages/test-utils/CHANGELOG.md b/packages/test-utils/CHANGELOG.md index 0d71e2ddd3..b4bc2bbe76 100644 --- a/packages/test-utils/CHANGELOG.md +++ b/packages/test-utils/CHANGELOG.md @@ -1,5 +1,14 @@ # @backstage/test-utils +## 0.1.3 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [902340451] + - @backstage/cli@0.3.0 + ## 0.1.2 ### Patch Changes diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 9a7682c76b..a8f5cd1661 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/test-utils", "description": "Utilities to test Backstage plugins and apps.", - "version": "0.1.2", + "version": "0.1.3", "private": false, "publishConfig": { "access": "public", @@ -29,7 +29,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@backstage/core-api": "^0.2.0", "@backstage/test-utils-core": "^0.1.1", "@backstage/theme": "^0.2.0", diff --git a/packages/theme/package.json b/packages/theme/package.json index f04ec4d589..68c70c656a 100644 --- a/packages/theme/package.json +++ b/packages/theme/package.json @@ -31,7 +31,7 @@ "@material-ui/core": "^4.11.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0" + "@backstage/cli": "^0.3.0" }, "files": [ "dist" diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json index 73764f70ee..bfefaef3a5 100644 --- a/plugins/api-docs/package.json +++ b/plugins/api-docs/package.json @@ -21,8 +21,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@kyma-project/asyncapi-react": "^0.14.2", "@material-icons/font": "^1.0.2", @@ -39,9 +39,9 @@ "swagger-ui-react": "^3.31.1" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/app-backend/CHANGELOG.md b/plugins/app-backend/CHANGELOG.md index 03ecafd574..3555da1959 100644 --- a/plugins/app-backend/CHANGELOG.md +++ b/plugins/app-backend/CHANGELOG.md @@ -1,5 +1,22 @@ # @backstage/plugin-app-backend +## 0.3.0 + +### Minor Changes + +- 1722cb53c: Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection. + + This removes the confusing behavior where configuration was only injected into the app at build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + - @backstage/config-loader@0.3.0 + ## 0.2.0 ### Minor Changes diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index 08266acb68..bf6167d46a 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-app-backend", - "version": "0.2.0", + "version": "0.3.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,8 +20,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", - "@backstage/config-loader": "^0.2.0", + "@backstage/backend-common": "^0.3.0", + "@backstage/config-loader": "^0.3.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", @@ -31,7 +31,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/supertest": "^2.0.8", "msw": "^0.20.5", "supertest": "^4.0.2" diff --git a/plugins/auth-backend/CHANGELOG.md b/plugins/auth-backend/CHANGELOG.md index e38bc15fe9..546855c92f 100644 --- a/plugins/auth-backend/CHANGELOG.md +++ b/plugins/auth-backend/CHANGELOG.md @@ -1,5 +1,17 @@ # @backstage/plugin-auth-backend +## 0.2.2 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] +- Updated dependencies [717e43de1] + - @backstage/backend-common@0.3.0 + - @backstage/catalog-client@0.3.0 + ## 0.2.1 ### Patch Changes diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 28ac51034e..8ce955350a 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-auth-backend", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,8 +20,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.1", - "@backstage/catalog-client": "^0.2.0", + "@backstage/backend-common": "^0.3.0", + "@backstage/catalog-client": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", @@ -53,7 +53,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/body-parser": "^1.19.0", "@types/cookie-parser": "^1.4.2", "@types/jwt-decode": "2.2.1", diff --git a/plugins/catalog-backend/CHANGELOG.md b/plugins/catalog-backend/CHANGELOG.md index e45da37691..1a3199ee14 100644 --- a/plugins/catalog-backend/CHANGELOG.md +++ b/plugins/catalog-backend/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-catalog-backend +## 0.2.1 + +### Patch Changes + +- f531d307c: An entity A, that exists in the catalog, can no longer be overwritten by registering a different location that also tries to supply an entity with the same kind+namespace+name. Writes of that new entity will instead be rejected with a log message similar to `Rejecting write of entity Component:default/artist-lookup from file:/Users/freben/dev/github/backstage/packages/catalog-model/examples/components/artist-lookup-component.yaml because entity existed from github:https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml` +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.2.0 ### Minor Changes diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 390c91551f..711a91dd8d 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@octokit/graphql": "^4.5.6", @@ -45,8 +45,8 @@ "yup": "^0.29.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/test-utils": "^0.1.3", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-graphql/CHANGELOG.md b/plugins/catalog-graphql/CHANGELOG.md index 30414d6998..35737b7dd1 100644 --- a/plugins/catalog-graphql/CHANGELOG.md +++ b/plugins/catalog-graphql/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-catalog-graphql +## 0.2.1 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.2.0 ### Minor Changes diff --git a/plugins/catalog-graphql/package.json b/plugins/catalog-graphql/package.json index aaa5faac36..c55be89b62 100644 --- a/plugins/catalog-graphql/package.json +++ b/plugins/catalog-graphql/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-graphql", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@graphql-modules/core": "^0.7.17", @@ -32,8 +32,8 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/test-utils": "^0.1.3", "@graphql-codegen/cli": "^1.17.7", "@graphql-codegen/typescript": "^1.17.7", "@graphql-codegen/typescript-resolvers": "^1.17.7", diff --git a/plugins/catalog/CHANGELOG.md b/plugins/catalog/CHANGELOG.md index 3e808478c8..1b8b884f1c 100644 --- a/plugins/catalog/CHANGELOG.md +++ b/plugins/catalog/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-catalog +## 0.2.2 + +### Patch Changes + +- 8b7737d0b: Add About Card tooltips +- Updated dependencies [1722cb53c] +- Updated dependencies [717e43de1] + - @backstage/core@0.3.1 + - @backstage/plugin-techdocs@0.2.2 + - @backstage/catalog-client@0.3.0 + ## 0.2.1 ### Patch Changes diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index 02ab709f79..9ee5d68270 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,11 +21,11 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/catalog-client": "^0.2.0", + "@backstage/catalog-client": "^0.3.0", "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/plugin-scaffolder": "^0.3.0", - "@backstage/plugin-techdocs": "^0.2.1", + "@backstage/plugin-techdocs": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -43,9 +43,9 @@ "swr": "^0.3.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/react-hooks": "^3.3.0", diff --git a/plugins/circleci/package.json b/plugins/circleci/package.json index fa31fe90f7..c9ad54659f 100644 --- a/plugins/circleci/package.json +++ b/plugins/circleci/package.json @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -38,9 +38,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/cloudbuild/package.json b/plugins/cloudbuild/package.json index cb7dab8e91..c0b36ef1e7 100644 --- a/plugins/cloudbuild/package.json +++ b/plugins/cloudbuild/package.json @@ -21,8 +21,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -39,9 +39,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/cost-insights/CHANGELOG.md b/plugins/cost-insights/CHANGELOG.md index cb7b2df919..157d92f251 100644 --- a/plugins/cost-insights/CHANGELOG.md +++ b/plugins/cost-insights/CHANGELOG.md @@ -1,5 +1,22 @@ # @backstage/plugin-cost-insights +## 0.4.0 + +### Minor Changes + +- 4040d4fcb: remove cost insights currency feature flag + +### Patch Changes + +- 1722cb53c: Added configuration schema +- 17a9f48f6: remove excessive margin from cost overview banner +- f360395d0: UI improvements: Increase width of first column in product entity dialog table + UI improvement: Display full cost amount in product entity dialog table +- 259d848ee: Fix savings/excess display calculation +- Updated dependencies [1722cb53c] + - @backstage/core@0.3.1 + - @backstage/test-utils@0.1.3 + ## 0.3.0 ### Minor Changes diff --git a/plugins/cost-insights/package.json b/plugins/cost-insights/package.json index 1d7850482c..8ebeac91c0 100644 --- a/plugins/cost-insights/package.json +++ b/plugins/cost-insights/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-cost-insights", - "version": "0.3.0", + "version": "0.4.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/config": "^0.1.1", - "@backstage/core": "^0.3.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/core": "^0.3.1", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -46,9 +46,9 @@ "yup": "^0.29.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/explore/package.json b/plugins/explore/package.json index 7770341ffb..153184bdb1 100644 --- a/plugins/explore/package.json +++ b/plugins/explore/package.json @@ -21,7 +21,7 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -33,9 +33,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/gcp-projects/package.json b/plugins/gcp-projects/package.json index e4af2bd4a3..0415a5ffb3 100644 --- a/plugins/gcp-projects/package.json +++ b/plugins/gcp-projects/package.json @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -31,9 +31,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index e0db90453a..49ecfc5423 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -22,9 +22,9 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/core-api": "^0.2.1", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -40,9 +40,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/gitops-profiles/package.json b/plugins/gitops-profiles/package.json index 167970f830..ad80d26e61 100644 --- a/plugins/gitops-profiles/package.json +++ b/plugins/gitops-profiles/package.json @@ -21,7 +21,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -32,9 +32,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/graphiql/package.json b/plugins/graphiql/package.json index be20cfb635..a5523afc6f 100644 --- a/plugins/graphiql/package.json +++ b/plugins/graphiql/package.json @@ -31,7 +31,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -43,9 +43,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/graphql/CHANGELOG.md b/plugins/graphql/CHANGELOG.md index f9a626d543..477bd1bb8b 100644 --- a/plugins/graphql/CHANGELOG.md +++ b/plugins/graphql/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-graphql-backend +## 0.1.3 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + - @backstage/plugin-catalog-graphql@0.2.1 + ## 0.1.2 ### Patch Changes diff --git a/plugins/graphql/package.json b/plugins/graphql/package.json index f62fecc648..53af8d9154 100644 --- a/plugins/graphql/package.json +++ b/plugins/graphql/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-graphql-backend", - "version": "0.1.2", + "version": "0.1.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -19,9 +19,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/config": "^0.1.1", - "@backstage/plugin-catalog-graphql": "^0.2.0", + "@backstage/plugin-catalog-graphql": "^0.2.1", "@graphql-modules/core": "^0.7.17", "@types/express": "^4.17.6", "apollo-server": "^2.16.1", @@ -35,7 +35,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/supertest": "^2.0.8", "eslint-plugin-graphql": "^4.0.0", "msw": "^0.20.5", diff --git a/plugins/jenkins/package.json b/plugins/jenkins/package.json index b4f1d25199..0279592a75 100644 --- a/plugins/jenkins/package.json +++ b/plugins/jenkins/package.json @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -36,9 +36,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/kubernetes-backend/CHANGELOG.md b/plugins/kubernetes-backend/CHANGELOG.md index 2834082657..7c55a7407d 100644 --- a/plugins/kubernetes-backend/CHANGELOG.md +++ b/plugins/kubernetes-backend/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-kubernetes-backend +## 0.1.3 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.1.2 ### Patch Changes diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index 7ac5f99f4c..e58bb4f047 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-kubernetes-backend", - "version": "0.1.2", + "version": "0.1.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/config": "^0.1.1", "@kubernetes/client-node": "^0.12.1", "@types/express": "^4.17.6", @@ -37,7 +37,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "supertest": "^4.0.2" }, "files": [ diff --git a/plugins/kubernetes/package.json b/plugins/kubernetes/package.json index c1457944de..36954cc5b9 100644 --- a/plugins/kubernetes/package.json +++ b/plugins/kubernetes/package.json @@ -22,8 +22,8 @@ "dependencies": { "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.3.0", - "@backstage/plugin-kubernetes-backend": "^0.1.2", + "@backstage/core": "^0.3.1", + "@backstage/plugin-kubernetes-backend": "^0.1.3", "@backstage/theme": "^0.2.1", "@kubernetes/client-node": "^0.12.1", "@material-ui/core": "^4.11.0", @@ -35,9 +35,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/lighthouse/CHANGELOG.md b/plugins/lighthouse/CHANGELOG.md index 3c5e94c476..666e7c4411 100644 --- a/plugins/lighthouse/CHANGELOG.md +++ b/plugins/lighthouse/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-lighthouse +## 0.2.2 + +### Patch Changes + +- 1722cb53c: Added configuration schema +- Updated dependencies [1722cb53c] +- Updated dependencies [8b7737d0b] + - @backstage/core@0.3.1 + - @backstage/plugin-catalog@0.2.2 + ## 0.2.1 ### Patch Changes diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 8172847120..90230f5248 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-lighthouse", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -23,9 +23,9 @@ "dependencies": { "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/core-api": "^0.2.1", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -38,9 +38,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/newrelic/package.json b/plugins/newrelic/package.json index 475f19d651..0e1f4e1352 100644 --- a/plugins/newrelic/package.json +++ b/plugins/newrelic/package.json @@ -21,7 +21,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -31,9 +31,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/proxy-backend/CHANGELOG.md b/plugins/proxy-backend/CHANGELOG.md index dcbe7eb7e8..b034d26c95 100644 --- a/plugins/proxy-backend/CHANGELOG.md +++ b/plugins/proxy-backend/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-proxy-backend +## 0.2.1 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.2.0 ### Minor Changes diff --git a/plugins/proxy-backend/package.json b/plugins/proxy-backend/package.json index f147e400e4..22acdd3fe8 100644 --- a/plugins/proxy-backend/package.json +++ b/plugins/proxy-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-proxy-backend", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -19,7 +19,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", @@ -33,7 +33,7 @@ "yup": "^0.29.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/http-proxy-middleware": "^0.19.3", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", diff --git a/plugins/register-component/package.json b/plugins/register-component/package.json index 9c87f33e29..06ae17cb0e 100644 --- a/plugins/register-component/package.json +++ b/plugins/register-component/package.json @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -36,9 +36,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/rollbar-backend/CHANGELOG.md b/plugins/rollbar-backend/CHANGELOG.md index 21f9ce8090..ec042b9ae1 100644 --- a/plugins/rollbar-backend/CHANGELOG.md +++ b/plugins/rollbar-backend/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-rollbar-backend +## 0.1.3 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.1.2 ### Patch Changes diff --git a/plugins/rollbar-backend/package.json b/plugins/rollbar-backend/package.json index b1764f238f..307c37e0da 100644 --- a/plugins/rollbar-backend/package.json +++ b/plugins/rollbar-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-rollbar-backend", - "version": "0.1.2", + "version": "0.1.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", "axios": "^0.20.0", @@ -37,7 +37,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@types/supertest": "^2.0.8", "supertest": "^4.0.2" }, diff --git a/plugins/rollbar/CHANGELOG.md b/plugins/rollbar/CHANGELOG.md index 3d53c1bf0d..e4a6671f90 100644 --- a/plugins/rollbar/CHANGELOG.md +++ b/plugins/rollbar/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-rollbar +## 0.2.2 + +### Patch Changes + +- 1722cb53c: Added configuration schema +- Updated dependencies [1722cb53c] +- Updated dependencies [8b7737d0b] + - @backstage/core@0.3.1 + - @backstage/plugin-catalog@0.2.2 + ## 0.2.1 ### Patch Changes diff --git a/plugins/rollbar/package.json b/plugins/rollbar/package.json index 18e4de3024..c107573ca8 100644 --- a/plugins/rollbar/package.json +++ b/plugins/rollbar/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-rollbar", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -37,9 +37,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/react-hooks": "^3.3.0", diff --git a/plugins/scaffolder-backend/CHANGELOG.md b/plugins/scaffolder-backend/CHANGELOG.md index 96f4ef75f7..a388a29a00 100644 --- a/plugins/scaffolder-backend/CHANGELOG.md +++ b/plugins/scaffolder-backend/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-scaffolder-backend +## 0.3.1 + +### Patch Changes + +- d33f5157c: Extracted pushToRemote function for reuse between publishers +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.3.0 ### Minor Changes diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 4bed7ea60c..73c07d158f 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder-backend", - "version": "0.3.0", + "version": "0.3.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.1", + "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@gitbeaker/core": "^25.2.0", @@ -48,7 +48,7 @@ "cross-fetch": "^3.0.6" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "@octokit/types": "^5.4.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index a423c64aa3..439fc0ad9a 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -22,8 +22,8 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", - "@backstage/plugin-catalog": "^0.2.1", + "@backstage/core": "^0.3.1", + "@backstage/plugin-catalog": "^0.2.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -41,9 +41,9 @@ "swr": "^0.3.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/search/package.json b/plugins/search/package.json index 6f356d1bf4..27620f2879 100644 --- a/plugins/search/package.json +++ b/plugins/search/package.json @@ -20,21 +20,21 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", - "@backstage/plugin-catalog": "^0.2.0", + "@backstage/plugin-catalog": "^0.2.2", "react-router-dom": "6.0.0-beta.0", "react": "^16.13.1", "react-dom": "^16.13.1", "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/sentry-backend/CHANGELOG.md b/plugins/sentry-backend/CHANGELOG.md index 93361e05ff..9ca0a5415e 100644 --- a/plugins/sentry-backend/CHANGELOG.md +++ b/plugins/sentry-backend/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-sentry-backend +## 0.1.3 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.1.2 ### Patch Changes diff --git a/plugins/sentry-backend/package.json b/plugins/sentry-backend/package.json index a1e1531b8b..2b6b5fc60d 100644 --- a/plugins/sentry-backend/package.json +++ b/plugins/sentry-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sentry-backend", - "version": "0.1.2", + "version": "0.1.3", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@types/express": "^4.17.6", "axios": "^0.20.0", "compression": "^1.7.4", @@ -34,7 +34,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0" + "@backstage/cli": "^0.3.0" }, "files": [ "dist" diff --git a/plugins/sentry/CHANGELOG.md b/plugins/sentry/CHANGELOG.md index 88adfe43a1..92327c1668 100644 --- a/plugins/sentry/CHANGELOG.md +++ b/plugins/sentry/CHANGELOG.md @@ -1,5 +1,13 @@ # @backstage/plugin-sentry +## 0.2.2 + +### Patch Changes + +- 1722cb53c: Added configuration schema +- Updated dependencies [1722cb53c] + - @backstage/core@0.3.1 + ## 0.2.1 ### Patch Changes diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index 2757c02db9..80a37c574e 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-sentry", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,7 +22,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -36,9 +36,9 @@ "timeago.js": "^4.0.2" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/sonarqube/package.json b/plugins/sonarqube/package.json index d7240aabde..dcde8fab14 100644 --- a/plugins/sonarqube/package.json +++ b/plugins/sonarqube/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -35,9 +35,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/tech-radar/package.json b/plugins/tech-radar/package.json index 431fd3d515..7f5707764b 100644 --- a/plugins/tech-radar/package.json +++ b/plugins/tech-radar/package.json @@ -21,8 +21,8 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.3.0", - "@backstage/test-utils": "^0.1.2", + "@backstage/core": "^0.3.1", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -35,9 +35,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/techdocs-backend/CHANGELOG.md b/plugins/techdocs-backend/CHANGELOG.md index 4191361a33..4a3fbf6821 100644 --- a/plugins/techdocs-backend/CHANGELOG.md +++ b/plugins/techdocs-backend/CHANGELOG.md @@ -1,5 +1,15 @@ # @backstage/plugin-techdocs-backend +## 0.2.1 + +### Patch Changes + +- Updated dependencies [1722cb53c] +- Updated dependencies [1722cb53c] +- Updated dependencies [7b37e6834] +- Updated dependencies [8e2effb53] + - @backstage/backend-common@0.3.0 + ## 0.2.0 ### Minor Changes diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index c1d1f6ad27..b139bb9acf 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs-backend", - "version": "0.2.0", + "version": "0.2.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -20,7 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.2.0", + "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@types/dockerode": "^2.5.34", @@ -37,7 +37,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.2.0", + "@backstage/cli": "^0.3.0", "supertest": "^4.0.2" }, "files": [ diff --git a/plugins/techdocs/CHANGELOG.md b/plugins/techdocs/CHANGELOG.md index 1c1dcb5c8a..bd3ae21ab8 100644 --- a/plugins/techdocs/CHANGELOG.md +++ b/plugins/techdocs/CHANGELOG.md @@ -1,5 +1,16 @@ # @backstage/plugin-techdocs +## 0.2.2 + +### Patch Changes + +- 1722cb53c: Added configuration schema +- Updated dependencies [1722cb53c] +- Updated dependencies [8b7737d0b] + - @backstage/core@0.3.1 + - @backstage/plugin-catalog@0.2.2 + - @backstage/test-utils@0.1.3 + ## 0.2.1 ### Patch Changes diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index a2b1349d53..35b196e124 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-techdocs", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -22,10 +22,10 @@ }, "dependencies": { "@backstage/catalog-model": "^0.2.0", - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/core-api": "^0.2.1", - "@backstage/plugin-catalog": "^0.2.1", - "@backstage/test-utils": "^0.1.2", + "@backstage/plugin-catalog": "^0.2.2", + "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -39,9 +39,9 @@ "sanitize-html": "^1.27.0" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/user-settings/CHANGELOG.md b/plugins/user-settings/CHANGELOG.md index 729d2a669a..9efbc6eba7 100644 --- a/plugins/user-settings/CHANGELOG.md +++ b/plugins/user-settings/CHANGELOG.md @@ -1,5 +1,13 @@ # @backstage/plugin-user-settings +## 0.2.2 + +### Patch Changes + +- 1722cb53c: Added configuration schema +- Updated dependencies [1722cb53c] + - @backstage/core@0.3.1 + ## 0.2.1 ### Patch Changes diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index f82a6b2b6f..1277ae214a 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-user-settings", - "version": "0.2.1", + "version": "0.2.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,7 +21,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -32,9 +32,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/plugins/welcome/package.json b/plugins/welcome/package.json index fcfbed2af7..47b02f971c 100644 --- a/plugins/welcome/package.json +++ b/plugins/welcome/package.json @@ -21,7 +21,7 @@ "start": "backstage-cli plugin:serve" }, "dependencies": { - "@backstage/core": "^0.3.0", + "@backstage/core": "^0.3.1", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", @@ -32,9 +32,9 @@ "react-use": "^15.3.3" }, "devDependencies": { - "@backstage/cli": "^0.2.0", - "@backstage/dev-utils": "^0.1.3", - "@backstage/test-utils": "^0.1.2", + "@backstage/cli": "^0.3.0", + "@backstage/dev-utils": "^0.1.4", + "@backstage/test-utils": "^0.1.3", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", From 69c924f24ae834d71a1534235c5f85ec321e0c6f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 19 Nov 2020 16:37:51 +0100 Subject: [PATCH 163/259] chore: fixing issue with resolutions for new versions of backstage --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index b97462114c..4140886bc3 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ ] }, "resolutions": { - "**/@roadiehq/backstage-plugin-*/@backstage/core": "0.3.0" + "**/@roadiehq/backstage-plugin-*/@backstage/core": "*" }, "version": "1.0.0", "devDependencies": { diff --git a/yarn.lock b/yarn.lock index bc9eea6285..de6ec42c7c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1290,7 +1290,7 @@ to-fast-properties "^2.0.0" "@backstage/core@^0.2.0": - version "0.3.0" + version "0.3.1" dependencies: "@backstage/config" "^0.1.1" "@backstage/core-api" "^0.2.1" From 1adfd1cfacade037d69b3c79ff31b57ebf60e41c Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak Date: Thu, 19 Nov 2020 17:57:29 +0100 Subject: [PATCH 164/259] Feat: Add Buildkite plugin, update plugin logotypes --- microsite/data/plugins/aws-lambda.yaml | 2 +- microsite/data/plugins/buildkite.yaml | 12 ++++++++++++ microsite/data/plugins/firebase-functions.yaml | 2 +- microsite/data/plugins/github-insights.yaml | 2 +- microsite/data/plugins/github-pull-requests.yaml | 2 +- microsite/data/plugins/security-insights.yaml | 2 +- microsite/data/plugins/travis-ci.yaml | 2 +- 7 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 microsite/data/plugins/buildkite.yaml diff --git a/microsite/data/plugins/aws-lambda.yaml b/microsite/data/plugins/aws-lambda.yaml index 7fa276f444..f325c0df8b 100644 --- a/microsite/data/plugins/aws-lambda.yaml +++ b/microsite/data/plugins/aws-lambda.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io category: Monitoring description: View AWS Lambda functions for your components in Backstage. documentation: https://roadie.io/backstage/plugins/aws-lambda -iconUrl: https://roadie.io/static/77f62f79e27ae8565496e4df7eef8be5/45f2b/logo.png +iconUrl: https://roadie.io/images/logos/lambda.png npmPackageName: '@roadiehq/backstage-plugin-aws-lambda' diff --git a/microsite/data/plugins/buildkite.yaml b/microsite/data/plugins/buildkite.yaml new file mode 100644 index 0000000000..eeb9440aac --- /dev/null +++ b/microsite/data/plugins/buildkite.yaml @@ -0,0 +1,12 @@ +--- +title: Buildkite +author: roadie.io +authorUrl: https://roadie.io +category: CI +description: View Buildkite CI builds for your service in Backstage. +documentation: https://roadie.io/backstage/plugins/buildkite +iconUrl: https://roadie.io/images/logos/buildkite.png +npmPackageName: '@roadiehq/backstage-plugin-buildkite' +tags: + - ci + - cd diff --git a/microsite/data/plugins/firebase-functions.yaml b/microsite/data/plugins/firebase-functions.yaml index 097ef00e50..6d399a2350 100644 --- a/microsite/data/plugins/firebase-functions.yaml +++ b/microsite/data/plugins/firebase-functions.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io/ category: Monitoring description: View Firebase Functions details for your service in Backstage. documentation: https://roadie.io/backstage/plugins/firebase-functions -iconUrl: https://roadie.io/static/49fb23200ad0eaa6703b4ddf75c78cf1/45f2b/logo-vertical.png +iconUrl: https://roadie.io/images/logos/github.png npmPackageName: '@roadiehq/backstage-plugin-firebase-functions' diff --git a/microsite/data/plugins/github-insights.yaml b/microsite/data/plugins/github-insights.yaml index 8ac8321da0..1dba3dcfd1 100644 --- a/microsite/data/plugins/github-insights.yaml +++ b/microsite/data/plugins/github-insights.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io category: Monitoring description: View GitHub Insights for your components in Backstage. documentation: https://roadie.io/backstage/plugins/github-insights -iconUrl: https://roadie.io/static/2ad5123c425908efde0c922d707e737b/06c84/code-icon.png +iconUrl: https://roadie.io/images/logos/insights.png npmPackageName: '@roadiehq/backstage-plugin-github-insights' diff --git a/microsite/data/plugins/github-pull-requests.yaml b/microsite/data/plugins/github-pull-requests.yaml index 6479452c8e..8e44a4bf56 100644 --- a/microsite/data/plugins/github-pull-requests.yaml +++ b/microsite/data/plugins/github-pull-requests.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io/ category: CI description: View GitHub pull requests for your service in Backstage. documentation: https://roadie.io/backstage/plugins/github-pull-requests -iconUrl: https://roadie.io/static/7f13bb8d861d8dedc5112fb939d215f9/351f2/GitHub-Mark-Light-120px-plus.png +iconUrl: https://roadie.io/images/logos/github.png npmPackageName: '@roadiehq/backstage-plugin-github-pull-requests' diff --git a/microsite/data/plugins/security-insights.yaml b/microsite/data/plugins/security-insights.yaml index 1cbcfbc3aa..4f34f79027 100644 --- a/microsite/data/plugins/security-insights.yaml +++ b/microsite/data/plugins/security-insights.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io/ category: Security description: View Security Insights for your components in Backstage. documentation: https://roadie.io/backstage/plugins/security-insights -iconUrl: https://roadie.io/static/7f13bb8d861d8dedc5112fb939d215f9/351f2/GitHub-Mark-Light-120px-plus.png +iconUrl: https://roadie.io/images/logos/github.png npmPackageName: '@roadiehq/backstage-plugin-security-insights' diff --git a/microsite/data/plugins/travis-ci.yaml b/microsite/data/plugins/travis-ci.yaml index 520b884c20..fe1b18c9d4 100644 --- a/microsite/data/plugins/travis-ci.yaml +++ b/microsite/data/plugins/travis-ci.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io/ category: CI description: View Travis CI builds for your service in Backstage. documentation: https://roadie.io/backstage/plugins/travis-ci -iconUrl: https://roadie.io/static/af2941eaf0af675facb281d566f42e14/45f2b/travis-ci-mascot-200x200.png +iconUrl: https://roadie.io/images/logos/travis.png npmPackageName: '@roadiehq/backstage-plugin-travis-ci' From 79431253dea96244af77c321d700aa0cd83b48c3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 19 Nov 2020 18:51:17 +0100 Subject: [PATCH 165/259] Update Governance --- .github/styles/vocab.txt | 8 +++++ GOVERNANCE.md | 69 ++++++++++++++++++---------------------- MAINTAINERS.md | 21 ------------ OWNERS.md | 24 ++++++++++++++ 4 files changed, 63 insertions(+), 59 deletions(-) delete mode 100644 MAINTAINERS.md create mode 100644 OWNERS.md diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 9f6c7d386d..61ec1d364c 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -3,12 +3,14 @@ Apdex api Api apis +andrewthauer args asciidoc async Avro backrub Balachandran +benjdlambert Bigtable Blackbox bool @@ -50,6 +52,7 @@ Dockerfile Dockerize dockerode Docusaurus +dzolotusky eg Ek env @@ -59,6 +62,7 @@ facto failover Figma Firekube +freben Fredrik github Github @@ -66,6 +70,7 @@ gitlab Gitlab graphql graphviz +Gustavsson Hackathons haproxy heroku @@ -159,6 +164,7 @@ Rollup Rosaceae rst rsync +rugvip ruleset sam scaffolded @@ -174,6 +180,7 @@ spotify Spotify squidfunk src +stefanalund subkey superfences Superfences @@ -209,6 +216,7 @@ xyz yaml Zalando Zhou +Zolotusky Billett cloudbuild Grafana diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 19bbb3bcb9..65841d1c6a 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -1,52 +1,45 @@ -# Backstage Governance +# Process for becoming a maintainer -This document defines project governance for the project. +## Your organization is not yet a maintainer -## Maintainers +- Express interest to the sponsors that your organization is interested in becoming a maintainer. Becoming a maintainer generally means that you are going to be spending substantial time on Backstage for the foreseeable future. You should have domain expertise and be extremely proficient in TypeScript. +- We will expect you to start contributing increasingly complicated PRs, under the guidance of the existing maintainers. +- We may ask you to do some PRs from our backlog. +- As you gain experience with the code base and our standards, we will ask you to do code reviews for incoming PRs. +- After a period of approximately 2-3 months of working together and making sure we see eye to eye, the existing sponsors and maintainers will confer and decide whether to grant maintainer status or not. We make no guarantees on the length of time this will take, but 2-3 months is the approximate goal. -Backstage Maintainers have write access to the Backstage GitHub repository https://github.com/backstage/backstage. The current maintainers can be found in [MAINTAINERS](MAINTAINERS.md). - -This privilege is granted with some expectation of responsibility: maintainers are people who care about the Backstage project and want to help it grow and improve. A maintainer is not just someone who can make changes, but someone who has demonstrated his or her ability to collaborate with the team, get the most knowledgeable people to review code, contribute high-quality code, and follow through to fix issues (in code or tests). - -A maintainer is a contributor to the Backstage project's success and a citizen helping the project succeed. - -## Becoming a Maintainer +## Your organization is currently a maintainer To become a maintainer you need to demonstrate the following: -- commitment to the project - - participate in discussions, contributions, code reviews for 3 months or more, - - perform code reviews for 10 non-trivial pull requests, - - contribute 10 non-trivial pull requests and have them merged into master, -- ability to write good code, -- ability to collaborate with the team, -- understanding of how the team works (policies, processes for testing and code review, etc), -- understanding of the project's code base and coding style. +- First decide whether your organization really needs more people with maintainer access. Valid reasons are "blast radius", a large organization that is working on multiple unrelated projects, etc. +- Contact a sponsor for your organization and express interest. +- Start doing PRs and code reviews under the guidance of your maintainer. +- After a period of 1-2 months the existing sponsors will discuss granting maintainer access. +- Maintainer access can be upgraded to sponsor access after another conference of the existing sponsors. -## Changes in Maintainership +# Maintainer responsibilities -A new maintainer must be proposed by an existing maintainer by opening an issue (with title `Maintainer Nomination`) to the Backstage GitHub repository (https://github.com/backstage/backstage) containing the following information: +- Monitor email aliases. +- Monitor Discord (delayed response is perfectly acceptable). +- Triage GitHub issues and perform pull request reviews for other maintainers and the community. +- Triage build issues - file issues for known flaky builds or bugs, and either fix or find someone to fix any master build breakages. +- During GitHub issue triage, apply all applicable ([labels](https://github.com/backstage/backstage/labels)) to each new issue. Labels are extremely useful for future issue follow up. Which labels to apply is somewhat subjective so just use your best judgment. A few of the most important labels that are not self explanatory are: + - good first issue: Mark any issue that can reasonably be accomplished by a new contributor with this label. + - help wanted: Unless it is immediately obvious that someone is going to work on an issue (and if so assign it), mark it help wanted. +- Make sure that ongoing PRs are moving forward at the right pace or closing them. +- Participate when called upon in the security release process. Note that although this should be a rare occurrence, if a serious vulnerability is found, the process may take up to several full days of work to implement. This reality should be taken into account when discussing time commitment obligations with employers. +- In general, continue to be willing to spend at least 25% of one's time working on Backstage (~1.25 business days per week). +- We currently maintain an "on-call" rotation within the maintainers. Each on-call is 1 week. Although all maintainers are welcome to perform all of the above tasks, it is the on-call maintainer's responsibility to triage incoming issues/questions and marshal ongoing work forward. To reiterate, it is not the responsibility of the on-call maintainer to answer all questions and do all reviews, but it is their responsibility to make sure that everything is being actively covered by someone. -- nominee's first and last name, -- nominee's email address and GitHub user name, -- an explanation of why the nominee should be a maintainer, -- a list of links to non-trivial pull requests (top 10) authored by the nominee. +# When does a maintainer lose maintainer status -Maintainers can be removed by a 2/3 majority vote. +If a maintainer is no longer interested or cannot perform the maintainer duties listed above, they should volunteer to be moved to emeritus status. In extreme cases this can also occur by a vote of the sponsors and maintainers per the voting process below. -## Approving PRs +# Conflict resolution and voting -PRs may be merged after receiving at least one approval from a maintainer. +In general, we prefer that technical issues and maintainer membership are amicably worked out between the persons involved. If a dispute cannot be decided independently, the sponsors and maintainers can be called in to decide an issue. If the sponsors and maintainers themselves cannot decide an issue, the issue will be resolved by voting. The voting process is a simple majority in which each sponsor receives two votes and each maintainer receives one vote. -## GitHub Project Administration +# Adding new projects to the Backstage GitHub organization -Maintainers will be added to the collaborators list of the Backstage repository with "Write" access. - -## Changes in Governance - -All changes in Governance require a 2/3 majority vote. - -## Other Changes - -Unless specified above, all other changes to the project require a 2/3 majority vote. -Additionally, any maintainer may request that any change require a 2/3 majority vote. +New projects will be added to the Backstage organization via GitHub issue discussion in one of the existing projects in the organization. Once sufficient discussion has taken place (~3-5 business days but depending on the volume of conversation), the maintainers of the project where the issue was opened (since different projects in the organization may have different maintainers) will decide whether the new project should be added. See the section above on voting if the maintainers cannot easily decide. diff --git a/MAINTAINERS.md b/MAINTAINERS.md deleted file mode 100644 index 33437692c9..0000000000 --- a/MAINTAINERS.md +++ /dev/null @@ -1,21 +0,0 @@ -# Maintainers - -- See [CONTRIBUTING.md](CONTRIBUTING.md) for general contribution guidelines. - -## Current Maintainers 🏓 - -- Stefan Ålund - Spotify (GitHub: @stefanalund, Discord: @stalund) -- Patrik Oldsberg - Spotify (GitHub: @Rugvip, Discord: @Rugvip) -- Fredrik Adelöw - Spotify (GitHub: @freben, Discord: @freben) -- Ben Lambert - Spotify (GitHub: @benjdlambert, Discord: @blam) - -## Plugin maintainers 🧩 - -Teams and individuals that maintain a plugin (or another non-core module of the code) can get write access to that part of the repo using CODEOWNERS. - -## Hall of Fame 👏 - -People that have made significant contributions to the project and earned write access: - -- Andrew Thauer - Wealthsimple (GitHub: @andrewthauer) -- Oliver Sand - SDA SE (GitHub: @Fox32) diff --git a/OWNERS.md b/OWNERS.md new file mode 100644 index 0000000000..8e5002fc0a --- /dev/null +++ b/OWNERS.md @@ -0,0 +1,24 @@ +- See [CONTRIBUTING.md](CONTRIBUTING.md) for general contribution guidelines. +- See [GOVERNANCE.md](GOVERNANCE.md) for governance guidelines and responsibilities. + +This page lists all active sponsors and maintainers. + +# Sponsors + +- Niklas Gustavsson ([protocol7](https://github.com/protocol7)) (ngn@spotify.com) +- Dave Zolotusky ([dzolotusky](https://github.com/dzolotusky)) (dzolo@spotify.com) +- Lee Mills ([leemills83](https://github.com/leemills83)) (leem@spotify.com) + +# Maintainers + +- Patrik Oldsberg ([rugvip](https://github.com/rugvip)) (Discord: @Rugvip) +- Fredrik Adelöw ([freben](https://github.com/freben)) (Discord: @freben) +- Ben Lambert ([benjdlambert](https://github.com/benjdlambert)) (Discord: @blam) +- Stefan Ålund ([stefanalund](https://github.com/stefanalund)) (Discord: @stalund) + +# Friends of Backstage + +People that have made significant contributions to the project and earned write access. + +- Andrew Thauer - Wealthsimple (GitHub: [andrewthauer](https://github.com/andrewthauer)) +- Oliver Sand - SDA SE (GitHub: [Fox32](https://github.com/Fox32)) From aa22f839caa16275af3890273aad634646164ff3 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Thu, 19 Nov 2020 15:04:07 -0500 Subject: [PATCH 166/259] fix icon configuration The icon subfield of `products` is optional. The plugin provides a fallback icon for products that do not define them. --- plugins/cost-insights/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/cost-insights/config.d.ts b/plugins/cost-insights/config.d.ts index 0f147c2cfe..5099be43cb 100644 --- a/plugins/cost-insights/config.d.ts +++ b/plugins/cost-insights/config.d.ts @@ -31,7 +31,7 @@ interface Config { /** * @visibility frontend */ - icon: 'compute' | 'data' | 'database' | 'storage' | 'search' | 'ml'; + icon?: 'compute' | 'data' | 'database' | 'storage' | 'search' | 'ml'; }; }; From 8e6728e254620883f8c2cf721a4ebb01d0d61a2f Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Thu, 19 Nov 2020 15:07:44 -0500 Subject: [PATCH 167/259] changeset --- .changeset/cost-insights-quiet-bikes-occur.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cost-insights-quiet-bikes-occur.md diff --git a/.changeset/cost-insights-quiet-bikes-occur.md b/.changeset/cost-insights-quiet-bikes-occur.md new file mode 100644 index 0000000000..aac283bbd4 --- /dev/null +++ b/.changeset/cost-insights-quiet-bikes-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +fix product icon configuration From d90bbb8380ed2bc54e1aab840252e214751f36aa Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 19 Nov 2020 20:32:47 -0500 Subject: [PATCH 168/259] FAQ Updates --- docs/FAQ.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 7b906a341b..ba87891205 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -48,10 +48,9 @@ source candidates. (And we'll probably end up writing some brand new ones, too.) ### What's the roadmap for Backstage? We envision three phases, which you can learn about in -[our project roadmap](https://github.com/backstage/backstage#project-roadmap). -Even though the open source version of Backstage is relatively new compared to -our internal version, we have already begun work on various aspects of all three -phases. Looking at the +[our project roadmap](overview/roadmap/). Even though the open source version of +Backstage is relatively new compared to our internal version, we have already +begun work on various aspects of all three phases. Looking at the [milestones for active issues](https://github.com/backstage/backstage/milestones) will also give you a sense of our progress. @@ -115,8 +114,7 @@ type of content. Plugins all use a common set of platform APIs and reusable UI components. Plugins can fetch data either from the backend or an API exposed through the proxy. -Learn more about -[the different components](https://github.com/backstage/backstage#overview) that +Learn more about [the different components](overview/what-is-backstage) that make up Backstage. ### Do I have to write plugins in TypeScript? @@ -126,17 +124,17 @@ APIs in TypeScript, but aren't forcing it on individual plugins. ### How do I find out if a plugin already exists? -Before you write a plugin, +You can browse and search for all available plugins in the +[Plugin Marketplace](https://backstage.io/plugins). + +If you can't find it in the marketplace, before you write a plugin, [search the plugin issues](https://github.com/backstage/backstage/issues?q=is%3Aissue+label%3Aplugin+) -to see if it already exists or is in the works. If no one's thought of it yet, -great! Open a new issue as +to see if is in the works. If no one's thought of it yet, great! Open a new +issue as [a plugin suggestion](https://github.com/backstage/backstage/issues/new/choose) and describe what your plugin will do. This will help coordinate our contributors' efforts and avoid duplicating existing functionality. -You can browse and search for all available plugins in the -[Plugin Marketplace](https://backstage.io/plugins). - ### Which plugin is used the most at Spotify? By far, our most-used plugin is our TechDocs plugin, which we use for creating @@ -182,6 +180,10 @@ comes to [deployment](https://backstage.io/docs/getting-started/deployment-k8s), the system integrator (typically, the infrastructure team in your organization) maintains Backstage in your own environment. +For more information, see our +[Owners](https://github.com/backstage/backstage/blob/master/OWNERS.md) and +[Governance](https://github.com/backstage/backstage/blob/master/GOVERNANCE.md). + ### Does Spotify provide a managed version of Backstage? No, this is not a service offering. We build the piece of software, and someone @@ -215,14 +217,14 @@ data is shared with. Yes. The core frontend framework could be used for building any large-scale web application where (1) multiple teams are building separate parts of the app, and (2) you want the overall experience to be consistent. That being said, in -[Phase 2](https://github.com/backstage/backstage#project-roadmap) of the project -we will add features that are needed for developer portals and systems for -managing software ecosystems. Our ambition will be to keep Backstage modular. +[Phase 2](overview/roadmap) of the project we will add features that are needed +for developer portals and systems for managing software ecosystems. Our ambition +will be to keep Backstage modular. ### How can I get involved? Jump right in! Come help us fix some of the -[early bugs and first issues](https://github.com/backstage/backstage/labels/good%20first%20issue) +[early bugs and good first issues](https://github.com/backstage/backstage/contribute) or reach [a new milestone](https://github.com/backstage/backstage/milestones). Or write an open source plugin for Backstage, like this [Lighthouse plugin](https://github.com/backstage/backstage/tree/master/plugins/lighthouse). From 3cada418620b7788316c1eb819a6d79981b1d966 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 19 Nov 2020 20:34:54 -0500 Subject: [PATCH 169/259] Remove trailing slash --- docs/FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index ba87891205..d28923b9d7 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -48,7 +48,7 @@ source candidates. (And we'll probably end up writing some brand new ones, too.) ### What's the roadmap for Backstage? We envision three phases, which you can learn about in -[our project roadmap](overview/roadmap/). Even though the open source version of +[our project roadmap](overview/roadmap). Even though the open source version of Backstage is relatively new compared to our internal version, we have already begun work on various aspects of all three phases. Looking at the [milestones for active issues](https://github.com/backstage/backstage/milestones) From 90dfe05c82532bdb5c43d97e82803731aae6b291 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 19 Nov 2020 20:35:48 -0500 Subject: [PATCH 170/259] Remove extra comma --- docs/FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index d28923b9d7..9abe5ea1e7 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -127,7 +127,7 @@ APIs in TypeScript, but aren't forcing it on individual plugins. You can browse and search for all available plugins in the [Plugin Marketplace](https://backstage.io/plugins). -If you can't find it in the marketplace, before you write a plugin, +If you can't find it in the marketplace, before you write a plugin [search the plugin issues](https://github.com/backstage/backstage/issues?q=is%3Aissue+label%3Aplugin+) to see if is in the works. If no one's thought of it yet, great! Open a new issue as From 7f6ebc4c3238918ee7306eda7cb56f48cf3a712b Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 19 Nov 2020 20:38:53 -0500 Subject: [PATCH 171/259] Fix links for mkdocs/relative --- docs/FAQ.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 9abe5ea1e7..9c11b015ea 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -48,8 +48,8 @@ source candidates. (And we'll probably end up writing some brand new ones, too.) ### What's the roadmap for Backstage? We envision three phases, which you can learn about in -[our project roadmap](overview/roadmap). Even though the open source version of -Backstage is relatively new compared to our internal version, we have already +[our project roadmap](overview/roadmap.md). Even though the open source version +of Backstage is relatively new compared to our internal version, we have already begun work on various aspects of all three phases. Looking at the [milestones for active issues](https://github.com/backstage/backstage/milestones) will also give you a sense of our progress. @@ -114,7 +114,7 @@ type of content. Plugins all use a common set of platform APIs and reusable UI components. Plugins can fetch data either from the backend or an API exposed through the proxy. -Learn more about [the different components](overview/what-is-backstage) that +Learn more about [the different components](overview/what-is-backstage.md) that make up Backstage. ### Do I have to write plugins in TypeScript? @@ -217,9 +217,9 @@ data is shared with. Yes. The core frontend framework could be used for building any large-scale web application where (1) multiple teams are building separate parts of the app, and (2) you want the overall experience to be consistent. That being said, in -[Phase 2](overview/roadmap) of the project we will add features that are needed -for developer portals and systems for managing software ecosystems. Our ambition -will be to keep Backstage modular. +[Phase 2](overview/roadmap.md) of the project we will add features that are +needed for developer portals and systems for managing software ecosystems. Our +ambition will be to keep Backstage modular. ### How can I get involved? From 2a71f4babca0ffe7f5d560008322e0bc9f10c420 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Fri, 20 Nov 2020 03:28:19 -0500 Subject: [PATCH 172/259] register-component: Remove link to catalog item on validation popup (#3359) * Remove catalog link on validate popup * Add changeset --- .changeset/empty-kids-look.md | 5 +++++ .../RegisterComponentResultDialog.tsx | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changeset/empty-kids-look.md diff --git a/.changeset/empty-kids-look.md b/.changeset/empty-kids-look.md new file mode 100644 index 0000000000..680cd7f25c --- /dev/null +++ b/.changeset/empty-kids-look.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-register-component': patch +--- + +Remove catalog link on validate popup diff --git a/plugins/register-component/src/components/RegisterComponentResultDialog/RegisterComponentResultDialog.tsx b/plugins/register-component/src/components/RegisterComponentResultDialog/RegisterComponentResultDialog.tsx index 2b57050661..c31d1344cb 100644 --- a/plugins/register-component/src/components/RegisterComponentResultDialog/RegisterComponentResultDialog.tsx +++ b/plugins/register-component/src/components/RegisterComponentResultDialog/RegisterComponentResultDialog.tsx @@ -100,7 +100,9 @@ export const RegisterComponentResultDialog = ({ metadata={{ name: entity.metadata.name, type: entity.spec.type, - link: ( + link: dryRun ? ( + entityPath + ) : ( {entityPath} From b1a4e6a237938d3e8b282ee59082ea00e3b72a71 Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak Date: Fri, 20 Nov 2020 11:27:41 +0100 Subject: [PATCH 173/259] Feat: update Firebase logo --- microsite/data/plugins/firebase-functions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microsite/data/plugins/firebase-functions.yaml b/microsite/data/plugins/firebase-functions.yaml index 6d399a2350..8c1ea0be0a 100644 --- a/microsite/data/plugins/firebase-functions.yaml +++ b/microsite/data/plugins/firebase-functions.yaml @@ -5,5 +5,5 @@ authorUrl: https://roadie.io/ category: Monitoring description: View Firebase Functions details for your service in Backstage. documentation: https://roadie.io/backstage/plugins/firebase-functions -iconUrl: https://roadie.io/images/logos/github.png +iconUrl: https://roadie.io/images/logos/firebase.png npmPackageName: '@roadiehq/backstage-plugin-firebase-functions' From 8cc862ebfc4ee0dbcfe08ba86913a6f57be24166 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 10:25:24 +0100 Subject: [PATCH 174/259] cli: enable transformation of workspace packages outside the workspace root --- packages/cli/src/lib/bundler/config.ts | 34 ++++++++++++++++++---- packages/cli/src/lib/bundler/transforms.ts | 21 ++++++++----- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 9e150e0ccb..b5a27dd624 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -70,13 +70,27 @@ async function readBuildInfo() { }; } +async function loadLernaPackages(): Promise< + { name: string; location: string }[] +> { + const LernaProject = require('@lerna/project'); + const project = new LernaProject(cliPaths.targetDir); + return project.getPackages(); +} + export async function createConfig( paths: BundlingPaths, options: BundlingOptions, ): Promise { const { checksEnabled, isDev, frontendConfig } = options; - const { plugins, loaders } = transforms(options); + const packages = await loadLernaPackages(); + const { plugins, loaders } = transforms({ + ...options, + externalTransforms: packages.map(({ name }) => + cliPaths.resolveTargetRoot('node_modules', name), + ), + }); const baseUrl = frontendConfig.getString('app.baseUrl'); const validBaseUrl = new URL(baseUrl); @@ -159,6 +173,10 @@ export async function createConfig( alias: { 'react-dom': '@hot-loader/react-dom', }, + // Enables proper resolution of packages when linking in external packages. + // Without this the packages would depend on dependencies in the node_modules + // of the external packages themselves, leading to module duplication + symlinks: false, }, module: { rules: loaders, @@ -181,17 +199,20 @@ export async function createBackendConfig( ): Promise { const { checksEnabled, isDev } = options; - const { loaders } = transforms(options); - // Find all local monorepo packages and their node_modules, and mark them as external. - const LernaProject = require('@lerna/project'); - const project = new LernaProject(cliPaths.targetDir); - const packages = await project.getPackages(); + const packages = await await loadLernaPackages(); const localPackageNames = packages.map((p: any) => p.name); const moduleDirs = packages.map((p: any) => resolvePath(p.location, 'node_modules'), ); + const { loaders } = transforms({ + ...options, + externalTransforms: packages.map(({ name }) => + cliPaths.resolveTargetRoot('node_modules', name), + ), + }); + return { mode: isDev ? 'development' : 'production', profile: false, @@ -240,6 +261,7 @@ export async function createBackendConfig( alias: { 'react-dom': '@hot-loader/react-dom', }, + symlinks: false, // See frontend config, added here for the same reason }, module: { rules: loaders, diff --git a/packages/cli/src/lib/bundler/transforms.ts b/packages/cli/src/lib/bundler/transforms.ts index d8186dac66..6dc32e6563 100644 --- a/packages/cli/src/lib/bundler/transforms.ts +++ b/packages/cli/src/lib/bundler/transforms.ts @@ -16,7 +16,6 @@ import webpack, { Module, Plugin } from 'webpack'; import MiniCssExtractPlugin from 'mini-css-extract-plugin'; -import { BundlingOptions, BackendBundlingOptions } from './types'; import { svgrTemplate } from '../svgrTemplate'; type Transforms = { @@ -24,17 +23,25 @@ type Transforms = { plugins: Plugin[]; }; -export const transforms = ( - options: BundlingOptions | BackendBundlingOptions, -): Transforms => { - const { isDev } = options; +type TransformOptions = { + isDev: boolean; + // External paths that should be transformed + externalTransforms: string[]; +}; + +export const transforms = (options: TransformOptions): Transforms => { + const { isDev, externalTransforms } = options; const extraTransforms = isDev ? ['react-hot-loader'] : []; + const transformExcludeCondition = { + and: [/node_modules/, { not: externalTransforms }], + }; + const loaders = [ { test: /\.(tsx?)$/, - exclude: /node_modules/, + exclude: transformExcludeCondition, loader: require.resolve('@sucrase/webpack-loader'), options: { transforms: ['typescript', 'jsx', ...extraTransforms], @@ -43,7 +50,7 @@ export const transforms = ( }, { test: /\.(jsx?|mjs)$/, - exclude: /node_modules/, + exclude: transformExcludeCondition, loader: require.resolve('@sucrase/webpack-loader'), options: { transforms: ['jsx', ...extraTransforms], From 206dce0bf827734252ce4580d4afb15adc119f48 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 10:49:41 +0100 Subject: [PATCH 175/259] docs/create-app: document external package linking --- docs/getting-started/create-an-app.md | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/getting-started/create-an-app.md b/docs/getting-started/create-an-app.md index c35eded6a5..552b6c3ec2 100644 --- a/docs/getting-started/create-an-app.md +++ b/docs/getting-started/create-an-app.md @@ -38,6 +38,42 @@ app-folder is the name that was provided when prompted. Inside that directory, it will generate all the files and folder structure needed for you to run your app. +### Linking in local Backstage packages + +It can often be useful to try out changes to the packages in the main Backstage +repo within your own app. For example if you want to make modifications to +`@backstage/core` and try them out in your app. + +To link in external packages, add them to your `package.json` and `lerna.json` +workspace paths. These can be either relative or absolute paths with or without +globs. For example: + +```json +"packages": [ + "packages/*", + "plugins/*", + "../backstage/packages/core", // New path added to work on @backstage/core +], +``` + +Then reinstall packages to make yarn set up symlinks: + +```bash +yarn install +``` + +With this in place you can now modify the `@backstage/core` package within the +main repo, and have those changes be reflected and tested in your app. Simply +run your app using `yarn start` as normal. + +Note that for backend packages you need to make sure that linked packages are +not dependencies of any non-linked package. If you for example want to work on +`@backstage/backend-common`, you need to also link in other backend plugins and +packages that depend on `@backstage/backend-common`, or temporarily disable +those plugins in your backend. This is because the transformation of backend +module tree stops whenever a non-local package is encountered, and from that +point node will `require` packages directly for that entire module subtree. + ### Troubleshooting The create app command doesn't always work as expected, this is a collection of From 29a0ccab2e8b713811a465c442825d24aa1c17cd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 10:52:53 +0100 Subject: [PATCH 176/259] changesets: add cli links change --- .changeset/cli-links.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cli-links.md diff --git a/.changeset/cli-links.md b/.changeset/cli-links.md new file mode 100644 index 0000000000..0a9779de93 --- /dev/null +++ b/.changeset/cli-links.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The CLI now detects and transforms linked packages. You can link in external packages by adding them to both the `lerna.json` and `package.json` workspace paths. From 7bf16061005ce6f904ec24c026cbb0a980eec1f3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 11:25:12 +0100 Subject: [PATCH 177/259] github/vocab: add subtree ._. --- .github/styles/vocab.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 9f6c7d386d..301c77ca6e 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -175,6 +175,7 @@ Spotify squidfunk src subkey +subtree superfences Superfences superset From ecf1e06cdac980f203f7d0b4ace19bddd2815a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 20 Nov 2020 12:10:40 +0100 Subject: [PATCH 178/259] docs: flesh out the catalog Extending the model section (#3352) --- .github/styles/vocab.txt | 21 +- .../software-catalog/extending-the-model.md | 315 ++++++++++++++++-- 2 files changed, 299 insertions(+), 37 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index dcd10e5266..95c6854db6 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -1,9 +1,9 @@ abc +andrewthauer Apdex api Api apis -andrewthauer args asciidoc async @@ -12,20 +12,22 @@ backrub Balachandran benjdlambert Bigtable +Billett Blackbox bool boolean +builtins Chai changeset changesets Changesets -changset chanwit Chanwit cisphobia cissexist classname cli +cloudbuild cncf codeblocks Codecov @@ -68,6 +70,7 @@ github Github gitlab Gitlab +Grafana graphql graphviz Gustavsson @@ -79,6 +82,7 @@ horizontalpodautoscalers Hostname http https +Iain img incentivised inlined @@ -98,8 +102,8 @@ learnings lerna Lerna magiclink -Maintainership mailto +maintainership Malus md microsite @@ -116,6 +120,7 @@ msw namespace namespaces Namespaces +namespacing neuro newrelic nginx @@ -175,6 +180,7 @@ semver Serverless Sinon smartsymobls +Snyk sparklines Spotifiers spotify @@ -215,15 +221,10 @@ Voi Wealthsimple Weaveworks Webpack +www +WWW xyz yaml Zalando Zhou Zolotusky -Billett -cloudbuild -Grafana -Iain -Snyk -www -WWW diff --git a/docs/features/software-catalog/extending-the-model.md b/docs/features/software-catalog/extending-the-model.md index cbacce40c7..d4dca0c059 100644 --- a/docs/features/software-catalog/extending-the-model.md +++ b/docs/features/software-catalog/extending-the-model.md @@ -1,7 +1,7 @@ --- id: extending-the-model title: Extending the model -description: Documentation on Extending the model +description: Documentation on extending the catalog model --- The Backstage catalog [entity data model](descriptor-format.md) is based on the @@ -28,63 +28,324 @@ Backstage comes with a number of catalog concepts out of the box: We'll list different possibilities for extending this below. +## Adding a New apiVersion of an Existing Kind + +Example intents: + +> "I want to evolve this core kind, tweaking the semantics a bit so I will bump +> the apiVersion a step" + +> "This core kind is a decent fit but we want to evolve it at will so we'll move +> it to our own company's apiVersion space and use that instead of +> `backstage.io`." + +The `backstage.io` apiVersion space is reserved for use by the Backstage +maintainers. Please do not change or add versions within that space. + +If you add an [apiVersion](descriptor-format.md#apiversion-and-kind-required) +space of your own, you are effectively branching out from the underlying kind +and making your own. An entity kind is identified by the apiVersion + kind pair, +so even though the resulting entity may be similar to the core one, there will +be no guarantees that plugins will be able to parse or understand its data. See +below about adding a new kind. + ## Adding a New Kind -> TODO: Fill in +Example intents: + +> "The kinds that come with the package are lacking. I want to model this other +> thing that is a poor fit for either of the builtins." + +> "This core kind is a decent fit but we want to evolve it at will so we'll move +> it to our own company's apiVersion space and use that instead of +> `backstage.io`." + +A [kind](descriptor-format.md#apiversion-and-kind-required) is an overarching +family, or an idea if you will, of entities that also share a schema. Backstage +comes with a number of builtin ones that we believe are useful for a large +variety of needs that one may want to model in Backstage. The primary ambition +is to map things to these kinds, but sometimes you may want or need to extend +beyond them. + +Introducing a new apiVersion is basically the same as adding a new kind. Bear in +mind that most plugins will be compiled against the builtin +`@backstage/catalog-model` package and have expectations that kinds align with +that. + +The catalog backend itself, from a storage and API standpoint, does not care +about the kind of entities it stores. Extending with new kinds is mainly a +matter of permitting them to pass validation when building the backend catalog +using the `CatalogBuilder`, and then to make plugins be able to understand the +new kind. + +For the consuming side, it's a different story. Adding a kind has a very large +impact. The very foundation of Backstage is to attach behavior and views and +functionality to entities that we ascribe some meaning to. There will be many +places where code checks `if (kind === 'X')` for some hard coded `X`, and casts +it to a concrete type that it imported from a package such as +`@backstage/catalog-model`. + +If you want to model something that doesn't feel like a fit for either of the +builtin kinds, feel free to reach out to the Backstage maintainers to discuss +how to best proceed. + +If you end up adding that new kind, you must namespace its `apiVersion` +accordingly with a prefix that makes sense, typically based on your organization +name - e.g. `my-company.net/v1`. Also do pick a new `kind` identifier that does +not collide with the builtin kinds. ## Adding a New Type of an Existing Kind -Backstage natively supports tracking of the following component -[`type`](descriptor-format.md)'s: +Example intents: -- Services -- Websites -- Libraries -- Documentation -- Other +> "This is clearly a component, but it's of a type that doesn't quite fit with +> the ones I've seen before." -![](../../assets/software-catalog/bsc-extend.png) +> "We don't call our teams "team", can't we put "flock" as the group type?" -Since these types are likely not the only kind of software you will want to -track in Backstage, it is possible to add your own software types that fit your -organization's data model. Inside Spotify our model has grown significantly over -the years, and now includes ML models, Apps, data pipelines and many more. +Some entity kinds have a `type` field in its spec. This is where an organization +are free to express the variety of entities within a kind. This field is +expected to follow some taxonomy that makes sense for yourself. The chosen value +may affect what operations and views are enabled in Backstage for that entity. +Inside Spotify our model has grown significantly over the years, and our +component types now include ML models, apps, data pipelines and many more. It might be tempting to put software that doesn't fit into any of the existing -types into Other. There are a few reasons why we advise against this; firstly, -we have found that it is preferred to match the conceptual model that your -engineers have when describing your software. Secondly, Backstage helps your -engineers manage their software by integrating the infrastructure tooling -through plugins. Different plugins are used for managing different types of -components. +types into an Other catch-all type. There are a few reasons why we advise +against this; firstly, we have found that it is preferred to match the +conceptual model that your engineers have when describing your software. +Secondly, Backstage helps your engineers manage their software by integrating +the infrastructure tooling through plugins. Different plugins are used for +managing different types of components. For example, the [Lighthouse plugin](https://github.com/backstage/backstage/tree/master/plugins/lighthouse) only makes sense for Websites. The more specific you can be in how you model your software, the easier it is to provide plugins that are contextual. -> TODO: Fill in +Adding a new type takes relatively little effort and carries little risk. Any +type value is accepted by the catalog backend, but plugins may have to be +updated if you want particular behaviors attached to that new type. + +## Changing the Validation Rules for The Entity Envelope or Metadata Fields + +Example intents: + +> "We want to import our old catalog but the default set of allowed characters +> for a metadata.name are too strict." + +> "I want to change the rules for annotations so that I'm allowed to store any +> data in annotation values, not just strings." + +After pieces of raw entity data have been read from a location, they are passed +through a fixed number of so called `Validators`, as part of the entity policy +check step. They ensure that the types and syntax of the base envelope and +metadata make sense - in short, things that aren't entity-kind-specific. Some or +all of these validators can be replaced when building the backend catalog using +the `CatalogBuilder`. + +The risk and impact of this type of extension varies, based on what it is that +you want to do. For example, extending the valid character set for kinds, +namespaces and names can be fairly harmless, with a few notable exceptions - +there is code that expects these to never ever contain a colon or slash, for +example, and introducing URL-unsafe characters risks breaking plugins that +aren't careful about encoding arguments. Supporting non-strings in annotations +may be possible but has not yet been tried out in the real world - there is +likely to be some level of plugin breakage that can be hard to predict. + +Before making this kind of extension, we recommend that you contact the +Backstage maintainers or a support partner to discuss your use case. ## Changing the Validation Rules for Core Entity Fields -> TODO: Fill in +Example intent: + +> "I don't like that the owner is mandatory. I'd like it to be optional." + +After reading and policy-checked entity data from a location, it is sent through +the processor chain looking for processors that implement the +`validateEntityKind` step, to see that the data is of a known kind and abides by +its schema. There is a builtin processor that implements this for all known core +kinds and matches the data against their fixed validation schema. This processor +can be replaced when building the backend catalog using the `CatalogBuilder`, +with a processor of your own that validates the data differently. + +This type of extension is high risk, and may have high impact across the +ecosystem depending on the type of change that is made. It is therefore not +recommended in normal cases. There will be a large number of plugins and +processors - and even the core itself - that make assumptions about the shape of +the data and import the typescript data type from the `@backstage/catalog-model` +package. ## Adding New Fields to the Metadata Object -> TODO: Fill in +Example intent: + +> "Our entities have this auxiliary property that I would like to express for +> several entity kinds and it doesn't really fit as a spec field." + +The metadata object is currently left open for extension. Any unknown fields +found in the metadata will just be stored verbatim in the catalog. However we +want to caution against extending the metadata excessively. Firstly, you run the +risk of colliding with future extensions to the model. Secondly, it is common +that this type of extension lives more comfortably elsewhere - primarily in the +metadata labels or annotations, but sometimes you even may want to make a new +component type or similar instead. + +There are some situations where metadata can be the right place. If you feel +that you have run into such a case and that it would apply to others, do feel +free to contact the Backstage maintainers or a support partner to discuss your +use case. Maybe we can extend the core model to benefit both you and others. ## Adding New Fields to the Spec Object of an Existing Kind -> TODO: Fill in +Example intent: + +> "The builtin Component kind is fine but we want to add an additional field to +> the spec for describing whether it's in prod or staging." + +A kind's schema validation typically doesn't forbid "unknown" fields in an +entity `spec`, and the catalog will happily store whatever is in it. So doing +this will usually work from the catalog's point of view. + +Adding fields like this is subject to the same risks as mentioned about metadata +extensions above. Firstly, you run the risk of colliding with future extensions +to the model. Secondly, it is common that this type of extension lives more +comfortably elsewhere - primarily in the metadata labels or annotations, but +sometimes you even may want to make a new component type or similar instead. + +There are some situations where the spec can be the right place. If you feel +that you have run into such a case and that it would apply to others, do feel +free to contact the Backstage maintainers or a support partner to discuss your +use case. Maybe we can extend the core model to benefit both you and others. ## Adding a New Annotation -> TODO: Fill in +Example intents: + +> "Our custom made build system has the concept of a named pipeline-set, and we +> want to associate individual components with their corresponding pipeline-sets +> so we can show their build status." + +> "We have an alerting system that automatically monitors service health, and +> there's this integration key that binds the service to an alerts pool. We want +> to be able to show the ongoing alerts for our services in Backstage so it'd be +> nice to attach that integration key to the entity somehow." + +Annotations are mainly intended to be consumed by plugins, for feature detection +or linking into external systems. Sometimes they are added by humans, but often +they are automatically generated at ingestion time by processors. There is a set +of [well-known annotations](well-known-annotations.md), but you are free to add +additional ones. This carries no risk or impact to other systems as long as you +abide by the following naming rules. + +- The `backstage.io` annotation prefix is reserved for use by the Backstage + maintainers. Reach out to us if you feel that you would like to make an + addition to that prefix. +- Annotations that pertain to a well known third party system should ideally be + prefixed with a domain, in a way that makes sense to a reader and connects it + clearly to the system (or the maker of the system). For example, you might use + a `pagerduty.com` prefix for pagerduty related annotations, but maybe not + `ldap.com` for LDAP annotations since it's not directly affiliated with or + owned by an LDAP foundation/company/similar. +- Annotations that have no prefix at all, are considered local to your Backstage + instance and can be used freely as such, but you should not make use of them + outside of your organization. For example, if you were to open source a plugin + that generates or consumes annotations, then those annotations must be + properly prefixed with your company domain or a domain that pertains to the + annotation at hand. ## Adding a New Label -> TODO: Fill in +Example intents: + +> "Our process reaping system wants to periodically scrape for components that +> have a certain property." + +> "It'd be nice if our service owners could just tag their components somehow to +> let the CD system know to automatically generate SRV records or not for that +> service." + +Labels are mainly intended to be used for filtering of entities, by external +systems that want to find entities that have some certain property. This is +sometimes used for feature detection / selection. An example could be to add a +label `deployments.my-company.net/register-srv: "true"`. + +At the time of writing this, the use of labels is very limited and we are still +settling together with the community on how to best use them. If you feel that +your use case fits the labels best, we would appreciate if you let the Backstage +maintainers know. + +You are free to add labels. This carries no risk or impact to other systems as +long as you abide by the following naming rules. + +- The `backstage.io` label prefix is reserved for use by the Backstage + maintainers. Reach out to us if you feel that you would like to make an + addition to that prefix. +- Labels that pertain to a well known third party system should ideally be + prefixed with a domain, in a way that makes sense to a reader and connects it + clearly to the system (or the maker of the system). For example, you might use + a `pagerduty.com` prefix for pagerduty related labels, but maybe not + `ldap.com` for LDAP labels since it's not directly affiliated with or owned by + an LDAP foundation/company/similar. +- Labels that have no prefix at all, are considered local to your Backstage + instance and can be used freely as such, but you should not make use of them + outside of your organization. For example, if you were to open source a plugin + that generates or consumes labels, then those labels must be properly prefixed + with your company domain or a domain that pertains to the label at hand. ## Adding a New Relation Type -> TODO: Fill in +Example intents: + +> "We have this concept of service maintainership, separate from ownership, that +> we would like to make relations to individual users for." + +> We feel that we want to explicitly model the team-to-global-department mapping +> as a relation, because it is core to our org setup and we frequently query for +> it. + +Any processor can emit relations for entities as they are being processed, and +new processors can be added when building the backend catalog using the +`CatalogBuilder`. They can emit relations based on the entity data itself, or +based on information gathered from elsewhere. Relations are directed and go from +a source entity to a target entity. They are also tied to the entity that +originated them - the one that was subject to processing when the relation was +emitted. Relations may be dangling (referencing something that does not actually +exist by that name in the catalog), and callers need to be aware of that. + +There is a set of [well-known relations](well-known-relations.md), but you are +free to emit your own as well. You cannot change the fact that they are directed +and have a source and target that have to be an +[entity reference](references.md), but you can invent your own types. You do not +have to make any changes to the catalog backend in order to accept new relation +types. + +At the time of writing this, we do not have any namespacing/prefixing scheme for +relation types. The type is also not validated to contain only some particular +set of characters. Until rules for this are settled, you should stick to using +only letters, dashes and digits, and to avoid collisions with future core +relation types, you may want to prefix the type somehow. For example: +`myCompany-maintainerOf` + `myCompany-maintainedBy`. + +If you have a suggestion for a relation type to be elevated to the core +offering, reach out to the Backstage maintainers or a support partner. + +## Using a Well-Known Relation Type for a New Purpose + +Example intents: + +> "The ownerOf/ownedBy relation types sound like a good fit for expressing how +> users are technical owners of our company specific ServiceAccount kind, and we +> want to reuse those relation types for that." + +At the time of writing, this is uncharted territory. If the documented use of a +relation states that one end of the relation commonly is a User or a Group, for +example, then consumers are likely to have conditional statements on the form +`if (x.kind === 'User') {} else {}`, which get confused when an unexpected kind +appears. + +If you want to extend the use of an established relation type in a way that has +an effect outside of your organization, reach out to the Backstage maintainers +or a support partner to discuss risk/impact. It may even be that one end of the +relation could be considered for addition to the core. From 0c21212400c0a9828c35f8e8c15f5b60163c25be Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 20 Nov 2020 12:53:12 +0100 Subject: [PATCH 179/259] Add support for reading groups and users from the Microsoft Graph API. (#3293) * Add support for reading groups and users from the Microsoft Graph API. * Limit amount of parallel requests * Add helper for paging in odata collections * Add tests for the microsoft graph reader * Output the correct relations between groups and users --- .changeset/curly-yaks-invite.md | 5 + app-config.yaml | 13 + .../well-known-annotations.md | 18 +- plugins/catalog-backend/package.json | 2 + .../MicrosoftGraphOrgReaderProcessor.ts | 100 +++++ .../src/ingestion/processors/index.ts | 1 + .../processors/microsoftGraph/client.test.ts | 324 +++++++++++++++++ .../processors/microsoftGraph/client.ts | 201 ++++++++++ .../processors/microsoftGraph/config.test.ts | 79 ++++ .../processors/microsoftGraph/config.ts | 58 +++ .../processors/microsoftGraph/constants.ts | 32 ++ .../processors/microsoftGraph/index.ts | 19 + .../processors/microsoftGraph/read.test.ts | 339 +++++++++++++++++ .../processors/microsoftGraph/read.ts | 342 ++++++++++++++++++ .../src/ingestion/processors/util/org.test.ts | 23 +- .../src/ingestion/processors/util/org.ts | 21 +- .../src/service/CatalogBuilder.ts | 2 + plugins/catalog/package.json | 1 + yarn.lock | 23 ++ 19 files changed, 1599 insertions(+), 4 deletions(-) create mode 100644 .changeset/curly-yaks-invite.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts diff --git a/.changeset/curly-yaks-invite.md b/.changeset/curly-yaks-invite.md new file mode 100644 index 0000000000..b9191a3895 --- /dev/null +++ b/.changeset/curly-yaks-invite.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add support for reading groups and users from the Microsoft Graph API. diff --git a/app-config.yaml b/app-config.yaml index a331d72d7c..84464a98cf 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -140,6 +140,19 @@ catalog: # dn: ou=access,ou=groups,ou=example,dc=example,dc=net # options: # filter: (&(objectClass=some-group-class)(!(groupType=email))) + microsoftGraphOrg: + ### Example for how to add your Microsoft Graph tenant + #providers: + # - target: https://graph.microsoft.com/v1.0/ + # authority: https://login.microsoftonline.com/ + # tenantId: + # $env: MICROSOFT_GRAPH_TENANT_ID + # clientId: + # $env: MICROSOFT_GRAPH_CLIENT_ID + # clientSecret: + # $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN + # userFilter: accountEnabled eq true and userType eq 'member' + # groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') locations: # Backstage example components diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 4d227ea568..7d16427a3c 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -190,9 +190,25 @@ metadata: ``` The value of these annotations are the corresponding attributes that were found -when ingestion the entity from LDAP. Not all of them may be present, depending +when ingesting the entity from LDAP. Not all of them may be present, depending on what attributes that the server presented at ingestion time. +### graph.microsoft.com/tenant-id, graph.microsoft.com/group-id, graph.microsoft.com/user-id + +```yaml +# Example: +metadata: + annotations: + graph.microsoft.com/tenant-id: 6902611b-ffc1-463f-8af3-4d5285dc057b + graph.microsoft.com/group-id: c57e8ba2-6cc4-1039-9ebc-d5f241a7ca21 + graph.microsoft.com/user-id: 2de244b5-104b-4e8f-a3b8-dce3c31e54b6 +``` + +The value of these annotations are the corresponding attributes that were found +when ingesting the entity from the Microsoft Graph API. Not all of them may be +present, depending on what attributes that the server presented at ingestion +time. + ### sonarqube.org/project-key ```yaml diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 711a91dd8d..12a17d6365 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -20,6 +20,7 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@azure/msal-node": "^1.0.0-alpha.8", "@backstage/backend-common": "^0.3.0", "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", @@ -37,6 +38,7 @@ "lodash": "^4.17.15", "morgan": "^1.10.0", "p-limit": "^3.0.2", + "qs": "^6.9.4", "sqlite3": "^5.0.0", "uuid": "^8.0.0", "winston": "^3.2.1", diff --git a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts new file mode 100644 index 0000000000..3456f2cafa --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -0,0 +1,100 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { Logger } from 'winston'; +import { + MicrosoftGraphClient, + MicrosoftGraphProviderConfig, + readMicrosoftGraphConfig, + readMicrosoftGraphOrg, +} from './microsoftGraph'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +/** + * Extracts teams and users out of an LDAP server. + */ +export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { + private readonly providers: MicrosoftGraphProviderConfig[]; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); + return new MicrosoftGraphOrgReaderProcessor({ + ...options, + providers: c ? readMicrosoftGraphConfig(c) : [], + }); + } + + constructor(options: { + providers: MicrosoftGraphProviderConfig[]; + logger: Logger; + }) { + this.providers = options.providers; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'microsoft-graph-org') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(p.target), + ); + if (!provider) { + throw new Error( + `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`, + ); + } + + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info('Reading Microsoft Graph users and groups'); + + // We create a client each time as we need one that matches the specific provider + const client = MicrosoftGraphClient.create(provider); + const { users, groups } = await readMicrosoftGraphOrg( + client, + provider.tenantId, + { + userFilter: provider.userFilter, + groupFilter: provider.groupFilter, + }, + ); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`, + ); + + // Done! + for (const group of groups) { + emit(results.entity(location, group)); + } + for (const user of users) { + emit(results.entity(location, user)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index ef9c6332cd..118f977125 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -27,6 +27,7 @@ export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { OwnerRelationProcessor } from './OwnerRelationProcessor'; export { LocationRefProcessor } from './LocationEntityProcessor'; +export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts new file mode 100644 index 0000000000..526db38349 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts @@ -0,0 +1,324 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as msal from '@azure/msal-node'; +import { msw } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { MicrosoftGraphClient } from './client'; + +describe('MicrosoftGraphClient', () => { + const confidentialClientApplication: jest.Mocked = { + acquireTokenByClientCredential: jest.fn(), + } as any; + let client: MicrosoftGraphClient; + const worker = setupServer(); + + msw.setupDefaultHandlers(worker); + + beforeEach(() => { + confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue( + { token: 'ACCESS_TOKEN' } as any, + ); + client = new MicrosoftGraphClient( + 'https://example.com', + confidentialClientApplication, + ); + }); + + afterEach(() => { + jest.resetAllMocks(); + worker.resetHandlers(); + }); + + it('should perform raw request', async () => { + worker.use( + rest.get('https://other.example.com/', (_, res, ctx) => + res(ctx.status(200), ctx.json({ value: 'example' })), + ), + ); + + const response = await client.requestRaw('https://other.example.com/'); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ value: 'example' }); + expect( + confidentialClientApplication.acquireTokenByClientCredential, + ).toBeCalledTimes(1); + expect( + confidentialClientApplication.acquireTokenByClientCredential, + ).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] }); + }); + + it('should perform simple api request', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res(ctx.status(200), ctx.json({ value: 'example' })), + ), + ); + + const response = await client.requestApi('users'); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ value: 'example' }); + }); + + it('should perform api request with filter, select and expand', async () => { + worker.use( + rest.get('https://example.com/users', (req, res, ctx) => + res(ctx.status(200), ctx.json({ queryString: req.url.search })), + ), + ); + + const response = await client.requestApi('users', { + filter: 'test eq true', + expand: ['children'], + select: ['id', 'children'], + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + queryString: + '?$filter=test%20eq%20true&$select=id,children&$expand=children', + }); + }); + + it('should perform collection request for a single page', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: ['first'], + }), + ), + ), + ); + + const values = await collectAsyncIterable( + client.requestCollection('users'), + ); + + expect(values).toEqual(['first']); + }); + + it('should perform collection request for multiple pages', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: ['first'], + '@odata.nextLink': 'https://example.com/users2', + }), + ), + ), + ); + worker.use( + rest.get('https://example.com/users2', (_, res, ctx) => + res(ctx.status(200), ctx.json({ value: ['second'] })), + ), + ); + + const values = await collectAsyncIterable( + client.requestCollection('users'), + ); + + expect(values).toEqual(['first', 'second']); + }); + + it('should load user profile', async () => { + worker.use( + rest.get('https://example.com/users/user-id', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + surname: 'Example', + }), + ), + ), + ); + + const userProfile = await client.getUserProfile('user-id'); + + expect(userProfile).toEqual({ surname: 'Example' }); + }); + + it('should throw expection if load user profile fails', async () => { + worker.use( + rest.get('https://example.com/users/user-id', (_, res, ctx) => + res(ctx.status(404)), + ), + ); + + await expect(() => client.getUserProfile('user-id')).rejects.toThrowError(); + }); + + it('should load user profile photo with max size of 120', async () => { + worker.use( + rest.get('https://example.com/users/user-id/photos', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [ + { + height: 120, + id: 120, + }, + { + height: 500, + id: 500, + }, + ], + }), + ), + ), + ); + worker.use( + rest.get( + 'https://example.com/users/user-id/photos/120/*', + (_, res, ctx) => res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getUserPhotoWithSizeLimit('user-id', 120); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should not fail if user has no profile photo', async () => { + worker.use( + rest.get('https://example.com/users/user-id/photos', (_, res, ctx) => + res(ctx.status(404)), + ), + ); + + const photo = await client.getUserPhotoWithSizeLimit('user-id', 120); + + expect(photo).toBeFalsy(); + }); + + it('should load profile photo', async () => { + worker.use( + rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) => + res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getUserPhoto('user-id'); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should load profile photo for size 120', async () => { + worker.use( + rest.get( + 'https://example.com/users/user-id/photos/120/*', + (_, res, ctx) => res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getUserPhoto('user-id', '120'); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should load users', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [{ surname: 'Example' }], + }), + ), + ), + ); + + const values = await collectAsyncIterable(client.getUsers()); + + expect(values).toEqual([{ surname: 'Example' }]); + }); + + it('should load groups', async () => { + worker.use( + rest.get('https://example.com/groups', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [{ displayName: 'Example' }], + }), + ), + ), + ); + + const values = await collectAsyncIterable(client.getGroups()); + + expect(values).toEqual([{ displayName: 'Example' }]); + }); + + it('should load group members', async () => { + worker.use( + rest.get('https://example.com/groups/group-id/members', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [ + { '@odata.type': '#microsoft.graph.user' }, + { '@odata.type': '#microsoft.graph.group' }, + ], + }), + ), + ), + ); + + const values = await collectAsyncIterable( + client.getGroupMembers('group-id'), + ); + + expect(values).toEqual([ + { '@odata.type': '#microsoft.graph.user' }, + { '@odata.type': '#microsoft.graph.group' }, + ]); + }); + + it('should load organization', async () => { + worker.use( + rest.get('https://example.com/organization/tentant-id', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + displayName: 'Example', + }), + ), + ), + ); + + const organization = await client.getOrganization('tentant-id'); + + expect(organization).toEqual({ displayName: 'Example' }); + }); +}); + +async function collectAsyncIterable( + iterable: AsyncIterable, +): Promise { + const values = []; + for await (const value of iterable) { + values.push(value); + } + return values; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts new file mode 100644 index 0000000000..98d4570e91 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts @@ -0,0 +1,201 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as msal from '@azure/msal-node'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import fetch from 'cross-fetch'; +import qs from 'qs'; +import { MicrosoftGraphProviderConfig } from './config'; + +export type ODataQuery = { + filter?: string; + expand?: string[]; + select?: string[]; +}; + +export type GroupMember = + | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' }) + | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' }); + +export class MicrosoftGraphClient { + static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient { + const clientConfig: msal.Configuration = { + auth: { + clientId: config.clientId, + clientSecret: config.clientSecret, + authority: `${config.authority}/${config.tenantId}`, + }, + }; + const pca = new msal.ConfidentialClientApplication(clientConfig); + return new MicrosoftGraphClient(config.target, pca); + } + + constructor( + private readonly baseUrl: string, + private readonly pca: msal.ConfidentialClientApplication, + ) {} + + async *requestCollection( + path: string, + query?: ODataQuery, + ): AsyncIterable { + let response = await this.requestApi(path, query); + + for (;;) { + if (response.status !== 200) { + await this.handleError(path, response); + } + + const result = await response.json(); + const elements: T[] = result.value; + + yield* elements; + + // Follow cursor to the next page if one is available + if (!result['@odata.nextLink']) { + return; + } + + response = await this.requestRaw(result['@odata.nextLink']); + } + } + + async requestApi(path: string, query?: ODataQuery): Promise { + const queryString = qs.stringify( + { + $filter: query?.filter, + $select: query?.select?.join(','), + $expand: query?.expand?.join(','), + }, + { + addQueryPrefix: true, + // Microsoft Graph doesn't like an encoded query string + encode: false, + }, + ); + + return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`); + } + + async requestRaw(url: string): Promise { + // Make sure that we always have a valid access token (might be cached) + const token = await this.pca.acquireTokenByClientCredential({ + scopes: ['https://graph.microsoft.com/.default'], + }); + + return await fetch(url, { + headers: { + Authorization: `Bearer ${token.accessToken}`, + }, + }); + } + + async getUserProfile(userId: string): Promise { + const response = await this.requestApi(`users/${userId}`); + + if (response.status !== 200) { + await this.handleError('user profile', response); + } + + return await response.json(); + } + + async getUserPhotoWithSizeLimit( + userId: string, + maxSize: number, + ): Promise { + const response = await this.requestApi(`users/${userId}/photos`); + + if (response.status === 404) { + return undefined; + } else if (response.status !== 200) { + await this.handleError('user photos', response); + } + + const result = await response.json(); + const photos = result.value as MicrosoftGraph.ProfilePhoto[]; + let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined; + + // Find the biggest picture that is small than the max size + for (const p of photos) { + if ( + !selectedPhoto || + (p.height! >= selectedPhoto.height! && p.height! <= maxSize) + ) { + selectedPhoto = p; + } + } + + if (!selectedPhoto) { + return undefined; + } + + return await this.getUserPhoto(userId, selectedPhoto.id!); + } + + async getUserPhoto( + userId: string, + sizeId?: string, + ): Promise { + const path = sizeId + ? `users/${userId}/photos/${sizeId}/$value` + : `users/${userId}/photo/$value`; + const response = await this.requestApi(path); + + if (response.status === 404) { + return undefined; + } else if (response.status !== 200) { + await this.handleError('photo', response); + } + + return `data:image/jpeg;base64,${Buffer.from( + await response.arrayBuffer(), + ).toString('base64')}`; + } + + async *getUsers(query?: ODataQuery): AsyncIterable { + yield* this.requestCollection(`users`, query); + } + + async *getGroups(query?: ODataQuery): AsyncIterable { + yield* this.requestCollection(`groups`, query); + } + + async *getGroupMembers(groupId: string): AsyncIterable { + yield* this.requestCollection(`groups/${groupId}/members`); + } + + async getOrganization( + tenantId: string, + ): Promise { + const response = await this.requestApi(`organization/${tenantId}`); + + if (response.status !== 200) { + await this.handleError('organization/${tenantId}', response); + } + + return await response.json(); + } + + private async handleError(path: string, response: Response): Promise { + const result = await response.json(); + const error = result.error as MicrosoftGraph.PublicError; + + throw new Error( + `Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`, + ); + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts new file mode 100644 index 0000000000..11c63828d4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readMicrosoftGraphConfig } from './config'; + +describe('readMicrosoftGraphConfig', () => { + it('applies all of the defaults', () => { + const config = { + providers: [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + ], + }; + const actual = readMicrosoftGraphConfig( + ConfigReader.fromConfigs([{ context: '', data: config }]), + ); + const expected = [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.microsoftonline.com', + userFilter: undefined, + groupFilter: undefined, + }, + ]; + expect(actual).toEqual(expected); + }); + + it('reads all the values', () => { + const config = { + providers: [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com/', + userFilter: 'accountEnabled eq true', + groupFilter: 'securityEnabled eq false', + }, + ], + }; + const actual = readMicrosoftGraphConfig( + ConfigReader.fromConfigs([{ context: '', data: config }]), + ); + const expected = [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com', + userFilter: 'accountEnabled eq true', + groupFilter: 'securityEnabled eq false', + }, + ]; + expect(actual).toEqual(expected); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts new file mode 100644 index 0000000000..c4d09e1372 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +export type MicrosoftGraphProviderConfig = { + target: string; + authority: string; + tenantId: string; + clientId: string; + clientSecret: string; + userFilter?: string; + groupFilter?: string; +}; + +export function readMicrosoftGraphConfig( + config: Config, +): MicrosoftGraphProviderConfig[] { + const providers: MicrosoftGraphProviderConfig[] = []; + const providerConfigs = config.getOptionalConfigArray('providers') ?? []; + + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + const authority = + providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') || + 'https://login.microsoftonline.com'; + const tenantId = providerConfig.getString('tenantId'); + const clientId = providerConfig.getString('clientId'); + const clientSecret = providerConfig.getString('clientSecret'); + const userFilter = providerConfig.getOptionalString('userFilter'); + const groupFilter = providerConfig.getOptionalString('groupFilter'); + + providers.push({ + target, + authority, + tenantId, + clientId, + clientSecret, + userFilter, + groupFilter, + }); + } + + return providers; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts new file mode 100644 index 0000000000..6d34d0c159 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The tenant id used by the Microsoft Graph API + */ +export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION = + 'graph.microsoft.com/tenant-id'; + +/** + * The group id used by the Microsoft Graph API + */ +export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION = + 'graph.microsoft.com/group-id'; + +/** + * The user id used by the Microsoft Graph API + */ +export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts new file mode 100644 index 0000000000..1ab567c78c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { MicrosoftGraphClient } from './client'; +export type { MicrosoftGraphProviderConfig } from './config'; +export { readMicrosoftGraphConfig } from './config'; +export { readMicrosoftGraphOrg } from './read'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts new file mode 100644 index 0000000000..3c2f33e922 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts @@ -0,0 +1,339 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import merge from 'lodash/merge'; +import { RecursivePartial } from '../../../util'; +import { GroupMember, MicrosoftGraphClient } from './client'; +import { + normalizeEntityName, + readMicrosoftGraphGroups, + readMicrosoftGraphOrganization, + readMicrosoftGraphUsers, + resolveRelations, +} from './read'; + +function user(data: RecursivePartial): UserEntity { + return merge( + {}, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'name' }, + spec: { profile: {}, memberOf: [] }, + } as UserEntity, + data, + ); +} + +function group(data: RecursivePartial): GroupEntity { + return merge( + {}, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'name', + }, + spec: { + ancestors: [], + children: [], + descendants: [], + type: 'team', + }, + } as GroupEntity, + data, + ); +} + +describe('read microsoft graph', () => { + const client: jest.Mocked = { + getUsers: jest.fn(), + getGroups: jest.fn(), + getGroupMembers: jest.fn(), + getUserPhotoWithSizeLimit: jest.fn(), + getOrganization: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + describe('normalizeEntityName', () => { + it('should normalize name to valid entity name', () => { + expect(normalizeEntityName('User Name')).toBe('user_name'); + }); + + it('should normalize e-mail to valid entity name', () => { + expect(normalizeEntityName('user.name@example.com')).toBe( + 'user.name_example.com', + ); + }); + }); + + describe('readMicrosoftGraphUsers', () => { + it('should read users', async () => { + async function* getExampleUsers() { + yield { + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + + client.getUsers.mockImplementation(getExampleUsers); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsers(client, { + userFilter: 'accountEnabled eq true', + }); + + expect(users).toEqual([ + user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'userid', + }, + name: 'user.name_example.com', + }, + spec: { + profile: { + displayName: 'User Name', + email: 'user.name@example.com', + picture: 'data:image/jpeg;base64,...', + }, + }, + }), + ]); + + expect(client.getUsers).toBeCalledTimes(1); + expect(client.getUsers).toBeCalledWith({ + filter: 'accountEnabled eq true', + select: ['id', 'displayName', 'mail'], + }); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); + }); + + describe('readMicrosoftGraphOrganization', () => { + it('should read organization', async () => { + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + + const { rootGroup } = await readMicrosoftGraphOrganization( + client, + 'tenantid', + ); + + expect(rootGroup).toEqual( + group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + }, + }), + ); + + expect(client.getOrganization).toBeCalledTimes(1); + expect(client.getOrganization).toBeCalledWith('tenantid'); + }); + }); + + describe('readMicrosoftGraphGroups', () => { + it('should read groups', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + + const { + groups, + groupMember, + groupMemberOf, + rootGroup, + } = await readMicrosoftGraphGroups(client, 'tenantid', { + groupFilter: 'securityEnabled eq false', + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Name', + }, + spec: { + type: 'team', + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + filter: 'securityEnabled eq false', + select: ['id', 'displayName', 'mailNickname'], + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + }); + }); + + describe('resolveRelations', () => { + it('should resolve relations', async () => { + const rootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenant-id-root', + }, + name: 'root', + }, + spec: { + type: 'root', + }, + }); + const groupA = group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'group-id-a', + }, + name: 'a', + }, + }); + const groupB = group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'group-id-b', + }, + name: 'b', + }, + }); + const groupC = group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'group-id-c', + }, + name: 'c', + }, + }); + const user1 = user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'user-id-1', + }, + name: 'user1', + }, + }); + const user2 = user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'user-id-2', + }, + name: 'user2', + }, + }); + const groups = [rootGroup, groupA, groupB, groupC]; + const users = [user1, user2]; + const groupMember = new Map>(); + groupMember.set('group-id-b', new Set(['group-id-c'])); + const groupMemberOf = new Map>(); + groupMemberOf.set('user-id-1', new Set(['group-id-a'])); + groupMemberOf.set('user-id-2', new Set(['group-id-c'])); + + // We have a root groups + // We have three groups: a, b, c. c is child of b + // we have two users: u1, u2. u1 is member of a, u2 is member of c + resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); + + expect(rootGroup.spec.parent).toBeUndefined(); + expect(rootGroup.spec.ancestors).toEqual(expect.arrayContaining([])); + expect(rootGroup.spec.children).toEqual( + expect.arrayContaining(['a', 'b']), + ); + expect(rootGroup.spec.descendants).toEqual( + expect.arrayContaining(['a', 'b', 'c']), + ); + + expect(groupA.spec.parent).toEqual('root'); + expect(groupA.spec.ancestors).toEqual(expect.arrayContaining(['root'])); + expect(groupA.spec.children).toEqual(expect.arrayContaining([])); + expect(groupA.spec.descendants).toEqual(expect.arrayContaining([])); + + expect(groupB.spec.parent).toEqual('root'); + expect(groupB.spec.ancestors).toEqual(expect.arrayContaining(['root'])); + expect(groupB.spec.children).toEqual(expect.arrayContaining(['c'])); + expect(groupB.spec.descendants).toEqual(expect.arrayContaining(['c'])); + + expect(groupC.spec.parent).toEqual('b'); + expect(groupC.spec.ancestors).toEqual( + expect.arrayContaining(['root', 'b']), + ); + expect(groupC.spec.children).toEqual(expect.arrayContaining([])); + expect(groupC.spec.descendants).toEqual(expect.arrayContaining([])); + + expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a'])); + + expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c'])); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts new file mode 100644 index 0000000000..6cde2649c4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts @@ -0,0 +1,342 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { buildMemberOf, buildOrgHierarchy } from '../util/org'; +import { MicrosoftGraphClient } from './client'; +import { + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, +} from './constants'; +import limiterFactory from 'p-limit'; + +export function normalizeEntityName(name: string): string { + return name + .trim() + .toLocaleLowerCase() + .replace(/[^a-zA-Z0-9_\-\.]/g, '_'); +} + +export async function readMicrosoftGraphUsers( + client: MicrosoftGraphClient, + options?: { userFilter?: string }, +): Promise<{ + users: UserEntity[]; // With all relations empty +}> { + const entities: UserEntity[] = []; + const picturePromises: Promise[] = []; + const limiter = limiterFactory(10); + + for await (const user of client.getUsers({ + filter: options?.userFilter, + select: ['id', 'displayName', 'mail'], + })) { + if (!user.id || !user.displayName || !user.mail) { + continue; + } + + const name = normalizeEntityName(user.mail); + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name, + annotations: { + [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!, + }, + }, + spec: { + profile: { + displayName: user.displayName!, + email: user.mail!, + + // TODO: Additional fields? + // jobTitle: user.jobTitle || undefined, + // officeLocation: user.officeLocation || undefined, + // mobilePhone: user.mobilePhone || undefined, + }, + memberOf: [], + }, + }; + + // Download the photos in parallel, otherwise it can take quite some time + const loadPhoto = limiter(async () => { + entity.spec.profile!.picture = await client.getUserPhotoWithSizeLimit( + user.id!, + // We are limiting the photo size, as users with full resolution photos + // can make the Backstage API slow + 120, + ); + }); + + picturePromises.push(loadPhoto); + entities.push(entity); + } + + // Wait for all photos to be downloaded + await Promise.all(picturePromises); + + return { users: entities }; +} + +export async function readMicrosoftGraphOrganization( + client: MicrosoftGraphClient, + tenantId: string, +): Promise<{ + rootGroup: GroupEntity; // With all relations empty +}> { + // For now we expect a single root orgranization + const organization = await client.getOrganization(tenantId); + const name = normalizeEntityName(organization.displayName!); + const rootGroup: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: name, + description: organization.displayName!, + annotations: { + [MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!, + }, + }, + spec: { + type: 'root', + ancestors: [], + children: [], + descendants: [], + }, + }; + + return { rootGroup }; +} + +export async function readMicrosoftGraphGroups( + client: MicrosoftGraphClient, + tenantId: string, + options?: { groupFilter?: string }, +): Promise<{ + groups: GroupEntity[]; // With all relations empty + rootGroup: GroupEntity | undefined; // With all relations empty + groupMember: Map>; + groupMemberOf: Map>; +}> { + const groups: GroupEntity[] = []; + const groupMember: Map> = new Map(); + const groupMemberOf: Map> = new Map(); + const limiter = limiterFactory(10); + + const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId); + groupMember.set(rootGroup.metadata.name, new Set()); + groups.push(rootGroup); + + const groupMemberPromises: Promise[] = []; + + for await (const group of client.getGroups({ + filter: options?.groupFilter, + select: ['id', 'displayName', 'mailNickname'], + })) { + if (!group.id || !group.displayName) { + continue; + } + + const name = normalizeEntityName(group.mailNickname || group.displayName); + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: name, + description: group.displayName, + annotations: { + [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id, + }, + }, + spec: { + type: 'team', + // TODO: We could include a group email and picture + ancestors: [], + children: [], + descendants: [], + }, + }; + + // Download the members in parallel, otherwise it can take quite some time + const loadGroupMembers = limiter(async () => { + for await (const member of client.getGroupMembers(group.id!)) { + if (!member.id) { + continue; + } + + if (member['@odata.type'] === '#microsoft.graph.user') { + ensureItem(groupMemberOf, member.id, group.id!); + } + + if (member['@odata.type'] === '#microsoft.graph.group') { + ensureItem(groupMember, group.id!, member.id); + } + } + }); + + groupMemberPromises.push(loadGroupMembers); + groups.push(entity); + } + + // Wait for all group members to be loaded + await Promise.all(groupMemberPromises); + + return { + groups, + rootGroup, + groupMember, + groupMemberOf, + }; +} + +export function resolveRelations( + rootGroup: GroupEntity | undefined, + groups: GroupEntity[], + users: UserEntity[], + groupMember: Map>, + groupMemberOf: Map>, +) { + // Build reference lookup tables, we reference them by the id the the graph + const groupMap: Map = new Map(); // by group-id or tenant-id + + for (const group of groups) { + if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) { + groupMap.set( + group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION], + group, + ); + } + if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) { + groupMap.set( + group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION], + group, + ); + } + } + + // Resolve all member relationships into the reverse direction + const parentGroups = new Map>(); + + groupMember.forEach((members, groupId) => + members.forEach(m => ensureItem(parentGroups, m, groupId)), + ); + + // Make sure every group (except root) has at least one parent. If the parent is missing, add the root. + if (rootGroup) { + const tenantId = rootGroup.metadata.annotations![ + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION + ]; + + groups.forEach(group => { + const groupId = group.metadata.annotations![ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ]; + + if (!groupId) { + return; + } + + if (retrieveItems(parentGroups, groupId).size === 0) { + ensureItem(parentGroups, groupId, tenantId); + ensureItem(groupMember, tenantId, groupId); + } + }); + } + + groups.forEach(group => { + const id = + group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ?? + group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]; + + retrieveItems(groupMember, id).forEach(m => { + const childGroup = groupMap.get(m); + if (childGroup) { + group.spec.children.push(childGroup.metadata.name); + } + }); + + retrieveItems(parentGroups, id).forEach(p => { + const parentGroup = groupMap.get(p); + if (parentGroup) { + // TODO: Only having a single parent group might not match every companies model, but fine for now. + group.spec.parent = parentGroup.metadata.name; + } + }); + }); + + // Make sure that all groups have proper ancestors and descendants + buildOrgHierarchy(groups); + + // Set relations for all users + users.forEach(user => { + const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION]; + + retrieveItems(groupMemberOf, id).forEach(p => { + const parentGroup = groupMap.get(p); + if (parentGroup) { + user.spec.memberOf.push(parentGroup.metadata.name); + } + }); + }); + + // Make sure all transitive memberships are available + buildMemberOf(groups, users); +} + +export async function readMicrosoftGraphOrg( + client: MicrosoftGraphClient, + tenantId: string, + options?: { userFilter?: string; groupFilter?: string }, +): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { + const { users } = await readMicrosoftGraphUsers(client, { + userFilter: options?.userFilter, + }); + const { + groups, + rootGroup, + groupMember, + groupMemberOf, + } = await readMicrosoftGraphGroups(client, tenantId, { + groupFilter: options?.groupFilter, + }); + + resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); + users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + + return { users, groups }; +} + +function ensureItem( + target: Map>, + key: string, + value: string, +) { + let set = target.get(key); + if (!set) { + set = new Set(); + target.set(key, set); + } + set!.add(value); +} + +function retrieveItems( + target: Map>, + key: string, +): Set { + return target.get(key) ?? new Set(); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts index e9dddc8226..f7afd63101 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { GroupEntity } from '@backstage/catalog-model'; -import { buildOrgHierarchy } from './org'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { buildMemberOf, buildOrgHierarchy } from './org'; function g( name: string, @@ -67,3 +67,22 @@ describe('buildOrgHierarchy', () => { expect(d.spec.ancestors).toEqual(expect.arrayContaining(['a'])); }); }); + +describe('buildMemberOf', () => { + it('fills indirect member of groups', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const u: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name }, + spec: { profile: {}, memberOf: ['c'] }, + }; + + const groups = [a, b, c]; + buildOrgHierarchy(groups); + buildMemberOf(groups, [u]); + expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.ts index a280a265a8..b033fe99d7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/org.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { GroupEntity } from '@backstage/catalog-model'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; export function buildOrgHierarchy(groups: GroupEntity[]) { const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); @@ -93,3 +93,22 @@ export function buildOrgHierarchy(groups: GroupEntity[]) { visitAncestors(group); } } + +// Ensure that users have their transitive group memberships. Requires that +// the groups were previously processed with buildOrgHierarchy() +export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + + users.forEach(user => { + const transitiveMemberOf = new Set([...user.spec.memberOf]); + + user.spec.memberOf.forEach(groupName => { + const group = groupsByName.get(groupName); + + if (group) { + group.spec.ancestors.forEach(g => transitiveMemberOf.add(g)); + } + }); + user.spec.memberOf = [...transitiveMemberOf]; + }); +} diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index b041dac5ab..3f95dcd0d3 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -46,6 +46,7 @@ import { LocationReaders, LocationRefProcessor, OwnerRelationProcessor, + MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, StaticLocationProcessor, @@ -278,6 +279,7 @@ export class CatalogBuilder { new FileReaderProcessor(), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), new CodeOwnersProcessor({ reader }), new LocationRefProcessor(), diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index 9ee5d68270..2c65096fd6 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -46,6 +46,7 @@ "@backstage/cli": "^0.3.0", "@backstage/dev-utils": "^0.1.4", "@backstage/test-utils": "^0.1.3", + "@microsoft/microsoft-graph-types": "^1.25.0", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/react-hooks": "^3.3.0", diff --git a/yarn.lock b/yarn.lock index de6ec42c7c..58d9c546ee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -87,6 +87,24 @@ resolved "https://registry.npmjs.org/@asyncapi/specs/-/specs-2.7.5.tgz#3a516d198fc41a1103695bd889fdd4fbbebe7f5d" integrity sha512-T1Ham9sqZKCtSowXRPaBCRy2oz3KHglqqrKiaO7lEudpP6lwH5SwXaq4qliyKzWaqd22srJHE4szdsorbFZKlw== +"@azure/msal-common@^1.6.2": + version "1.6.2" + resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-1.6.2.tgz#91f3732866d727e20f1e142e6e88a981268fbff2" + integrity sha512-GShzp1q7Ld8SwYiDEjQZ9PmFOY4x+2stE86maiguylE9/d/c2muqKjc8aepmEqyjbV7o/omDvEf2Sr9QcIqkSA== + dependencies: + debug "^4.1.1" + +"@azure/msal-node@^1.0.0-alpha.8": + version "1.0.0-alpha.12" + resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-alpha.12.tgz#09d8d52f5cea90b133c3d48fe4ec477693040c91" + integrity sha512-uGLOJRWiEhfJIrTv/lwdm4RxQFm++00h83zNgDn0O3NkXlzAoCCq9QFYW84PjMR/Q2PUvVy7uW+6yKL/Nq3gBA== + dependencies: + "@azure/msal-common" "^1.6.2" + axios "^0.19.2" + debug "^4.1.1" + jsonwebtoken "^8.5.1" + uuid "^8.3.0" + "@babel/code-frame@7.0.0": version "7.0.0" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8" @@ -3372,6 +3390,11 @@ resolved "https://registry.npmjs.org/@mdx-js/react/-/react-1.5.9.tgz#31873ab097fbe58c61c7585fc0be64e83182b6df" integrity sha512-rengdUSedIdIQbXPSeafItCacTYocARAjUA51b6R1KNHmz+59efz7UmyTKr73viJQZ98ouu7iRGmOTtjRrbbWA== +"@microsoft/microsoft-graph-types@^1.25.0": + version "1.25.0" + resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-1.25.0.tgz#1f543ebc029a115dd1d48a1ae99d7ddd5ee9af57" + integrity sha512-RsuA+ROaU3voWzG9TVBkRKxmLatteRGduFDi5p0k3FUHho49rm9SvrA7DUyYbSXLy2xXRx9AnjKM9klYBeKEiQ== + "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" resolved "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde" From c6cf38c6a4297e6a4a3108a0308a98d54726f0f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Fri, 13 Nov 2020 19:26:57 +0100 Subject: [PATCH 180/259] Rename ArchiveResponse to TarArchiveResponse --- .../reading/tree/ReadTreeResponseFactory.ts | 4 ++-- ...onse.test.ts => TarArchiveResponse.test.ts} | 18 +++++++++--------- ...rchiveResponse.ts => TarArchiveResponse.ts} | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) rename packages/backend-common/src/reading/tree/{ArchiveResponse.test.ts => TarArchiveResponse.test.ts} (86%) rename packages/backend-common/src/reading/tree/{ArchiveResponse.ts => TarArchiveResponse.ts} (96%) diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index a134d185f3..90d74e1f20 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -18,7 +18,7 @@ import os from 'os'; import { Readable } from 'stream'; import { Config } from '@backstage/config'; import { ReadTreeResponse } from '../types'; -import { ArchiveResponse } from './ArchiveResponse'; +import { TarArchiveResponse } from './TarArchiveResponse'; type FromArchiveOptions = { // A binary stream of a tar archive. @@ -40,7 +40,7 @@ export class ReadTreeResponseFactory { constructor(private readonly workDir: string) {} async fromArchive(options: FromArchiveOptions): Promise { - return new ArchiveResponse( + return new TarArchiveResponse( options.stream, options.path ?? '', this.workDir, diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts similarity index 86% rename from packages/backend-common/src/reading/tree/ArchiveResponse.test.ts rename to packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts index fd351863b2..3589a8d333 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts @@ -17,13 +17,13 @@ import fs from 'fs-extra'; import mockFs from 'mock-fs'; import { resolve as resolvePath } from 'path'; -import { ArchiveResponse } from './ArchiveResponse'; +import { TarArchiveResponse } from './TarArchiveResponse'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/repo.tar.gz'), ); -describe('ArchiveResponse', () => { +describe('TarArchiveResponse', () => { beforeEach(() => { mockFs({ '/test-archive.tar.gz': archiveData, @@ -38,7 +38,7 @@ describe('ArchiveResponse', () => { it('should read files', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp'); + const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp'); const files = await res.files(); expect(files).toEqual([ @@ -61,7 +61,7 @@ describe('ArchiveResponse', () => { it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path => + const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path => path.endsWith('.yml'), ); const files = await res.files(); @@ -79,14 +79,14 @@ describe('ArchiveResponse', () => { it('should read as archive and files', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp'); + const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new ArchiveResponse(buffer, '', '/tmp'); + const res2 = new TarArchiveResponse(buffer, '', '/tmp'); const files = await res2.files(); expect(files).toEqual([ @@ -109,7 +109,7 @@ describe('ArchiveResponse', () => { it('should extract entire archive into directory', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, '', '/tmp'); + const res = new TarArchiveResponse(stream, '', '/tmp'); const dir = await res.dir(); await expect( @@ -123,7 +123,7 @@ describe('ArchiveResponse', () => { it('should extract archive into directory with a subpath', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); + const res = new TarArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); const dir = await res.dir(); expect(dir).toMatch(/^\/tmp\/.*$/); @@ -135,7 +135,7 @@ describe('ArchiveResponse', () => { it('should extract archive into directory with a subpath and filter', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path => + const res = new TarArchiveResponse(stream, 'mock-repo/', '/tmp', path => path.endsWith('.yml'), ); const dir = await res.dir({ targetDir: '/tmp' }); diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts similarity index 96% rename from packages/backend-common/src/reading/tree/ArchiveResponse.ts rename to packages/backend-common/src/reading/tree/TarArchiveResponse.ts index e16be63127..5d18ec7dc6 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -34,7 +34,7 @@ const pipeline = promisify(pipelineCb); /** * Wraps a tar archive stream into a tree response reader. */ -export class ArchiveResponse implements ReadTreeResponse { +export class TarArchiveResponse implements ReadTreeResponse { private read = false; constructor( @@ -49,7 +49,7 @@ export class ArchiveResponse implements ReadTreeResponse { } if (subPath.startsWith('/')) { throw new TypeError( - `ArchiveResponse subPath must not start with a /, got '${subPath}'`, + `TarArchiveResponse subPath must not start with a /, got '${subPath}'`, ); } } From 9367dde00d4b8d731143274ed13cbbf866eff2ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Mon, 16 Nov 2020 16:08:05 +0100 Subject: [PATCH 181/259] Introduce ZipArchiveResponse --- packages/backend-common/package.json | 4 + .../src/reading/__fixtures__/repo.zip | Bin 0 -> 629 bytes .../reading/tree/ZipArchiveResponse.test.ts | 151 +++++++++++++ .../src/reading/tree/ZipArchiveResponse.ts | 154 +++++++++++++ yarn.lock | 208 +++++++++++++++++- 5 files changed, 512 insertions(+), 5 deletions(-) create mode 100644 packages/backend-common/src/reading/__fixtures__/repo.zip create mode 100644 packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts create mode 100644 packages/backend-common/src/reading/tree/ZipArchiveResponse.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 89150302b5..d4f0052d28 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -36,6 +36,7 @@ "@backstage/test-utils": "^0.1.3", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", + "archiver": "^5.0.2", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", @@ -55,6 +56,7 @@ "selfsigned": "^1.10.7", "stoppable": "^1.1.0", "tar": "^6.0.5", + "unzipper": "^0.10.11", "winston": "^3.2.1" }, "peerDependencies": { @@ -67,6 +69,7 @@ }, "devDependencies": { "@backstage/cli": "^0.3.0", + "@types/archiver": "^3.1.1", "@types/compression": "^1.7.0", "@types/concat-stream": "^1.6.0", "@types/fs-extra": "^9.0.3", @@ -78,6 +81,7 @@ "@types/stoppable": "^1.1.0", "@types/supertest": "^2.0.8", "@types/tar": "^4.0.3", + "@types/unzipper": "^0.10.3", "@types/webpack-env": "^1.15.2", "@types/yaml": "^1.9.7", "get-port": "^5.1.1", diff --git a/packages/backend-common/src/reading/__fixtures__/repo.zip b/packages/backend-common/src/reading/__fixtures__/repo.zip new file mode 100644 index 0000000000000000000000000000000000000000..47956335edad571c8153646cd5288916c9d8e5f4 GIT binary patch literal 629 zcmWIWW@Zs#0D+CQ(ScwFl;8r=x%tW2x<#o4`T7BHb=%O?@uR3q$xki@D+Xz2U;rte zCUQQ49mob@aUez*3Raw%my%kcmz$!j5RzJ4!UeJjqIUyO@7K0B`vib`Kp5mqgx=h2 zkZ!%o+??XflGOOT#N1RXm-p=~S=0I>~^!vFvP literal 0 HcmV?d00001 diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts new file mode 100644 index 0000000000..922284c759 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { resolve as resolvePath } from 'path'; +import { ZipArchiveResponse } from './ZipArchiveResponse'; + +const archiveData = fs.readFileSync( + resolvePath(__filename, '../../__fixtures__/repo.zip'), +); + +describe('ZipArchiveResponse', () => { + beforeEach(() => { + mockFs({ + '/test-archive.zip': archiveData, + '/tmp': mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should read files', async () => { + const stream = fs.createReadStream('/test-archive.zip'); + + const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp'); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'docs/index.md', + content: expect.any(Function), + }, + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + '# Test', + 'site_name: Test', + ]); + }); + + it('should read files with filter', async () => { + const stream = fs.createReadStream('/test-archive.zip'); + + const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path => + path.endsWith('.yml'), + ); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + ]); + const content = await files[0].content(); + expect(content.toString('utf8').trim()).toEqual('site_name: Test'); + }); + + it('should read as archive and files', async () => { + const stream = fs.createReadStream('/test-archive.zip'); + + const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp'); + const buffer = await res.archive(); + + await expect(res.archive()).rejects.toThrow( + 'Response has already been read', + ); + + const res2 = new ZipArchiveResponse(buffer, '', '/tmp'); + const files = await res2.files(); + + expect(files).toEqual([ + { + path: 'docs/index.md', + content: expect.any(Function), + }, + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + '# Test', + 'site_name: Test', + ]); + }); + + it('should extract entire archive into directory', async () => { + const stream = fs.createReadStream('/test-archive.zip'); + + const res = new ZipArchiveResponse(stream, '', '/tmp'); + const dir = await res.dir(); + + await expect( + fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); + + it('should extract archive into directory with a subpath', async () => { + const stream = fs.createReadStream('/test-archive.zip'); + + const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); + const dir = await res.dir(); + + expect(dir).toMatch(/^\/tmp\/.*$/); + await expect( + fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); + + it('should extract archive into directory with a subpath and filter', async () => { + const stream = fs.createReadStream('/test-archive.zip'); + + const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path => + path.endsWith('.yml'), + ); + const dir = await res.dir({ targetDir: '/tmp' }); + + expect(dir).toBe('/tmp'); + await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( + true, + ); + await expect( + fs.pathExists(resolvePath(dir, 'docs/index.md')), + ).resolves.toBe(false); + }); +}); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts new file mode 100644 index 0000000000..23545cd96e --- /dev/null +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import fs from 'fs-extra'; +import unzipper, { Entry } from 'unzipper'; +import archiver from 'archiver'; +import { Readable } from 'stream'; +import { + ReadTreeResponse, + ReadTreeResponseFile, + ReadTreeResponseDirOptions, +} from '../types'; + +/** + * Wraps a zip archive stream into a tree response reader. + */ +export class ZipArchiveResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: Readable, + private readonly subPath: string, + private readonly workDir: string, + private readonly filter?: (path: string) => boolean, + ) { + if (subPath) { + if (!subPath.endsWith('/')) { + this.subPath += '/'; + } + if (subPath.startsWith('/')) { + throw new TypeError( + `ZipArchiveResponse subPath must not start with a /, got '${subPath}'`, + ); + } + } + } + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + private getPath(entry: Entry): string { + return entry.path.slice(this.subPath.length); + } + + private shouldBeIncluded(entry: Entry): boolean { + if (this.subPath) { + if (!entry.path.startsWith(this.subPath)) { + return false; + } + } + if (this.filter) { + return this.filter(this.getPath(entry)); + } + return true; + } + + async files(): Promise { + this.onlyOnce(); + + const files = Array(); + + await this.stream + .pipe(unzipper.Parse()) + .on('entry', (entry: Entry) => { + if (entry.type === 'Directory') { + entry.resume(); + return; + } + + if (this.shouldBeIncluded(entry)) { + files.push({ + path: this.getPath(entry), + content: () => entry.buffer(), + }); + } else { + entry.autodrain(); + } + }) + .promise(); + + return files; + } + + async archive(): Promise { + this.onlyOnce(); + + if (!this.subPath) { + return this.stream; + } + + const archive = archiver('zip'); + await this.stream + .pipe(unzipper.Parse()) + .on('entry', (entry: Entry) => { + if (entry.type === 'File' && this.shouldBeIncluded(entry)) { + archive.append(entry, { name: this.getPath(entry) }); + } else { + entry.autodrain(); + } + }) + .promise(); + archive.finalize(); + + return archive; + } + + async dir(options?: ReadTreeResponseDirOptions): Promise { + this.onlyOnce(); + + const dir = + options?.targetDir ?? + (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); + + await this.stream + .pipe(unzipper.Parse()) + .on('entry', (entry: Entry) => { + if (this.shouldBeIncluded(entry)) { + if (entry.type === 'Directory') { + const directoryPath = this.getPath(entry); + if (directoryPath) { + fs.mkdirSync(path.join(dir, this.getPath(entry))); + } + entry.resume(); + return; + } + entry.pipe(fs.createWriteStream(path.join(dir, this.getPath(entry)))); + } else { + entry.autodrain(); + } + }) + .promise(); + + return dir; + } +} diff --git a/yarn.lock b/yarn.lock index 58d9c546ee..a35f6499e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4873,6 +4873,13 @@ resolved "https://registry.npmjs.org/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a" integrity sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA== +"@types/archiver@^3.1.1": + version "3.1.1" + resolved "https://registry.npmjs.org/@types/archiver/-/archiver-3.1.1.tgz#10cc1be44af8911e57484342c7b3b32a5f178a1a" + integrity sha512-TzVZ9204sH1TuFylfr1cw/AA/3/VldAAXswEwKLXUOzA9mDg+m6gHF9EaqKNlozcjc6knX5m1KAqJzksPLSEfw== + dependencies: + "@types/glob" "*" + "@types/aria-query@^4.2.0": version "4.2.0" resolved "https://registry.npmjs.org/@types/aria-query/-/aria-query-4.2.0.tgz#14264692a9d6e2fa4db3df5e56e94b5e25647ac0" @@ -6016,6 +6023,13 @@ resolved "https://registry.npmjs.org/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e" integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ== +"@types/unzipper@^0.10.3": + version "0.10.3" + resolved "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.3.tgz#9eea872fb1fa460da76f253878b6275af588f464" + integrity sha512-01mQdTLp3/KuBVDhP82FNBf+enzVOjJ9dGsCWa5z8fcYAFVgA9bqIQ2NmsgNFzN/DhD0PUQj4n5p7k6I9mq80g== + dependencies: + "@types/node" "*" + "@types/uuid@^8.0.0": version "8.0.0" resolved "https://registry.npmjs.org/@types/uuid/-/uuid-8.0.0.tgz#165aae4819ad2174a17476dbe66feebd549556c0" @@ -6927,6 +6941,35 @@ arch@^2.1.2: resolved "https://registry.npmjs.org/arch/-/arch-2.1.2.tgz#0c52bbe7344bb4fa260c443d2cbad9c00ff2f0bf" integrity sha512-NTBIIbAfkJeIletyABbVtdPgeKfDafR+1mZV/AyyfC1UkVkp9iUjV+wwmqtUgphHYajbI86jejBJp5e+jkGTiQ== +archiver-utils@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz#e8a460e94b693c3e3da182a098ca6285ba9249e2" + integrity sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw== + dependencies: + glob "^7.1.4" + graceful-fs "^4.2.0" + lazystream "^1.0.0" + lodash.defaults "^4.2.0" + lodash.difference "^4.5.0" + lodash.flatten "^4.4.0" + lodash.isplainobject "^4.0.6" + lodash.union "^4.6.0" + normalize-path "^3.0.0" + readable-stream "^2.0.0" + +archiver@^5.0.2: + version "5.0.2" + resolved "https://registry.npmjs.org/archiver/-/archiver-5.0.2.tgz#b2c435823499b1f46eb07aa18e7bcb332f6ca3fc" + integrity sha512-Tq3yV/T4wxBsD2Wign8W9VQKhaUxzzRmjEiSoOK0SLqPgDP/N1TKdYyBeIEu56T4I9iO4fKTTR0mN9NWkBA0sg== + dependencies: + archiver-utils "^2.1.0" + async "^3.2.0" + buffer-crc32 "^0.2.1" + readable-stream "^3.6.0" + readdir-glob "^1.0.0" + tar-stream "^2.1.4" + zip-stream "^4.0.0" + are-we-there-yet@~1.1.2: version "1.1.5" resolved "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" @@ -7680,6 +7723,11 @@ base64-js@^1.0.2, base64-js@^1.2.0: resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + base64url@3.x.x, base64url@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d" @@ -7751,6 +7799,11 @@ bfj@^7.0.2: hoopy "^0.1.4" tryer "^1.0.1" +big-integer@^1.6.17: + version "1.6.48" + resolved "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e" + integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w== + big.js@^5.2.2: version "5.2.2" resolved "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" @@ -7766,6 +7819,14 @@ binary-extensions@^2.0.0: resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c" integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow== +binary@~0.3.0: + version "0.3.0" + resolved "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79" + integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk= + dependencies: + buffers "~0.1.1" + chainsaw "~0.1.0" + bindings@^1.5.0: version "1.5.0" resolved "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" @@ -7786,7 +7847,7 @@ bl@^1.0.0: readable-stream "^2.3.5" safe-buffer "^5.1.1" -bl@^4.0.1: +bl@^4.0.1, bl@^4.0.3: version "4.0.3" resolved "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz#12d6287adc29080e22a705e5764b2a9522cdc489" integrity sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg== @@ -7807,6 +7868,11 @@ bluebird@3.7.2, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.5 resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +bluebird@~3.4.1: + version "3.4.7" + resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" + integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM= + bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.4.0: version "4.11.8" resolved "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz#2cde09eb5ee341f484746bb0309b3253b1b1442f" @@ -8041,7 +8107,7 @@ buffer-alloc@^1.2.0: buffer-alloc-unsafe "^1.1.0" buffer-fill "^1.0.0" -buffer-crc32@~0.2.3: +buffer-crc32@^0.2.1, buffer-crc32@^0.2.13, buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= @@ -8061,6 +8127,11 @@ buffer-from@1.x, buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer-indexof-polyfill@~1.0.0: + version "1.0.2" + resolved "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz#d2732135c5999c64b277fcf9b1abe3498254729c" + integrity sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A== + buffer-indexof@^1.0.0: version "1.1.1" resolved "https://registry.npmjs.org/buffer-indexof/-/buffer-indexof-1.1.1.tgz#52fabcc6a606d1a00302802648ef68f639da268c" @@ -8090,6 +8161,14 @@ buffer@^4.3.0: ieee754 "^1.1.4" isarray "^1.0.0" +buffer@^5.1.0: + version "5.7.1" + resolved "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" + integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.1.13" + buffer@^5.5.0, buffer@^5.6.0: version "5.6.0" resolved "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" @@ -8098,6 +8177,11 @@ buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.0.2" ieee754 "^1.1.4" +buffers@~0.1.1: + version "0.1.1" + resolved "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb" + integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s= + bufferutil@^4.0.1: version "4.0.1" resolved "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.1.tgz#3a177e8e5819a1243fe16b63a199951a7ad8d4a7" @@ -8391,6 +8475,13 @@ caseless@~0.12.0: resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= +chainsaw@~0.1.0: + version "0.1.0" + resolved "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98" + integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg= + dependencies: + traverse ">=0.3.0 <0.4" + chalk@2.4.2, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.1, chalk@^2.3.2, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -8933,6 +9024,16 @@ component-emitter@^1.2.0, component-emitter@^1.2.1: resolved "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== +compress-commons@^4.0.0: + version "4.0.1" + resolved "https://registry.npmjs.org/compress-commons/-/compress-commons-4.0.1.tgz#c5fa908a791a0c71329fba211d73cd2a32005ea8" + integrity sha512-xZm9o6iikekkI0GnXCmAl3LQGZj5TBDj0zLowsqi7tJtEa3FMGSEcHcqrSJIrOAk1UG/NBbDn/F1q+MG/p/EsA== + dependencies: + buffer-crc32 "^0.2.13" + crc32-stream "^4.0.0" + normalize-path "^3.0.0" + readable-stream "^3.6.0" + compressible@~2.0.16: version "2.0.18" resolved "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" @@ -9300,6 +9401,21 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +crc32-stream@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.0.tgz#05b7ca047d831e98c215538666f372b756d91893" + integrity sha512-tyMw2IeUX6t9jhgXI6um0eKfWq4EIDpfv5m7GX4Jzp7eVelQ360xd8EPXJhp2mHwLQIkqlnMLjzqSZI3a+0wRw== + dependencies: + crc "^3.4.4" + readable-stream "^3.4.0" + +crc@^3.4.4: + version "3.8.0" + resolved "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz#ad60269c2c856f8c299e2c4cc0de4556914056c6" + integrity sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ== + dependencies: + buffer "^5.1.0" + create-ecdh@^4.0.0: version "4.0.3" resolved "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff" @@ -10623,6 +10739,13 @@ dotenv@^8.0.0, dotenv@^8.2.0: resolved "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw== +duplexer2@~0.1.4: + version "0.1.4" + resolved "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" + integrity sha1-ixLauHjA1p4+eJEFFmKjL8a93ME= + dependencies: + readable-stream "^2.0.2" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" @@ -15414,6 +15537,13 @@ lazy-universal-dotenv@^3.0.1: dotenv "^8.0.0" dotenv-expand "^5.1.0" +lazystream@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz#f6995fe0f820392f61396be89462407bb77168e4" + integrity sha1-9plf4PggOS9hOWvolGJAe7dxaOQ= + dependencies: + readable-stream "^2.0.5" + lcid@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" @@ -15558,6 +15688,11 @@ lint-staged@^10.1.0: string-argv "0.3.1" stringify-object "^3.3.0" +listenercount@~1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz#84c8a72ab59c4725321480c975e6508342e70937" + integrity sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc= + listr-silent-renderer@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e" @@ -15760,6 +15895,16 @@ lodash.debounce@^4, lodash.debounce@^4.0.8: resolved "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" + integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw= + +lodash.difference@^4.5.0: + version "4.5.0" + resolved "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz#9ccb4e505d486b91651345772885a2df27fd017c" + integrity sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw= + lodash.flatten@^4.4.0: version "4.4.0" resolved "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f" @@ -15855,6 +16000,11 @@ lodash.throttle@^4.1.1: resolved "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4" integrity sha1-wj6RtxAkKscMN/HhzaknTMOb8vQ= +lodash.union@^4.6.0: + version "4.6.0" + resolved "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88" + integrity sha1-SLtQiECfFvGCFmZkHETdGqrjzYg= + lodash.uniq@^4.5.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" @@ -20205,7 +20355,7 @@ read@1, read@~1.0.1: dependencies: mute-stream "~0.0.4" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -20218,7 +20368,7 @@ read@1, read@~1.0.1: string_decoder "~1.1.1" util-deprecate "~1.0.1" -"readable-stream@2 || 3", readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0: +"readable-stream@2 || 3", readable-stream@^3.0.2, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -20227,6 +20377,13 @@ read@1, read@~1.0.1: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readdir-glob@^1.0.0: + version "1.1.1" + resolved "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4" + integrity sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA== + dependencies: + minimatch "^3.0.4" + readdir-scoped-modules@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309" @@ -21216,7 +21373,7 @@ set-value@^2.0.0, set-value@^2.0.1: is-plain-object "^2.0.3" split-string "^3.0.1" -setimmediate@^1.0.4, setimmediate@^1.0.5: +setimmediate@^1.0.4, setimmediate@^1.0.5, setimmediate@~1.0.4: version "1.0.5" resolved "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU= @@ -22471,6 +22628,17 @@ tar-stream@^2.0.0: inherits "^2.0.3" readable-stream "^3.1.1" +tar-stream@^2.1.4: + version "2.1.4" + resolved "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz#c4fb1a11eb0da29b893a5b25476397ba2d053bfa" + integrity sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar@^2.0.0: version "2.2.2" resolved "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" @@ -22905,6 +23073,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +"traverse@>=0.3.0 <0.4": + version "0.3.9" + resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" + integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk= + traverse@~0.6.6: version "0.6.6" resolved "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz#cbdf560fd7b9af632502fed40f918c157ea97137" @@ -23426,6 +23599,22 @@ untildify@^4.0.0: resolved "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== +unzipper@^0.10.11: + version "0.10.11" + resolved "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz#0b4991446472cbdb92ee7403909f26c2419c782e" + integrity sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw== + dependencies: + big-integer "^1.6.17" + binary "~0.3.0" + bluebird "~3.4.1" + buffer-indexof-polyfill "~1.0.0" + duplexer2 "~0.1.4" + fstream "^1.0.12" + graceful-fs "^4.2.2" + listenercount "~1.0.1" + readable-stream "~2.3.6" + setimmediate "~1.0.4" + upath@^1.1.1, upath@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" @@ -24624,6 +24813,15 @@ zenscroll@^4.0.2: resolved "https://registry.npmjs.org/zenscroll/-/zenscroll-4.0.2.tgz#e8d5774d1c0738a47bcfa8729f3712e2deddeb25" integrity sha1-6NV3TRwHOKR7z6hynzcS4t7d6yU= +zip-stream@^4.0.0: + version "4.0.2" + resolved "https://registry.npmjs.org/zip-stream/-/zip-stream-4.0.2.tgz#3a20f1bd7729c2b59fd4efa04df5eb7a5a217d2e" + integrity sha512-TGxB2g+1ur6MHkvM644DuZr8Uzyz0k0OYWtS3YlpfWBEmK4woaC2t3+pozEL3dBfIPmpgmClR5B2QRcMgGt22g== + dependencies: + archiver-utils "^2.1.0" + compress-commons "^4.0.0" + readable-stream "^3.6.0" + zombie@^6.1.4: version "6.1.4" resolved "https://registry.npmjs.org/zombie/-/zombie-6.1.4.tgz#9f0f53f3d9a032beb7f3fe5b382146a3475a4d47" From 34a51a288a2a2b931c98261ef312c744b9329882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Mon, 16 Nov 2020 16:08:37 +0100 Subject: [PATCH 182/259] Expose ZipArchiveResponse through ReadTreeResponseFactory --- .../src/reading/AzureUrlReader.test.ts | 245 +++++++++++------- .../src/reading/AzureUrlReader.ts | 61 ++++- .../src/reading/GithubUrlReader.ts | 2 +- .../reading/tree/ReadTreeResponseFactory.ts | 12 +- 4 files changed, 221 insertions(+), 99 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 6efc7641b8..ded54cba62 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ +import fs from 'fs'; +import path from 'path'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '../logging'; -import { AzureUrlReader } from './AzureUrlReader'; +import { AzureUrlReader, getDownloadUrl } from './AzureUrlReader'; import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; @@ -32,104 +34,165 @@ describe('AzureUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); - beforeEach(() => { - worker.use( - rest.get('*', (req, res, ctx) => - res( - ctx.status(200), - ctx.json({ - url: req.url.toString(), - headers: req.headers.getAllHeaders(), - }), + describe('read', () => { + beforeEach(() => { + worker.use( + rest.get('*', (req, res, ctx) => + res( + ctx.status(200), + ctx.json({ + url: req.url.toString(), + headers: req.headers.getAllHeaders(), + }), + ), ), - ), - ); - }); - - const createConfig = (token?: string) => - new ConfigReader( - { - integrations: { azure: [{ host: 'dev.azure.com', token }] }, - }, - 'test-config', - ); - - it.each([ - { - url: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', - }), - }, - { - url: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', - }), - }, - { - url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', - config: createConfig('0123456789'), - response: expect.objectContaining({ - headers: expect.objectContaining({ - authorization: 'Basic OjAxMjM0NTY3ODk=', - }), - }), - }, - { - url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', - config: createConfig(undefined), - response: expect.objectContaining({ - headers: expect.not.objectContaining({ - authorization: expect.anything(), - }), - }), - }, - ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = AzureUrlReader.factory({ - config, - logger, - treeResponseFactory, + ); }); - const data = await reader.read(url); - const res = await JSON.parse(data.toString('utf-8')); - expect(res).toEqual(response); - }); + const createConfig = (token?: string) => + new ConfigReader( + { + integrations: { azure: [{ host: 'dev.azure.com', token }] }, + }, + 'test-config', + ); - it.each([ - { - url: 'https://api.com/a/b/blob/master/path/to/c.yaml', - config: createConfig(), - error: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', - }, - { - url: 'com/a/b/blob/master/path/to/c.yaml', - config: createConfig(), - error: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - { - url: '', - config: createConfig(''), - error: - "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string", - }, - ])('should handle error path %#', async ({ url, config, error }) => { - await expect(async () => { + it.each([ + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + }), + }, + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + config: createConfig(), + response: expect.objectContaining({ + url: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + }), + }, + { + url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', + config: createConfig('0123456789'), + response: expect.objectContaining({ + headers: expect.objectContaining({ + authorization: 'Basic OjAxMjM0NTY3ODk=', + }), + }), + }, + { + url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml', + config: createConfig(undefined), + response: expect.objectContaining({ + headers: expect.not.objectContaining({ + authorization: expect.anything(), + }), + }), + }, + ])('should handle happy path %#', async ({ url, config, response }) => { const [{ reader }] = AzureUrlReader.factory({ config, logger, treeResponseFactory, }); - await reader.read(url); - }).rejects.toThrow(error); + + const data = await reader.read(url); + const res = await JSON.parse(data.toString('utf-8')); + expect(res).toEqual(response); + }); + + it.each([ + { + url: 'https://api.com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + url: 'com/a/b/blob/master/path/to/c.yaml', + config: createConfig(), + error: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + url: '', + config: createConfig(''), + error: + "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string", + }, + ])('should handle error path %#', async ({ url, config, error }) => { + await expect(async () => { + const [{ reader }] = AzureUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); + await reader.read(url); + }).rejects.toThrow(error); + }); + }); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://dev.azure.com/organization/project/_apis/git/repositories/repository/items', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const processor = new AzureUrlReader( + { + host: 'dev.azure.com', + }, + treeResponseFactory, + ); + + const response = await processor.readTree( + 'https://dev.azure.com/organization/project/_git/repository', + ); + + const files = await response.files(); + + expect(files.length).toBe(2); + const mkDocsFile = await files[1].content(); + const indexMarkdownFile = await files[0].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + }); + + describe('getDownloadUrl', () => { + it('add no scopePath if no path is specified', async () => { + const result = getDownloadUrl( + 'https://dev.azure.com/organization/project/_git/repository', + ); + + expect(result.searchParams.get('scopePath')).toBeNull(); + }); + + it('add the scopePath if a path is specified', async () => { + const result = getDownloadUrl( + 'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs', + ); + expect(result.searchParams.get('scopePath')).toEqual('docs'); + }); }); }); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index db8b738667..068f4d2409 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -19,22 +19,53 @@ import { readAzureIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; +import { Readable } from 'stream'; +import parseGitUri from 'git-url-parse'; import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + UrlReader, +} from './types'; +import { ReadTreeResponseFactory } from './tree'; + +export function getDownloadUrl(url: string): URL { + const { + name: repoName, + owner: project, + organization, + protocol, + resource, + filepath, + } = parseGitUri(url); + + // scopePath will limit the downloaded content + // /docs will only download the docs folder and everything below it + // /docs/index.md will only download index.md but put it in the root of the archive + const scopePath = filepath ? `&scopePath=${filepath}` : ''; + + return new URL( + `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`, + ); +} export class AzureUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readAzureIntegrationConfigs( config.getOptionalConfigArray('integrations.azure') ?? [], ); return configs.map(options => { - const reader = new AzureUrlReader(options); + const reader = new AzureUrlReader(options, treeResponseFactory); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); }; - constructor(private readonly options: AzureIntegrationConfig) { + constructor( + private readonly options: AzureIntegrationConfig, + private readonly treeResponseFactory: ReadTreeResponseFactory, + ) { if (options.host !== 'dev.azure.com') { throw Error( `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`, @@ -64,8 +95,26 @@ export class AzureUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('AzureUrlReader does not implement readTree'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const response = await fetch(getDownloadUrl(url).toString(), { + ...this.getRequestOptions(), + headers: { Accept: 'application/zip' }, + }); + if (!response.ok) { + const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return this.treeResponseFactory.fromZipArchive({ + stream: (response.body as unknown) as Readable, + filter: options?.filter, + }); } // Converts diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 2fbaa0b32b..907f2ada7a 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -207,7 +207,7 @@ export class GithubUrlReader implements UrlReader { const path = `${repoName}-${ref}/${filepath}`; - return this.deps.treeResponseFactory.fromArchive({ + return this.deps.treeResponseFactory.fromTarArchive({ // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want // to stick to using that in exclusively backend code. stream: (response.body as unknown) as Readable, diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 90d74e1f20..986a0302bc 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -19,6 +19,7 @@ import { Readable } from 'stream'; import { Config } from '@backstage/config'; import { ReadTreeResponse } from '../types'; import { TarArchiveResponse } from './TarArchiveResponse'; +import { ZipArchiveResponse } from './ZipArchiveResponse'; type FromArchiveOptions = { // A binary stream of a tar archive. @@ -39,7 +40,7 @@ export class ReadTreeResponseFactory { constructor(private readonly workDir: string) {} - async fromArchive(options: FromArchiveOptions): Promise { + async fromTarArchive(options: FromArchiveOptions): Promise { return new TarArchiveResponse( options.stream, options.path ?? '', @@ -47,4 +48,13 @@ export class ReadTreeResponseFactory { options.filter, ); } + + async fromZipArchive(options: FromArchiveOptions): Promise { + return new ZipArchiveResponse( + options.stream, + options.path ?? '', + this.workDir, + options.filter, + ); + } } From bff3305aa9a36c597b7387df1bf2993de3ca8e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Mon, 16 Nov 2020 16:16:21 +0100 Subject: [PATCH 183/259] Add changeset --- .changeset/clever-moons-shake.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/clever-moons-shake.md diff --git a/.changeset/clever-moons-shake.md b/.changeset/clever-moons-shake.md new file mode 100644 index 0000000000..0fb6c690c2 --- /dev/null +++ b/.changeset/clever-moons-shake.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Added readTree support to AzureUrlReader From a9155afb83aff9ba4d31df635ac96708f50be2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Tue, 17 Nov 2020 21:18:55 +0100 Subject: [PATCH 184/259] Some small changes --- .../src/reading/tree/ZipArchiveResponse.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 23545cd96e..048b2ab46d 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -134,15 +134,15 @@ export class ZipArchiveResponse implements ReadTreeResponse { .pipe(unzipper.Parse()) .on('entry', (entry: Entry) => { if (this.shouldBeIncluded(entry)) { + const entryPath = this.getPath(entry); if (entry.type === 'Directory') { - const directoryPath = this.getPath(entry); - if (directoryPath) { - fs.mkdirSync(path.join(dir, this.getPath(entry))); + if (entryPath) { + fs.mkdirSync(path.join(dir, entryPath)); } entry.resume(); - return; + } else { + entry.pipe(fs.createWriteStream(path.join(dir, entryPath))); } - entry.pipe(fs.createWriteStream(path.join(dir, this.getPath(entry)))); } else { entry.autodrain(); } From c08e5cbd6ffad15d69a6c27c1bd95ac1bc46e8b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Wed, 18 Nov 2020 11:27:31 +0100 Subject: [PATCH 185/259] Fix parameters and encode URI component --- .../backend-common/src/reading/AzureUrlReader.test.ts | 7 ++++--- packages/backend-common/src/reading/AzureUrlReader.ts | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index ded54cba62..91a1e40c10 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -161,7 +161,7 @@ describe('AzureUrlReader', () => { { host: 'dev.azure.com', }, - treeResponseFactory, + { treeResponseFactory }, ); const response = await processor.readTree( @@ -180,7 +180,7 @@ describe('AzureUrlReader', () => { }); describe('getDownloadUrl', () => { - it('add no scopePath if no path is specified', async () => { + it('do not add scopePath if no path is specified', async () => { const result = getDownloadUrl( 'https://dev.azure.com/organization/project/_git/repository', ); @@ -188,10 +188,11 @@ describe('AzureUrlReader', () => { expect(result.searchParams.get('scopePath')).toBeNull(); }); - it('add the scopePath if a path is specified', async () => { + it('add scopePath if a path is specified', async () => { const result = getDownloadUrl( 'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs', ); + console.log(result.searchParams); expect(result.searchParams.get('scopePath')).toEqual('docs'); }); }); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 068f4d2409..e5e290eb13 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -43,7 +43,9 @@ export function getDownloadUrl(url: string): URL { // scopePath will limit the downloaded content // /docs will only download the docs folder and everything below it // /docs/index.md will only download index.md but put it in the root of the archive - const scopePath = filepath ? `&scopePath=${filepath}` : ''; + const scopePath = filepath + ? `&scopePath=${encodeURIComponent(filepath)}` + : ''; return new URL( `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`, @@ -56,7 +58,7 @@ export class AzureUrlReader implements UrlReader { config.getOptionalConfigArray('integrations.azure') ?? [], ); return configs.map(options => { - const reader = new AzureUrlReader(options, treeResponseFactory); + const reader = new AzureUrlReader(options, { treeResponseFactory }); const predicate = (url: URL) => url.host === options.host; return { reader, predicate }; }); @@ -64,7 +66,7 @@ export class AzureUrlReader implements UrlReader { constructor( private readonly options: AzureIntegrationConfig, - private readonly treeResponseFactory: ReadTreeResponseFactory, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { if (options.host !== 'dev.azure.com') { throw Error( @@ -111,7 +113,7 @@ export class AzureUrlReader implements UrlReader { throw new Error(message); } - return this.treeResponseFactory.fromZipArchive({ + return this.deps.treeResponseFactory.fromZipArchive({ stream: (response.body as unknown) as Readable, filter: options?.filter, }); From b1b6537bc360922980fb45c2e972a6b4f836991a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Wed, 18 Nov 2020 16:28:11 +0100 Subject: [PATCH 186/259] Handle request headers correctly --- .../src/reading/AzureUrlReader.test.ts | 1 - .../backend-common/src/reading/AzureUrlReader.ts | 14 ++++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 91a1e40c10..ab97d1b073 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -192,7 +192,6 @@ describe('AzureUrlReader', () => { const result = getDownloadUrl( 'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs', ); - console.log(result.searchParams); expect(result.searchParams.get('scopePath')).toEqual('docs'); }); }); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index e5e290eb13..ad990d1d5d 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -101,10 +101,10 @@ export class AzureUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const response = await fetch(getDownloadUrl(url).toString(), { - ...this.getRequestOptions(), - headers: { Accept: 'application/zip' }, - }); + const response = await fetch( + getDownloadUrl(url).toString(), + this.getRequestOptions({ Accept: 'application/zip' }), + ); if (!response.ok) { const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; if (response.status === 404) { @@ -178,8 +178,10 @@ export class AzureUrlReader implements UrlReader { } } - private getRequestOptions(): RequestInit { - const headers: HeadersInit = {}; + private getRequestOptions(additionalHeaders?: { + [key: string]: string; + }): RequestInit { + const headers: HeadersInit = additionalHeaders ?? {}; if (this.options.token) { headers.Authorization = `Basic ${Buffer.from( From 3092806838fe337c7d2e8089430e7d464cef4c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Wed, 18 Nov 2020 16:28:27 +0100 Subject: [PATCH 187/259] Support ZIP files without directory entries --- .../src/reading/__fixtures__/repo.zip | Bin 629 -> 387 bytes .../src/reading/tree/ZipArchiveResponse.ts | 13 ++++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/src/reading/__fixtures__/repo.zip b/packages/backend-common/src/reading/__fixtures__/repo.zip index 47956335edad571c8153646cd5288916c9d8e5f4..f66bf2d612eafc3c4b65216b83d1b3e884726236 100644 GIT binary patch literal 387 zcmWIWW@h1H0D*0_(Sg%M&PT8V*&r;=Aj6QGpPa2*lv0=yB1 jV>%w$@CX#ck-Y*m8RQiVlUdn74q^hr?Lc}Ph{FH?Ktx*D literal 629 zcmWIWW@Zs#0D+CQ(ScwFl;8r=x%tW2x<#o4`T7BHb=%O?@uR3q$xki@D+Xz2U;rte zCUQQ49mob@aUez*3Raw%my%kcmz$!j5RzJ4!UeJjqIUyO@7K0B`vib`Kp5mqgx=h2 zkZ!%o+??XflGOOT#N1RXm-p=~S=0I>~^!vFvP diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 048b2ab46d..89f9803bd2 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -136,13 +136,16 @@ export class ZipArchiveResponse implements ReadTreeResponse { if (this.shouldBeIncluded(entry)) { const entryPath = this.getPath(entry); if (entry.type === 'Directory') { - if (entryPath) { - fs.mkdirSync(path.join(dir, entryPath)); - } + // Ignore directory entries since we handle that with the file entries + // since a zip can have files with directories without directory entries entry.resume(); - } else { - entry.pipe(fs.createWriteStream(path.join(dir, entryPath))); + return; } + const dirname = path.dirname(entryPath); + if (dirname) { + fs.mkdirSync(path.join(dir, dirname), { recursive: true }); + } + entry.pipe(fs.createWriteStream(path.join(dir, entryPath))); } else { entry.autodrain(); } From f31d689d826e32e3c65b39e573a4bca1227dfe3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Wed, 18 Nov 2020 20:28:38 +0100 Subject: [PATCH 188/259] Switch to using async fs operations --- .../src/reading/tree/ZipArchiveResponse.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 89f9803bd2..4106d49a11 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -132,18 +132,14 @@ export class ZipArchiveResponse implements ReadTreeResponse { await this.stream .pipe(unzipper.Parse()) - .on('entry', (entry: Entry) => { - if (this.shouldBeIncluded(entry)) { + .on('entry', async (entry: Entry) => { + // Ignore directory entries since we handle that with the file entries + // as a zip can have files with directories without directory entries + if (entry.type === 'File' && this.shouldBeIncluded(entry)) { const entryPath = this.getPath(entry); - if (entry.type === 'Directory') { - // Ignore directory entries since we handle that with the file entries - // since a zip can have files with directories without directory entries - entry.resume(); - return; - } const dirname = path.dirname(entryPath); if (dirname) { - fs.mkdirSync(path.join(dir, dirname), { recursive: true }); + await fs.mkdirp(path.join(dir, dirname)); } entry.pipe(fs.createWriteStream(path.join(dir, entryPath))); } else { From 742196725f78d444324adc9ce88dc8e010dece13 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 20 Nov 2020 13:40:16 +0100 Subject: [PATCH 189/259] Fix some minor things from my previous PR See #3293 --- app-config.yaml | 4 ++-- .../src/ingestion/processors/microsoftGraph/client.test.ts | 1 - .../src/ingestion/processors/microsoftGraph/client.ts | 2 +- .../src/ingestion/processors/microsoftGraph/index.ts | 5 +++++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 84464a98cf..32d3470f1b 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -143,8 +143,8 @@ catalog: microsoftGraphOrg: ### Example for how to add your Microsoft Graph tenant #providers: - # - target: https://graph.microsoft.com/v1.0/ - # authority: https://login.microsoftonline.com/ + # - target: https://graph.microsoft.com/v1.0 + # authority: https://login.microsoftonline.com # tenantId: # $env: MICROSOFT_GRAPH_TENANT_ID # clientId: diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts index 526db38349..e51a7753af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts @@ -41,7 +41,6 @@ describe('MicrosoftGraphClient', () => { afterEach(() => { jest.resetAllMocks(); - worker.resetHandlers(); }); it('should perform raw request', async () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts index 98d4570e91..6dded56aa1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts @@ -129,7 +129,7 @@ export class MicrosoftGraphClient { const photos = result.value as MicrosoftGraph.ProfilePhoto[]; let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined; - // Find the biggest picture that is small than the max size + // Find the biggest picture that is smaller than the max size for (const p of photos) { if ( !selectedPhoto || diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts index 1ab567c78c..882125fd84 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts @@ -17,3 +17,8 @@ export { MicrosoftGraphClient } from './client'; export type { MicrosoftGraphProviderConfig } from './config'; export { readMicrosoftGraphConfig } from './config'; export { readMicrosoftGraphOrg } from './read'; +export { + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, +} from './constants'; From 11598f6324397c26352d926e1619f8dd8b0731cd Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 20 Nov 2020 14:37:09 +0100 Subject: [PATCH 190/259] chore: fixing up the test timeout --- .../components/MultistepJsonForm/MultistepJsonForm.tsx | 9 +++------ .../src/components/TemplatePage/TemplatePage.tsx | 8 +++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx index 9700d43c00..528d42b94f 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx +++ b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx @@ -27,7 +27,7 @@ import { } from '@material-ui/core'; import { FormProps, IChangeEvent, withTheme } from '@rjsf/core'; import { Theme as MuiTheme } from '@rjsf/material-ui'; -import React, { useState, useEffect } from 'react'; +import React, { useState } from 'react'; const Form = withTheme(MuiTheme); type Step = { @@ -54,7 +54,6 @@ export const MultistepJsonForm = ({ onFinish, }: Props) => { const [activeStep, setActiveStep] = useState(0); - const [formDataEvent, setFormDataEvent] = useState({ formData: {} }); const handleReset = () => { setActiveStep(0); @@ -63,9 +62,7 @@ export const MultistepJsonForm = ({ const handleNext = () => setActiveStep(Math.min(activeStep + 1, steps.length)); const handleBack = () => setActiveStep(Math.max(activeStep - 1, 0)); - useEffect(() => { - onChange(formDataEvent as IChangeEvent); - }, [formDataEvent, onChange]); + return ( <> @@ -77,7 +74,7 @@ export const MultistepJsonForm = ({ key={label} noHtml5Validate formData={formData} - onChange={e => setFormDataEvent(e)} + onChange={onChange} schema={schema as FormProps['schema']} onSubmit={e => { if (e.errors.length === 0) handleNext(); diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index ebefc6522d..d8b803f893 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -26,7 +26,7 @@ import { import { catalogApiRef } from '@backstage/plugin-catalog'; import { LinearProgress } from '@material-ui/core'; import { IChangeEvent } from '@rjsf/core'; -import React, { useState } from 'react'; +import React, { useState, useCallback } from 'react'; import { Navigate } from 'react-router'; import { useParams } from 'react-router-dom'; import { useAsync } from 'react-use'; @@ -86,8 +86,10 @@ export const TemplatePage = () => { const [formState, setFormState] = useState({}); const handleFormReset = () => setFormState({}); - const handleChange = (e: IChangeEvent) => - setFormState({ ...formState, ...e.formData }); + const handleChange = useCallback( + (e: IChangeEvent) => setFormState({ ...formState, ...e.formData }), + [setFormState, formState], + ); const [jobId, setJobId] = useState(null); From 475fc0aaa33b044265cf03724676fdafab489be6 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 20 Nov 2020 15:14:18 +0100 Subject: [PATCH 191/259] Make sidebar search field work (#3362) * Make sidebar search field work Extend the search page to have the ability to react to query parameters. The search in the sidebar now navigates to the search page and passes the query parameter. The search box on the search page is now debounced. Closes #3341 * Fix sidebar search while the search page is already open --- .changeset/breezy-cobras-deny.md | 5 +++ .changeset/small-worms-check.md | 6 ++++ packages/app/src/components/Root/Root.tsx | 10 ++---- packages/core/src/hooks/useQueryParamState.ts | 11 +++++- packages/core/src/layout/Sidebar/Items.tsx | 2 ++ plugins/search/package.json | 9 ++--- .../src/components/SearchPage/SearchPage.tsx | 22 ++++++++---- .../SidebarSearch/SidebarSearch.tsx | 35 +++++++++++++++++++ .../src/components/SidebarSearch/index.ts | 16 +++++++++ plugins/search/src/components/index.tsx | 1 + plugins/search/src/index.ts | 1 + 11 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 .changeset/breezy-cobras-deny.md create mode 100644 .changeset/small-worms-check.md create mode 100644 plugins/search/src/components/SidebarSearch/SidebarSearch.tsx create mode 100644 plugins/search/src/components/SidebarSearch/index.ts diff --git a/.changeset/breezy-cobras-deny.md b/.changeset/breezy-cobras-deny.md new file mode 100644 index 0000000000..5f035c05ce --- /dev/null +++ b/.changeset/breezy-cobras-deny.md @@ -0,0 +1,5 @@ +--- +'@backstage/core': patch +--- + +Clear sidebar search field once a search is executed diff --git a/.changeset/small-worms-check.md b/.changeset/small-worms-check.md new file mode 100644 index 0000000000..e39265f8f9 --- /dev/null +++ b/.changeset/small-worms-check.md @@ -0,0 +1,6 @@ +--- +'example-app': patch +'@backstage/plugin-search': patch +--- + +Using the search field in the sidebar now navigates to the search result page. diff --git a/packages/app/src/components/Root/Root.tsx b/packages/app/src/components/Root/Root.tsx index e947e5c91d..47e7d32d1a 100644 --- a/packages/app/src/components/Root/Root.tsx +++ b/packages/app/src/components/Root/Root.tsx @@ -33,12 +33,12 @@ import { SidebarContext, SidebarItem, SidebarDivider, - SidebarSearchField, SidebarSpace, } from '@backstage/core'; import { NavLink } from 'react-router-dom'; import { graphiQLRouteRef } from '@backstage/plugin-graphiql'; import { Settings as SidebarSettings } from '@backstage/plugin-user-settings'; +import { SidebarSearch } from '@backstage/plugin-search'; const useSidebarLogoStyles = makeStyles({ root: { @@ -73,17 +73,11 @@ const SidebarLogo: FC<{}> = () => { ); }; -const handleSearch = (query: string): void => { - // XXX (@koroeskohr): for testing purposes - // eslint-disable-next-line no-console - console.log(query); -}; - const Root: FC<{}> = ({ children }) => ( - + {/* Global nav, not org-specific */} diff --git a/packages/core/src/hooks/useQueryParamState.ts b/packages/core/src/hooks/useQueryParamState.ts index 9317dcd3e1..dc2279cb53 100644 --- a/packages/core/src/hooks/useQueryParamState.ts +++ b/packages/core/src/hooks/useQueryParamState.ts @@ -14,8 +14,9 @@ * limitations under the License. */ +import { isEqual } from 'lodash'; import qs from 'qs'; -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import { useLocation, useNavigate } from 'react-router-dom'; import { useDebounce } from 'react-use'; @@ -64,6 +65,14 @@ export function useQueryParamState( extractState(location.search, stateName), ); + useEffect(() => { + const newState = extractState(location.search, stateName); + + setQueryParamState(oldState => + isEqual(newState, oldState) ? oldState : newState, + ); + }, [location, stateName]); + useDebounce( () => { const queryString = joinQueryString( diff --git a/packages/core/src/layout/Sidebar/Items.tsx b/packages/core/src/layout/Sidebar/Items.tsx index bb47701280..498946577d 100644 --- a/packages/core/src/layout/Sidebar/Items.tsx +++ b/packages/core/src/layout/Sidebar/Items.tsx @@ -221,6 +221,7 @@ export const SidebarSearchField: FC = props => { const handleEnter: KeyboardEventHandler = ev => { if (ev.key === 'Enter') { props.onSearch(input); + setInput(''); } }; @@ -233,6 +234,7 @@ export const SidebarSearchField: FC = props => { { - const [searchQuery, setSearchQuery] = useState(''); + const [queryString, setQueryString] = useQueryParamState('query'); + const [searchQuery, setSearchQuery] = useState(queryString ?? ''); const handleSearch = (event: React.ChangeEvent) => { event.preventDefault(); setSearchQuery(event.target.value); }; + useEffect(() => setSearchQuery(queryString ?? ''), [queryString]); + + useDebounce( + () => { + setQueryString(searchQuery); + }, + 200, + [searchQuery], + ); + const handleClearSearchBar = () => { setSearchQuery(''); }; @@ -46,7 +56,7 @@ export const SearchPage = () => { /> - + diff --git a/plugins/search/src/components/SidebarSearch/SidebarSearch.tsx b/plugins/search/src/components/SidebarSearch/SidebarSearch.tsx new file mode 100644 index 0000000000..6a279e0743 --- /dev/null +++ b/plugins/search/src/components/SidebarSearch/SidebarSearch.tsx @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useCallback } from 'react'; +import qs from 'qs'; +import { useNavigate } from 'react-router-dom'; +import { SidebarSearchField } from '@backstage/core'; + +export const SidebarSearch = () => { + const navigate = useNavigate(); + const handleSearch = useCallback( + (query: string): void => { + const queryString = qs.stringify({ query }, { addQueryPrefix: true }); + + // TODO: Here the url to the search plugin is hardcoded. We need a way to query the route in the future. + // Maybe an API that I can just call from other places? + navigate(`/search${queryString}`); + }, + [navigate], + ); + + return ; +}; diff --git a/plugins/search/src/components/SidebarSearch/index.ts b/plugins/search/src/components/SidebarSearch/index.ts new file mode 100644 index 0000000000..33869ffb77 --- /dev/null +++ b/plugins/search/src/components/SidebarSearch/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { SidebarSearch } from './SidebarSearch'; diff --git a/plugins/search/src/components/index.tsx b/plugins/search/src/components/index.tsx index f8e6a5a09e..ac47860dc2 100644 --- a/plugins/search/src/components/index.tsx +++ b/plugins/search/src/components/index.tsx @@ -18,3 +18,4 @@ export * from './Filters'; export * from './SearchBar'; export * from './SearchPage'; export * from './SearchResult'; +export * from './SidebarSearch'; diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts index 224e293890..77ad7f9266 100644 --- a/plugins/search/src/index.ts +++ b/plugins/search/src/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export { plugin } from './plugin'; +export * from './components'; From 74c43ce23193ea6eb4db4f3109be32cc3e152f6c Mon Sep 17 00:00:00 2001 From: Shashank Bairy R Date: Fri, 20 Nov 2020 19:52:49 +0530 Subject: [PATCH 192/259] feat: improve search match (#3365) --- plugins/search/src/components/SearchResult/SearchResult.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index e4163c0de7..1670c13963 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -159,6 +159,9 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => { withFilters = withFilters.filter( (result: Result) => result.name?.toLowerCase().includes(searchQuery) || + result.name + ?.toLowerCase() + .includes(searchQuery.split(' ').join('-')) || result.description?.toLowerCase().includes(searchQuery), ); } From ece530ab9538231885220c015158d89edd857a44 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 15:42:25 +0100 Subject: [PATCH 193/259] remove dependabot config --- .github/dependabot.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 59847fd65f..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 -updates: - - package-ecosystem: npm - directory: '/' - schedule: - interval: daily - time: '04:00' - open-pull-requests-limit: 5 - labels: - - dependencies - - package-ecosystem: npm - directory: '/microsite/' - schedule: - interval: daily - time: '04:00' - open-pull-requests-limit: 2 - labels: - - dependencies From 7a8476a3fba78f21e35ad567670838ea180b0917 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 15:44:18 +0100 Subject: [PATCH 194/259] Create dependabot.yml --- .github/dependabot.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..c3ece8a6f0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: '/' + schedule: + interval: daily + time: '04:00' + open-pull-requests-limit: 5 + labels: + - dependencies + - package-ecosystem: npm + directory: '/microsite/' + schedule: + interval: daily + time: '04:00' + open-pull-requests-limit: 2 + labels: + - dependencies From 2c74aa7e796e7fa7eceef63325d47b3a9b72e3c1 Mon Sep 17 00:00:00 2001 From: Mateusz Lewtak Date: Fri, 20 Nov 2020 15:50:33 +0100 Subject: [PATCH 195/259] Feat: update plugin versions --- packages/app/package.json | 8 +- .../components/catalog/EntityPage.test.tsx | 6 +- .../app/src/components/catalog/EntityPage.tsx | 8 +- packages/app/src/plugins.ts | 2 +- yarn.lock | 86 ++++++------------- 5 files changed, 37 insertions(+), 73 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index 529c5d4f63..042fd47188 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -35,10 +35,10 @@ "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@octokit/rest": "^18.0.0", - "@roadiehq/backstage-plugin-github-insights": "^0.2.14", - "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.2", - "@roadiehq/backstage-plugin-travis-ci": "^0.2.7", - "@roadiehq/backstage-plugin-buildkite": "^0.1.2", + "@roadiehq/backstage-plugin-github-insights": "^0.2.15", + "@roadiehq/backstage-plugin-github-pull-requests": "^0.6.3", + "@roadiehq/backstage-plugin-travis-ci": "^0.2.8", + "@roadiehq/backstage-plugin-buildkite": "^0.1.3", "history": "^5.0.0", "prop-types": "^15.7.2", "react": "^16.12.0", diff --git a/packages/app/src/components/catalog/EntityPage.test.tsx b/packages/app/src/components/catalog/EntityPage.test.tsx index f4dfafc7d3..1392f77a6d 100644 --- a/packages/app/src/components/catalog/EntityPage.test.tsx +++ b/packages/app/src/components/catalog/EntityPage.test.tsx @@ -18,7 +18,7 @@ import { CICDSwitcher } from './EntityPage'; import { UrlPatternDiscovery, ApiProvider, ApiRegistry } from '@backstage/core'; import { buildKiteApiRef, - BuildKiteApi, + BuildkiteApi, } from '@roadiehq/backstage-plugin-buildkite'; import { renderWithEffects, wrapInTestApp } from '@backstage/test-utils'; @@ -42,11 +42,11 @@ describe('EntityPage Test', () => { const discoveryApi = UrlPatternDiscovery.compile('http://exampleapi.com'); const apis = ApiRegistry.from([ - [buildKiteApiRef, new BuildKiteApi({ discoveryApi })], + [buildKiteApiRef, new BuildkiteApi({ discoveryApi })], ]); describe('CICDSwitcher Test', () => { - it('Should render BuildKite View', async () => { + it('Should render Buildkite View', async () => { const renderedComponent = await renderWithEffects( wrapInTestApp( diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index 73d84ef1ca..d826bfa25e 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -67,8 +67,8 @@ import { PullRequestsStatsCard, } from '@roadiehq/backstage-plugin-github-pull-requests'; import { - Router as BuildKiteRouter, - isPluginApplicableToEntity as isBuildKiteAvailable, + Router as BuildkiteRouter, + isPluginApplicableToEntity as isBuildkiteAvailable, } from '@roadiehq/backstage-plugin-buildkite'; export const CICDSwitcher = ({ entity }: { entity: Entity }) => { @@ -77,8 +77,8 @@ export const CICDSwitcher = ({ entity }: { entity: Entity }) => { switch (true) { case isJenkinsAvailable(entity): return ; - case isBuildKiteAvailable(entity): - return ; + case isBuildkiteAvailable(entity): + return ; case isGitHubActionsAvailable(entity): return ; case isCircleCIAvailable(entity): diff --git a/packages/app/src/plugins.ts b/packages/app/src/plugins.ts index d6577b4ce4..87b4c0ceea 100644 --- a/packages/app/src/plugins.ts +++ b/packages/app/src/plugins.ts @@ -38,5 +38,5 @@ export { plugin as Cloudbuild } from '@backstage/plugin-cloudbuild'; export { plugin as CostInsights } from '@backstage/plugin-cost-insights'; export { plugin as GitHubInsights } from '@roadiehq/backstage-plugin-github-insights'; export { plugin as UserSettings } from '@backstage/plugin-user-settings'; -export { plugin as BuildKite } from '@roadiehq/backstage-plugin-buildkite'; +export { plugin as Buildkite } from '@roadiehq/backstage-plugin-buildkite'; export { plugin as Search } from '@backstage/plugin-search'; diff --git a/yarn.lock b/yarn.lock index 58d9c546ee..96e3c4bf89 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1307,42 +1307,6 @@ lodash "^4.17.19" to-fast-properties "^2.0.0" -"@backstage/core@^0.2.0": - version "0.3.1" - dependencies: - "@backstage/config" "^0.1.1" - "@backstage/core-api" "^0.2.1" - "@backstage/theme" "^0.2.1" - "@material-ui/core" "^4.11.0" - "@material-ui/icons" "^4.9.1" - "@material-ui/lab" "4.0.0-alpha.45" - "@types/dagre" "^0.7.44" - "@types/react" "^16.9" - "@types/react-sparklines" "^1.7.0" - classnames "^2.2.6" - clsx "^1.1.0" - d3-selection "^2.0.0" - d3-shape "^2.0.0" - d3-zoom "^2.0.0" - dagre "^0.8.5" - immer "^7.0.9" - lodash "^4.17.15" - material-table "^1.69.1" - prop-types "^15.7.2" - qs "^6.9.4" - rc-progress "^3.0.0" - react "^16.12.0" - react-dom "^16.12.0" - react-helmet "6.1.0" - react-hook-form "^6.6.0" - react-markdown "^5.0.2" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-sparklines "^1.7.0" - react-syntax-highlighter "^13.5.1" - react-use "^15.3.3" - remark-gfm "^1.0.0" - "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -3710,14 +3674,14 @@ resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.4.0.tgz#1b5859298bf3f61137d7b05084f058a775d6fd73" integrity sha512-U8F/suzg4MuV+8mK1/ufs0Y6c3O8hc1wnuD2IKoOVJvegGfz5JCafyoyGAW6iyuT1DZBMPzVWEqfiuYPmoE7pw== -"@roadiehq/backstage-plugin-buildkite@^0.1.2": - version "0.1.2" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-buildkite/-/backstage-plugin-buildkite-0.1.2.tgz#2f2b414acc18ed7820abc6fb0c042f00c31d7c9a" - integrity sha512-NO1ogAK6lfh/YUmftqbYn6pkUmMJXk9cfda0YDnG23XlcKIst1d3xO5FggozVtU5IZiJDxxwxOOo4ylhBWrGeQ== +"@roadiehq/backstage-plugin-buildkite@^0.1.3": + version "0.1.3" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-buildkite/-/backstage-plugin-buildkite-0.1.3.tgz#5a116bf677dfad22088212dde398cf3e9b48d6e4" + integrity sha512-q+cnAvZmjLu0DcqRKJZJhWTVcKaA9j7cM44tx6JcshEyb3UgzrlECLvx4ethnYyrmhM1/FSpMLa+t3N5A1Xz4g== dependencies: "@backstage/catalog-model" "^0.2.0" - "@backstage/core" "^0.2.0" - "@backstage/plugin-catalog" "^0.2.0" + "@backstage/core" "^0.3.0" + "@backstage/plugin-catalog" "^0.2.1" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" @@ -3729,14 +3693,14 @@ react-router-dom "6.0.0-beta.0" react-use "^15.3.3" -"@roadiehq/backstage-plugin-github-insights@^0.2.14": - version "0.2.14" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.14.tgz#2e4bf61495e650bb2b7a1711f97c9c4995215588" - integrity sha512-xhzHAmmTu7op1V7K3ytomGlmzHMs9jnb2Lc2YTrU3ame4WZcrrMCrqi8X9v8B8VHfwMBrwa55tyxRYObXrPr7A== +"@roadiehq/backstage-plugin-github-insights@^0.2.15": + version "0.2.15" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-insights/-/backstage-plugin-github-insights-0.2.15.tgz#b127e7795d3a2440286548fc33af49f12ba42b7f" + integrity sha512-4jLDHlr5B77Kveb7Q1q78mbMATufJLgS/zy4T+1LW/54KKmJrset4AWrZLTAOwjkjOfDfO85NEwmgsbrprAGrg== dependencies: "@backstage/catalog-model" "^0.2.0" - "@backstage/core" "^0.2.0" - "@backstage/theme" "^0.2.0" + "@backstage/core" "^0.3.0" + "@backstage/theme" "^0.2.1" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" @@ -3748,13 +3712,13 @@ react-router "^6.0.0-beta.0" react-use "^15.3.3" -"@roadiehq/backstage-plugin-github-pull-requests@^0.6.2": - version "0.6.2" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.6.2.tgz#02f4a7a03e1a7dc24342ec695130017ce7d3ad8d" - integrity sha512-OdBWO6NdiBKlol1WlbFQGeHxV2C8wl8EiI2NgA15lozsqFA22kWis6Z16cAHe4ZdRCjihDmqINtAIJzeZZ1gOg== +"@roadiehq/backstage-plugin-github-pull-requests@^0.6.3": + version "0.6.3" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.6.3.tgz#46b63e90f3f5412a4b8f0df96ada62cff7046478" + integrity sha512-ofWH9k4WVVwTbK/XAVqmtH03QW3rsT822p4neOse0Wy0dAKlqlSU4nwl5jBKMJXsf8lfc0c7gbc50R7VumzoOQ== dependencies: "@backstage/catalog-model" "^0.2.0" - "@backstage/core" "^0.2.0" + "@backstage/core" "^0.3.0" "@backstage/plugin-catalog" "^0.2.0" "@backstage/theme" "^0.2.0" "@material-ui/core" "^4.11.0" @@ -3770,16 +3734,16 @@ react-router "6.0.0-beta.0" react-use "^15.3.3" -"@roadiehq/backstage-plugin-travis-ci@^0.2.7": - version "0.2.7" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-travis-ci/-/backstage-plugin-travis-ci-0.2.7.tgz#bc7968b461016b2710794d10766266de9ab4f759" - integrity sha512-uXF5t2uZqd9TNGFSMYlyk6NAweVb5KLlM4GMiltattzxRWiqbCp5VumubrLTTsA8dvsWCrXWfhe95MStcUnl+A== +"@roadiehq/backstage-plugin-travis-ci@^0.2.8": + version "0.2.8" + resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-travis-ci/-/backstage-plugin-travis-ci-0.2.8.tgz#c730531519bf3e3dba35522d5e6c822176ddb9c8" + integrity sha512-8R8waHoviT5chHGybAMVVRHHmeytn+8Po7sU08WCYIt4Vkw2/gwu36fd90y+lW4x6T0s3ZiaJrsQ3KlA7LmRnQ== dependencies: "@backstage/catalog-model" "^0.2.0" - "@backstage/core" "^0.2.0" - "@backstage/core-api" "^0.2.0" - "@backstage/plugin-catalog" "^0.2.0" - "@backstage/theme" "^0.2.0" + "@backstage/core" "^0.3.0" + "@backstage/core-api" "^0.2.1" + "@backstage/plugin-catalog" "^0.2.1" + "@backstage/theme" "^0.2.1" "@material-ui/core" "^4.9.1" "@material-ui/icons" "^4.9.1" "@material-ui/lab" "4.0.0-alpha.45" From 0a559f4500e015cfb719a61d4d74e9b8cef1437f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Nov 2020 15:51:17 +0100 Subject: [PATCH 196/259] Update dependabot.yml --- .github/dependabot.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c3ece8a6f0..59847fd65f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,18 +1,18 @@ -version: 2 -updates: - - package-ecosystem: npm - directory: '/' - schedule: - interval: daily - time: '04:00' - open-pull-requests-limit: 5 - labels: - - dependencies - - package-ecosystem: npm - directory: '/microsite/' - schedule: - interval: daily - time: '04:00' - open-pull-requests-limit: 2 - labels: +version: 2 +updates: + - package-ecosystem: npm + directory: '/' + schedule: + interval: daily + time: '04:00' + open-pull-requests-limit: 5 + labels: + - dependencies + - package-ecosystem: npm + directory: '/microsite/' + schedule: + interval: daily + time: '04:00' + open-pull-requests-limit: 2 + labels: - dependencies From 36b9dfd41cae69b32bdfa78d0fc8aef757e4f3fa Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 20 Nov 2020 16:20:13 +0100 Subject: [PATCH 197/259] Rework the search field behavior in the sidebar We just tested it with another user an noticed some more issues: * If the user is on the search route it should be reflected in the sidebar * Clicking on the search icon alone should lead to the search page --- packages/core/src/layout/Sidebar/Items.tsx | 12 +++++++++--- .../src/components/SidebarSearch/SidebarSearch.tsx | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/core/src/layout/Sidebar/Items.tsx b/packages/core/src/layout/Sidebar/Items.tsx index 498946577d..b8786886cc 100644 --- a/packages/core/src/layout/Sidebar/Items.tsx +++ b/packages/core/src/layout/Sidebar/Items.tsx @@ -26,7 +26,6 @@ import { IconComponent } from '@backstage/core-api'; import SearchIcon from '@material-ui/icons/Search'; import clsx from 'clsx'; import React, { - FC, useContext, useState, KeyboardEventHandler, @@ -212,9 +211,10 @@ export const SidebarItem = forwardRef( type SidebarSearchFieldProps = { onSearch: (input: string) => void; + to?: string; }; -export const SidebarSearchField: FC = props => { +export const SidebarSearchField = (props: SidebarSearchFieldProps) => { const [input, setInput] = useState(''); const classes = useStyles(); @@ -229,12 +229,18 @@ export const SidebarSearchField: FC = props => { setInput(ev.target.value); }; + const handleClick = (ev: React.MouseEvent) => { + // Clicking into the search fields shouldn't navigate to the search page + ev.preventDefault(); + }; + return (
- + { [navigate], ); - return ; + return ; }; From 9d69a87ee28d8f271a26f4a8749cb7260c4f4ad9 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Fri, 20 Nov 2020 10:26:11 -0500 Subject: [PATCH 198/259] truncate large percentages --- .../src/components/BarChart/BarChartTooltip.tsx | 11 ++++++++--- .../cost-insights/src/utils/formatters.test.ts | 15 +++++++++++++++ plugins/cost-insights/src/utils/formatters.ts | 6 ++++++ plugins/cost-insights/src/utils/styles.ts | 3 +++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx index 8ab787a443..64a35e2658 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx @@ -15,6 +15,7 @@ */ import React, { ReactNode, PropsWithChildren } from 'react'; +import classnames from 'classnames'; import { Box, Divider, Typography } from '@material-ui/core'; import { useTooltipStyles as useStyles } from '../../utils/styles'; @@ -35,6 +36,10 @@ export const BarChartTooltip = ({ children, }: PropsWithChildren) => { const classes = useStyles(); + const titleClassName = classnames(classes.truncate, { + [classes.maxWidth]: topRight === undefined, + }); + return ( - + {title} {subtitle && ( @@ -55,10 +60,10 @@ export const BarChartTooltip = ({ )} - {topRight && {topRight}} + {topRight && {topRight}} {content && ( - + {content} diff --git a/plugins/cost-insights/src/utils/formatters.test.ts b/plugins/cost-insights/src/utils/formatters.test.ts index 69e9086be1..db25b6f3b7 100644 --- a/plugins/cost-insights/src/utils/formatters.test.ts +++ b/plugins/cost-insights/src/utils/formatters.test.ts @@ -16,6 +16,7 @@ import { formatPeriod, + formatPercent, lengthyCurrencyFormatter, quarterOf, } from './formatters'; @@ -69,3 +70,17 @@ describe.each` expect(formatPeriod(duration, date, isEndDate)).toBe(output); }); }); + +describe.each` + ratio | expected + ${0.0} | ${'0%'} + ${0.000000000001} | ${'0%'} + ${-0.00000000001} | ${'0%'} + ${0.123123} | ${'12%'} + ${1.123} | ${'112%'} + ${10.123} | ${'>1000%'} +`('formatPercent', ({ ratio, expected }) => { + it(`correctly formats ${ratio} as ${expected}`, () => { + expect(formatPercent(ratio)).toBe(expected); + }); +}); diff --git a/plugins/cost-insights/src/utils/formatters.ts b/plugins/cost-insights/src/utils/formatters.ts index 42505e7072..19255d3c56 100644 --- a/plugins/cost-insights/src/utils/formatters.ts +++ b/plugins/cost-insights/src/utils/formatters.ts @@ -83,9 +83,15 @@ export function formatPercent(n: number): string { if (isNaN(n) || Math.abs(n) < 0.01) { return '0%'; } + + if (Math.abs(n) > 10) { + return `>1000%`; + } + if (Math.abs(n) >= 1e19) { return '∞%'; } + return `${(n * 100).toFixed(0)}%`; } diff --git a/plugins/cost-insights/src/utils/styles.ts b/plugins/cost-insights/src/utils/styles.ts index 1e8fd50899..542896d6fc 100644 --- a/plugins/cost-insights/src/utils/styles.ts +++ b/plugins/cost-insights/src/utils/styles.ts @@ -386,6 +386,9 @@ export const useTooltipStyles = makeStyles( boxShadow: theme.shadows[1], color: theme.palette.tooltip.color, fontSize: theme.typography.fontSize, + minWidth: 300, + }, + maxWidth: { maxWidth: 300, }, actions: { From c93a14b496cb3dd005dc1b96b23179bc96fc34c0 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Fri, 20 Nov 2020 10:29:14 -0500 Subject: [PATCH 199/259] changeset --- .changeset/cost-insights-tiny-llamas-perform.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cost-insights-tiny-llamas-perform.md diff --git a/.changeset/cost-insights-tiny-llamas-perform.md b/.changeset/cost-insights-tiny-llamas-perform.md new file mode 100644 index 0000000000..c81c28f8f8 --- /dev/null +++ b/.changeset/cost-insights-tiny-llamas-perform.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': patch +--- + +truncate large percentages > 1000% From 4ff4a25e4096cc9dc7084d12ecd7d845cfc7fc67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Nov 2020 15:33:39 +0000 Subject: [PATCH 200/259] build(deps-dev): bump prettier from 2.1.2 to 2.2.0 in /microsite Bumps [prettier](https://github.com/prettier/prettier) from 2.1.2 to 2.2.0. - [Release notes](https://github.com/prettier/prettier/releases) - [Changelog](https://github.com/prettier/prettier/blob/master/CHANGELOG.md) - [Commits](https://github.com/prettier/prettier/compare/2.1.2...2.2.0) Signed-off-by: dependabot[bot] --- microsite/package.json | 2 +- microsite/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/microsite/package.json b/microsite/package.json index a84364c214..3b5ab2eeb4 100644 --- a/microsite/package.json +++ b/microsite/package.json @@ -17,7 +17,7 @@ "@spotify/prettier-config": "^9.0.0", "docusaurus": "^2.0.0-alpha.66", "js-yaml": "^3.14.0", - "prettier": "^2.0.5" + "prettier": "^2.2.0" }, "prettier": "@spotify/prettier-config" } diff --git a/microsite/yarn.lock b/microsite/yarn.lock index b120cb2d47..2147373bfe 100644 --- a/microsite/yarn.lock +++ b/microsite/yarn.lock @@ -5204,10 +5204,10 @@ prepend-http@^2.0.0: resolved "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc= -prettier@^2.0.5: - version "2.1.2" - resolved "https://registry.npmjs.org/prettier/-/prettier-2.1.2.tgz#3050700dae2e4c8b67c4c3f666cdb8af405e1ce5" - integrity sha512-16c7K+x4qVlJg9rEbXl7HEGmQyZlG4R9AgP+oHKRMsMsuk8s+ATStlf1NpDqyBI1HpVyfjLOeMhH2LvuNvV5Vg== +prettier@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.0.tgz#8a03c7777883b29b37fb2c4348c66a78e980418b" + integrity sha512-yYerpkvseM4iKD/BXLYUkQV5aKt4tQPqaGW6EsZjzyu0r7sVZZNPJW4Y8MyKmicp6t42XUPcBVA+H6sB3gqndw== prismjs@^1.17.1: version "1.21.0" From 640c59fae735e4fb251f5678a4008da7ff8b2f87 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Fri, 20 Nov 2020 10:59:34 -0500 Subject: [PATCH 201/259] remove unreachable condition --- plugins/cost-insights/src/utils/formatters.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/cost-insights/src/utils/formatters.ts b/plugins/cost-insights/src/utils/formatters.ts index 19255d3c56..9c3b2d643e 100644 --- a/plugins/cost-insights/src/utils/formatters.ts +++ b/plugins/cost-insights/src/utils/formatters.ts @@ -88,10 +88,6 @@ export function formatPercent(n: number): string { return `>1000%`; } - if (Math.abs(n) >= 1e19) { - return '∞%'; - } - return `${(n * 100).toFixed(0)}%`; } From 26484d413304cdfacaa6a53af9a7fc6678a2213d Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Fri, 20 Nov 2020 16:54:10 +0100 Subject: [PATCH 202/259] Add config schema for the sonarqube plugin --- .changeset/long-flowers-wink.md | 5 +++++ plugins/sonarqube/config.d.ts | 26 ++++++++++++++++++++++++++ plugins/sonarqube/package.json | 6 ++++-- 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 .changeset/long-flowers-wink.md create mode 100644 plugins/sonarqube/config.d.ts diff --git a/.changeset/long-flowers-wink.md b/.changeset/long-flowers-wink.md new file mode 100644 index 0000000000..7f9cf12315 --- /dev/null +++ b/.changeset/long-flowers-wink.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-sonarqube': patch +--- + +Add configuration schema diff --git a/plugins/sonarqube/config.d.ts b/plugins/sonarqube/config.d.ts new file mode 100644 index 0000000000..5facd9cef7 --- /dev/null +++ b/plugins/sonarqube/config.d.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Optional configurations for the SonarQube plugin */ + sonarQube?: { + /** + * The base url of the sonarqube installation. Defaults to https://sonarcloud.io. + * @visibility frontend + */ + baseUrl: string; + }; +} diff --git a/plugins/sonarqube/package.json b/plugins/sonarqube/package.json index dcde8fab14..be8a878e50 100644 --- a/plugins/sonarqube/package.json +++ b/plugins/sonarqube/package.json @@ -47,6 +47,8 @@ "msw": "^0.21.2" }, "files": [ - "dist" - ] + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" } From b3fb9a08cee409ec309b041e96db4048ffaa41f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Nov 2020 16:41:01 +0000 Subject: [PATCH 203/259] build(deps-dev): bump @types/lodash from 4.14.161 to 4.14.165 Bumps [@types/lodash](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/lodash) from 4.14.161 to 4.14.165. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/lodash) Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index a90d885927..bd1757cea0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5434,9 +5434,9 @@ "@types/node" "*" "@types/lodash@^4.14.151": - version "4.14.161" - resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.161.tgz#a21ca0777dabc6e4f44f3d07f37b765f54188b18" - integrity sha512-EP6O3Jkr7bXvZZSZYlsgt5DIjiGr0dXP1/jVEwVLTFgg0d+3lWVQkRavYVQszV7dYUwvg0B8R0MBDpcmXg7XIA== + version "4.14.165" + resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.165.tgz#74d55d947452e2de0742bad65270433b63a8c30f" + integrity sha512-tjSSOTHhI5mCHTy/OOXYIhi2Wt1qcbHmuXD1Ha7q70CgI/I71afO4XtLb/cVexki1oVYchpul/TOuu3Arcdxrg== "@types/long@^4.0.0": version "4.0.1" From 4afc446c24caac9d508aa879ab8c70861558e066 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 20 Nov 2020 18:17:58 +0100 Subject: [PATCH 204/259] fixes dark mode issue on searchQuery + changes its class name from searchTerm to be more consistent --- .../search/src/components/SearchResult/SearchResult.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index 1670c13963..a5dc11aa89 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -31,8 +31,9 @@ import { FiltersButton, Filters, FiltersState } from '../Filters'; import SearchApi, { Result, SearchResults } from '../../apis'; const useStyles = makeStyles(theme => ({ - searchTerm: { - background: '#eee', + searchQuery: { + color: theme.palette.text.primary, + background: theme.palette.background.default, borderRadius: '10%', }, tableHeader: { @@ -107,7 +108,7 @@ const TableHeader = ({ {`${numberOfResults} `} {numberOfResults > 1 ? `results for ` : `result for `} - "{searchQuery}"{' '} + "{searchQuery}"{' '} ) : ( {`${numberOfResults} results`} From 8d948b243b778378622ffa76d19b0d6eb8d76bec Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Mon, 16 Nov 2020 14:04:24 -0800 Subject: [PATCH 205/259] Implement a generic OpenIdConnect auth-provider --- app-config.yaml | 14 ++ packages/app/src/identityProviders.ts | 7 + .../core-api/src/apis/definitions/auth.ts | 14 ++ .../src/apis/implementations/auth/index.ts | 1 + .../implementations/auth/oidc/Oidc.test.ts | 152 ++++++++++++ .../apis/implementations/auth/oidc/Oidc.ts | 183 +++++++++++++++ .../apis/implementations/auth/oidc/index.ts | 18 ++ .../apis/implementations/auth/oidc/types.ts | 28 +++ packages/core/src/api-wrappers/defaultApis.ts | 11 + plugins/auth-backend/package.json | 4 + .../auth-backend/src/providers/factories.ts | 2 + .../auth-backend/src/providers/oidc/index.ts | 17 ++ .../src/providers/oidc/provider.ts | 216 ++++++++++++++++++ plugins/auth-backend/src/service/router.ts | 5 +- yarn.lock | 58 ++++- 15 files changed, 727 insertions(+), 3 deletions(-) create mode 100644 packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts create mode 100644 packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts create mode 100644 packages/core-api/src/apis/implementations/auth/oidc/index.ts create mode 100644 packages/core-api/src/apis/implementations/auth/oidc/types.ts create mode 100644 plugins/auth-backend/src/providers/oidc/index.ts create mode 100644 plugins/auth-backend/src/providers/oidc/provider.ts diff --git a/app-config.yaml b/app-config.yaml index 32d3470f1b..cfa47618e8 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -235,6 +235,20 @@ auth: $env: AUTH_OAUTH2_AUTH_URL tokenUrl: $env: AUTH_OAUTH2_TOKEN_URL + oidc: + development: + adUrl: + $env: AUTH_OIDC_AD_URL + clientId: + $env: AUTH_OIDC_CLIENT_ID + clientSecret: + $env: AUTH_OIDC_CLIENT_SECRET + authorizationUrl: + $env: AUTH_OIDC_AUTH_URL + tokenUrl: + $env: AUTH_OIDC_TOKEN_URL + tokenSignedResponseAlg: + $env: AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG auth0: development: clientId: diff --git a/packages/app/src/identityProviders.ts b/packages/app/src/identityProviders.ts index 06ff6b07d3..ba657328f2 100644 --- a/packages/app/src/identityProviders.ts +++ b/packages/app/src/identityProviders.ts @@ -22,9 +22,16 @@ import { samlAuthApiRef, microsoftAuthApiRef, oneloginAuthApiRef, + oidcApiRef, } from '@backstage/core'; export const providers = [ + { + id: 'oidc-auth-provider', + title: 'Oidc', + message: 'Sign In using OpenId Connect', + apiRef: oidcApiRef, + }, { id: 'google-auth-provider', title: 'Google', diff --git a/packages/core-api/src/apis/definitions/auth.ts b/packages/core-api/src/apis/definitions/auth.ts index c538065f3a..5dae5532ad 100644 --- a/packages/core-api/src/apis/definitions/auth.ts +++ b/packages/core-api/src/apis/definitions/auth.ts @@ -311,6 +311,20 @@ export const oauth2ApiRef: ApiRef< description: 'Example of how to use oauth2 custom provider', }); +/** + * Provides authentication for custom OpenID Connect identity providers. + */ +export const oidcApiRef: ApiRef< + OAuthApi & + OpenIdConnectApi & + ProfileInfoApi & + BackstageIdentityApi & + SessionApi +> = createApiRef({ + id: 'core.auth.oidc', + description: 'Example of how to use oidc custom provider', +}); + /** * Provides authentication for saml based identity providers */ diff --git a/packages/core-api/src/apis/implementations/auth/index.ts b/packages/core-api/src/apis/implementations/auth/index.ts index 7ef78d19cf..5adfc6f621 100644 --- a/packages/core-api/src/apis/implementations/auth/index.ts +++ b/packages/core-api/src/apis/implementations/auth/index.ts @@ -23,3 +23,4 @@ export * from './saml'; export * from './auth0'; export * from './microsoft'; export * from './onelogin'; +export * from './oidc'; diff --git a/packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts b/packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts new file mode 100644 index 0000000000..4286fe7e5d --- /dev/null +++ b/packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import OpenIdConnect from './Oidc'; + +const theFuture = new Date(Date.now() + 3600000); +const thePast = new Date(Date.now() - 10); + +const PREFIX = 'https://www.googleapis.com/auth/'; + +const scopeTransform = (x: string[]) => x; + +describe('OpenIdConnect', () => { + it('should get refreshed access token', async () => { + const getSession = jest.fn().mockResolvedValue({ + providerInfo: { accessToken: 'access-token', expiresAt: theFuture }, + }); + const openIdConnect = new OpenIdConnect({ + sessionManager: { getSession } as any, + scopeTransform, + }); + + expect(await openIdConnect.getAccessToken('my-scope my-scope2')).toBe( + 'access-token', + ); + expect(getSession).toBeCalledTimes(1); + expect(getSession.mock.calls[0][0].scopes).toEqual( + new Set(['my-scope', 'my-scope2']), + ); + }); + + it('should transform scopes', async () => { + const getSession = jest.fn().mockResolvedValue({ + providerInfo: { accessToken: 'access-token', expiresAt: theFuture }, + }); + const openIdConnect = new OpenIdConnect({ + sessionManager: { getSession } as any, + scopeTransform: scopes => scopes.map(scope => `my-prefix/${scope}`), + }); + + expect(await openIdConnect.getAccessToken('my-scope')).toBe('access-token'); + expect(getSession).toBeCalledTimes(1); + expect(getSession.mock.calls[0][0].scopes).toEqual( + new Set(['my-prefix/my-scope']), + ); + }); + + it('should get refreshed id token', async () => { + const getSession = jest.fn().mockResolvedValue({ + providerInfo: { idToken: 'id-token', expiresAt: theFuture }, + }); + const openIdConnect = new OpenIdConnect({ + sessionManager: { getSession } as any, + scopeTransform, + }); + + expect(await openIdConnect.getIdToken()).toBe('id-token'); + expect(getSession).toBeCalledTimes(1); + }); + + it('should get optional id token', async () => { + const getSession = jest.fn().mockResolvedValue({ + providerInfo: { idToken: 'id-token', expiresAt: theFuture }, + }); + const openIdConnect = new OpenIdConnect({ + sessionManager: { getSession } as any, + scopeTransform, + }); + + expect(await openIdConnect.getIdToken({ optional: true })).toBe('id-token'); + expect(getSession).toBeCalledTimes(1); + }); + + it('should share popup closed errors', async () => { + const error = new Error('NOPE'); + error.name = 'RejectedError'; + const getSession = jest + .fn() + .mockResolvedValueOnce({ + providerInfo: { + accessToken: 'access-token', + expiresAt: theFuture, + scopes: new Set([`${PREFIX}not-enough`]), + }, + }) + .mockRejectedValue(error); + const openIdConnect = new OpenIdConnect({ + sessionManager: { getSession } as any, + scopeTransform, + }); + + // Make sure we have a session before we do the double request, so that we get past the !this.currentSession check + await expect(openIdConnect.getAccessToken()).resolves.toBe('access-token'); + + const promise1 = openIdConnect.getAccessToken('more'); + const promise2 = openIdConnect.getAccessToken('more'); + await expect(promise1).rejects.toBe(error); + await expect(promise2).rejects.toBe(error); + expect(getSession).toBeCalledTimes(3); + }); + + it('should wait for all session refreshes', async () => { + const initialSession = { + providerInfo: { + idToken: 'token1', + expiresAt: theFuture, + scopes: new Set(), + }, + }; + const getSession = jest + .fn() + .mockResolvedValueOnce(initialSession) + .mockResolvedValue({ + providerInfo: { + idToken: 'token2', + expiresAt: theFuture, + scopes: new Set(), + }, + }); + const openIdConnect = new OpenIdConnect({ + sessionManager: { getSession } as any, + scopeTransform, + }); + + // Grab the expired session first + await expect(openIdConnect.getIdToken()).resolves.toBe('token1'); + expect(getSession).toBeCalledTimes(1); + + initialSession.providerInfo.expiresAt = thePast; + + const promise1 = openIdConnect.getIdToken(); + const promise2 = openIdConnect.getIdToken(); + const promise3 = openIdConnect.getIdToken(); + await expect(promise1).resolves.toBe('token2'); + await expect(promise2).resolves.toBe('token2'); + await expect(promise3).resolves.toBe('token2'); + expect(getSession).toBeCalledTimes(4); // De-duping of session requests happens in client + }); +}); diff --git a/packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts b/packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts new file mode 100644 index 0000000000..deb8331432 --- /dev/null +++ b/packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts @@ -0,0 +1,183 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import OpenIdConnectIcon from '@material-ui/icons/AcUnit'; +import { DefaultAuthConnector } from '../../../../lib/AuthConnector'; +import { RefreshingAuthSessionManager } from '../../../../lib/AuthSessionManager'; +import { SessionManager } from '../../../../lib/AuthSessionManager/types'; +import { Observable } from '../../../../types'; +import { + AuthRequestOptions, + BackstageIdentity, + OAuthApi, + OpenIdConnectApi, + ProfileInfo, + ProfileInfoApi, + SessionState, + SessionApi, + BackstageIdentityApi, +} from '../../../definitions/auth'; +import { OpenIdConnectSession } from './types'; +import { OAuthApiCreateOptions } from '../types'; + +type Options = { + sessionManager: SessionManager; + scopeTransform: (scopes: string[]) => string[]; +}; + +type CreateOptions = OAuthApiCreateOptions & { + scopeTransform?: (scopes: string[]) => string[]; +}; + +export type OpenIdConnectResponse = { + providerInfo: { + accessToken: string; + idToken: string; + scope: string; + expiresInSeconds: number; + }; + profile: ProfileInfo; + backstageIdentity: BackstageIdentity; +}; + +const DEFAULT_PROVIDER = { + id: 'oidc', + title: 'Open ID Connect Identity Provider', + icon: OpenIdConnectIcon, +}; + +class OpenIdConnect + implements + OAuthApi, + OpenIdConnectApi, + ProfileInfoApi, + BackstageIdentityApi, + SessionApi { + static create({ + discoveryApi, + environment = 'development', + provider = DEFAULT_PROVIDER, + oauthRequestApi, + defaultScopes = [], + scopeTransform = x => x, + }: CreateOptions) { + const connector = new DefaultAuthConnector({ + discoveryApi, + environment, + provider, + oauthRequestApi: oauthRequestApi, + sessionTransform(res: OpenIdConnectResponse): OpenIdConnectSession { + return { + ...res, + providerInfo: { + idToken: res.providerInfo.idToken, + accessToken: res.providerInfo.accessToken, + scopes: OpenIdConnect.normalizeScopes( + scopeTransform, + res.providerInfo.scope, + ), + expiresAt: new Date( + Date.now() + res.providerInfo.expiresInSeconds * 1000, + ), + }, + }; + }, + }); + + const sessionManager = new RefreshingAuthSessionManager({ + connector, + defaultScopes: new Set(defaultScopes), + sessionScopes: (session: OpenIdConnectSession) => + session.providerInfo.scopes, + sessionShouldRefresh: (session: OpenIdConnectSession) => { + const expiresInSec = + (session.providerInfo.expiresAt.getTime() - Date.now()) / 1000; + return expiresInSec < 60 * 5; + }, + }); + + return new OpenIdConnect({ sessionManager, scopeTransform }); + } + + private readonly sessionManager: SessionManager; + private readonly scopeTransform: (scopes: string[]) => string[]; + + constructor(options: Options) { + this.sessionManager = options.sessionManager; + this.scopeTransform = options.scopeTransform; + } + + async signIn() { + await this.getAccessToken(); + } + + async signOut() { + await this.sessionManager.removeSession(); + } + + sessionState$(): Observable { + return this.sessionManager.sessionState$(); + } + + async getAccessToken( + scope?: string | string[], + options?: AuthRequestOptions, + ) { + const normalizedScopes = OpenIdConnect.normalizeScopes( + this.scopeTransform, + scope, + ); + const session = await this.sessionManager.getSession({ + ...options, + scopes: normalizedScopes, + }); + return session?.providerInfo.accessToken ?? ''; + } + + async getIdToken(options: AuthRequestOptions = {}) { + const session = await this.sessionManager.getSession(options); + return session?.providerInfo.idToken ?? ''; + } + + async getBackstageIdentity( + options: AuthRequestOptions = {}, + ): Promise { + const session = await this.sessionManager.getSession(options); + return session?.backstageIdentity; + } + + async getProfile(options: AuthRequestOptions = {}) { + const session = await this.sessionManager.getSession(options); + return session?.profile; + } + + private static normalizeScopes( + scopeTransform: (scopes: string[]) => string[], + scopes?: string | string[], + ): Set { + if (!scopes) { + return new Set(); + } + + const scopeList = Array.isArray(scopes) + ? scopes + : scopes.split(/[\s|,]/).filter(Boolean); + + return new Set(scopeTransform(scopeList)); + } +} + +export default OpenIdConnect; diff --git a/packages/core-api/src/apis/implementations/auth/oidc/index.ts b/packages/core-api/src/apis/implementations/auth/oidc/index.ts new file mode 100644 index 0000000000..cf9add0757 --- /dev/null +++ b/packages/core-api/src/apis/implementations/auth/oidc/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { default as OpenIdConnect } from './Oidc'; +export * from './types'; diff --git a/packages/core-api/src/apis/implementations/auth/oidc/types.ts b/packages/core-api/src/apis/implementations/auth/oidc/types.ts new file mode 100644 index 0000000000..ae9696c229 --- /dev/null +++ b/packages/core-api/src/apis/implementations/auth/oidc/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ProfileInfo, BackstageIdentity } from '../../../definitions'; + +export type OpenIdConnectSession = { + providerInfo: { + idToken: string; + accessToken: string; + scopes: Set; + expiresAt: Date; + }; + profile: ProfileInfo; + backstageIdentity: BackstageIdentity; +}; diff --git a/packages/core/src/api-wrappers/defaultApis.ts b/packages/core/src/api-wrappers/defaultApis.ts index 61d4f3a7e1..adc5fce30a 100644 --- a/packages/core/src/api-wrappers/defaultApis.ts +++ b/packages/core/src/api-wrappers/defaultApis.ts @@ -46,6 +46,8 @@ import { SamlAuth, oneloginAuthApiRef, OneLoginAuth, + oidcApiRef, + OpenIdConnect, } from '@backstage/core-api'; export const defaultApis = [ @@ -153,4 +155,13 @@ export const defaultApis = [ factory: ({ discoveryApi, oauthRequestApi }) => OneLoginAuth.create({ discoveryApi, oauthRequestApi }), }), + createApiFactory({ + api: oidcApiRef, + deps: { + discoveryApi: discoveryApiRef, + oauthRequestApi: oauthRequestApiRef, + }, + factory: ({ discoveryApi, oauthRequestApi }) => + OpenIdConnect.create({ discoveryApi, oauthRequestApi }), + }), ]; diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 8ce955350a..d28251badf 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -25,12 +25,15 @@ "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", + "@types/express-session": "^1.17.2", + "@types/openid-client": "^3.7.0", "compression": "^1.7.4", "cookie-parser": "^1.4.5", "cors": "^2.8.5", "cross-fetch": "^3.0.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", + "express-session": "^1.17.1", "fs-extra": "^9.0.0", "got": "^11.5.2", "helmet": "^4.0.0", @@ -39,6 +42,7 @@ "knex": "^0.21.6", "moment": "^2.26.0", "morgan": "^1.10.0", + "openid-client": "^4.2.1", "passport": "^0.4.1", "passport-github2": "^0.1.12", "passport-gitlab2": "^5.0.0", diff --git a/plugins/auth-backend/src/providers/factories.ts b/plugins/auth-backend/src/providers/factories.ts index a764d0b50e..d4a29c0959 100644 --- a/plugins/auth-backend/src/providers/factories.ts +++ b/plugins/auth-backend/src/providers/factories.ts @@ -18,6 +18,7 @@ import { createGithubProvider } from './github'; import { createGitlabProvider } from './gitlab'; import { createGoogleProvider } from './google'; import { createOAuth2Provider } from './oauth2'; +import { createOidcProvider } from './oidc'; import { createOktaProvider } from './okta'; import { createSamlProvider } from './saml'; import { createAuth0Provider } from './auth0'; @@ -34,6 +35,7 @@ const factories: { [providerId: string]: AuthProviderFactory } = { auth0: createAuth0Provider, microsoft: createMicrosoftProvider, oauth2: createOAuth2Provider, + oidc: createOidcProvider, onelogin: createOneLoginProvider, }; diff --git a/plugins/auth-backend/src/providers/oidc/index.ts b/plugins/auth-backend/src/providers/oidc/index.ts new file mode 100644 index 0000000000..acacbde73d --- /dev/null +++ b/plugins/auth-backend/src/providers/oidc/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createOidcProvider } from './provider'; diff --git a/plugins/auth-backend/src/providers/oidc/provider.ts b/plugins/auth-backend/src/providers/oidc/provider.ts new file mode 100644 index 0000000000..3217c2a537 --- /dev/null +++ b/plugins/auth-backend/src/providers/oidc/provider.ts @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import { + Issuer, + Client, + Strategy as OidcStrategy, + TokenSet, + UserinfoResponse, +} from 'openid-client'; +import { + OAuthAdapter, + OAuthProviderOptions, + OAuthHandlers, + OAuthResponse, + OAuthEnvironmentHandler, + OAuthStartRequest, + encodeState, + OAuthRefreshRequest, +} from '../../lib/oauth'; +import { + executeFetchUserProfileStrategy, + executeFrameHandlerStrategy, + executeRedirectStrategy, + executeRefreshTokenStrategy, + PassportDoneCallback, +} from '../../lib/passport'; +import { RedirectInfo, AuthProviderFactory, ProfileInfo } from '../types'; + +type PrivateInfo = { + refreshToken: string; +}; + +export type OidcAuthProviderOptions = OAuthProviderOptions & { + authorizationUrl: string; + tokenUrl: string; + adUrl: string; + tokenSignedResponseAlg?: string; +}; + +export class OidcAuthProvider implements OAuthHandlers { + private readonly _strategy: Promise>; + + constructor(options: OidcAuthProviderOptions) { + this._strategy = this.setupStrategy(options); + } + + async start(req: OAuthStartRequest): Promise { + const strategy = await this._strategy; + return await executeRedirectStrategy(req, strategy, { + accessType: 'offline', + prompt: 'consent', + scope: `${req.scope} default`, + state: encodeState(req.state), + }); + } + + async handler( + req: express.Request, + ): Promise<{ response: OAuthResponse; refreshToken: string }> { + const strategy = await this._strategy; + const { response, privateInfo } = await executeFrameHandlerStrategy< + OAuthResponse, + PrivateInfo + >(req, strategy); + + return { + response: await this.populateIdentity(response), + refreshToken: privateInfo.refreshToken, + }; + } + + async refresh(req: OAuthRefreshRequest): Promise { + const strategy = await this._strategy; + const refreshTokenResponse = await executeRefreshTokenStrategy( + strategy, + req.refreshToken, + req.scope, + ); + const { + accessToken, + params, + refreshToken: updatedRefreshToken, + } = refreshTokenResponse; + + const profile = await executeFetchUserProfileStrategy( + strategy, + accessToken, + params.id_token, + ); + + return this.populateIdentity({ + providerInfo: { + accessToken, + refreshToken: updatedRefreshToken, + idToken: params.id_token, + expiresInSeconds: params.expires_in, + scope: params.scope, + }, + profile, + }); + } + + private async setupStrategy( + options: OidcAuthProviderOptions, + ): Promise> { + const issuer = await Issuer.discover(options.adUrl); + // console.log('Discovered issuer %s %O', issuer.issuer, issuer.metadata); + const client = new issuer.Client({ + client_id: options.clientId, + client_secret: options.clientSecret, + redirect_uris: [options.callbackUrl], + response_types: ['code'], + id_token_signed_response_alg: options.tokenSignedResponseAlg || 'RS256', + }); + + const strategy = new OidcStrategy( + { + client, + passReqToCallback: false as true, + }, + ( + tokenset: TokenSet, + userinfo: UserinfoResponse, + done: PassportDoneCallback, + ) => { + const profile: ProfileInfo = { + displayName: userinfo.name, + email: userinfo.email, + picture: userinfo.picture, + }; + + done( + undefined, + { + providerInfo: { + idToken: tokenset.id_token || '', + accessToken: tokenset.access_token || '', + scope: tokenset.scope || '', + expiresInSeconds: tokenset.expires_in, + }, + profile, + }, + { + refreshToken: tokenset.refresh_token || '', + }, + ); + }, + ); + strategy.error = console.error; + return strategy; + } + + // Use this function to grab the user profile info from the token + // Then populate the profile with it + private async populateIdentity( + response: OAuthResponse, + ): Promise { + const { profile } = response; + + if (!profile.email) { + throw new Error('Profile does not contain an email'); + } + const id = profile.email.split('@')[0]; + + return { ...response, backstageIdentity: { id } }; + } +} + +export const createOidcProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + const providerId = 'oidc'; + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + const authorizationUrl = envConfig.getString('authorizationUrl'); + const adUrl = envConfig.getString('adUrl'); + const tokenUrl = envConfig.getString('tokenUrl'); + const tokenSignedResponseAlg = envConfig.getString( + 'tokenSignedResponseAlg', + ); + + const provider = new OidcAuthProvider({ + clientId, + clientSecret, + callbackUrl, + authorizationUrl, + tokenUrl, + tokenSignedResponseAlg, + adUrl, + }); + + return OAuthAdapter.fromConfig(globalConfig, provider, { + disableRefresh: false, + providerId, + tokenIssuer, + }); + }); diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 0d19e67fc4..b74f7d8d38 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -27,6 +27,7 @@ import Router from 'express-promise-router'; import { Logger } from 'winston'; import { createOidcRouter, DatabaseKeyStore, TokenFactory } from '../identity'; import { createAuthProvider } from '../providers'; +import session from 'express-session'; export interface RouterOptions { logger: Logger; @@ -59,7 +60,9 @@ export async function createRouter({ }); const catalogApi = new CatalogClient({ discoveryApi: discovery }); - router.use(cookieParser()); + const secret = 'backstage secret'; // TODO: Allow an override here + router.use(cookieParser(secret)); + router.use(session({ secret })); router.use(express.urlencoded({ extended: false })); router.use(express.json()); diff --git a/yarn.lock b/yarn.lock index bd1757cea0..2c4d23e8d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5138,6 +5138,13 @@ "@types/qs" "*" "@types/range-parser" "*" +"@types/express-session@^1.17.2": + version "1.17.2" + resolved "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.2.tgz#ed6a36dd9f267c7fef86004f653bfb9b5cea3c21" + integrity sha512-QRm/fUuvr/BAosL9CvK351SDQP7wpD8+h3S8ZEE/8IvHJ/ZqHrjZbjx/flYfazyPw7yNi9O5fbjFZbh0vZ1ccg== + dependencies: + "@types/express" "*" + "@types/express@*", "@types/express@4.17.7", "@types/express@^4.17.6", "@types/express@^4.17.7": version "4.17.7" resolved "https://registry.npmjs.org/@types/express/-/express-4.17.7.tgz#42045be6475636d9801369cd4418ef65cdb0dd59" @@ -5566,6 +5573,13 @@ dependencies: "@types/node" "*" +"@types/openid-client@^3.7.0": + version "3.7.0" + resolved "https://registry.npmjs.org/@types/openid-client/-/openid-client-3.7.0.tgz#8406e12798d16083df09cc3625973f5a00dd57fa" + integrity sha512-hW+QbAeUyfUHABwUjUECOcv56Mf5tN29xK3GPN7wy3R3XfIQdkm37XELA4wbe2RNG9GYUpN8l2ytfoW05XmpgQ== + dependencies: + openid-client "*" + "@types/ora@^3.2.0": version "3.2.0" resolved "https://registry.npmjs.org/@types/ora/-/ora-3.2.0.tgz#b2f65d1283a8f36d8b0f9ee767e0732a2f429362" @@ -11564,6 +11578,20 @@ express-promise-router@^3.0.3: lodash.flattendeep "^4.0.0" methods "^1.0.0" +express-session@^1.17.1: + version "1.17.1" + resolved "https://registry.npmjs.org/express-session/-/express-session-1.17.1.tgz#36ecbc7034566d38c8509885c044d461c11bf357" + integrity sha512-UbHwgqjxQZJiWRTMyhvWGvjBQduGCSBDhhZXYenziMFjxst5rMV+aJZ6hKPHZnPyHGsrqRICxtX8jtEbm/z36Q== + dependencies: + cookie "0.4.0" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~2.0.0" + on-headers "~1.0.2" + parseurl "~1.3.3" + safe-buffer "5.2.0" + uid-safe "~2.1.5" + express@^4.0.0, express@^4.17.0, express@^4.17.1: version "4.17.1" resolved "https://registry.npmjs.org/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" @@ -12792,7 +12820,7 @@ got@^11.6.2: p-cancelable "^2.0.0" responselike "^2.0.0" -got@^11.7.0: +got@^11.7.0, got@^11.8.0: version "11.8.0" resolved "https://registry.npmjs.org/got/-/got-11.8.0.tgz#be0920c3586b07fd94add3b5b27cb28f49e6545f" integrity sha512-k9noyoIIY9EejuhaBNLyZ31D5328LeqnyPNXJQb2XlJZcKakLqN5m6O/ikhq/0lw56kUYS54fVm+D1x57YC9oQ== @@ -17740,6 +17768,20 @@ opencollective-postinstall@^2.0.2: resolved "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz#5657f1bede69b6e33a45939b061eb53d3c6c3a89" integrity sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw== +openid-client@*, openid-client@^4.2.1: + version "4.2.1" + resolved "https://registry.npmjs.org/openid-client/-/openid-client-4.2.1.tgz#8200c0ab6a3b8e954727dfa790847dc5cb8999c2" + integrity sha512-07eOcJeMH3ZHNvx5DVMZQmy3vZSTQqKSSunbtM1pXb+k5LBPi5hMum1vJCFReXlo4wuLEqZ/OgbsZvXPhbGRtA== + dependencies: + base64url "^3.0.1" + got "^11.8.0" + jose "^2.0.2" + lru-cache "^6.0.0" + make-error "^1.3.6" + object-hash "^2.0.1" + oidc-token-hash "^5.0.0" + p-any "^3.0.0" + openid-client@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/openid-client/-/openid-client-4.1.1.tgz#3e8a25584c4292e9b9b03e60358f5549fb85197a" @@ -19573,6 +19615,11 @@ ramda@^0.26, ramda@~0.26.1: resolved "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz#8d41351eb8111c55353617fc3bbffad8e4d35d06" integrity sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ== +random-bytes@~1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b" + integrity sha1-T2ih3Arli9P7lYSMMDJNt11kNgs= + randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -21086,7 +21133,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0: +safe-buffer@5.2.0, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0: version "5.2.0" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== @@ -23357,6 +23404,13 @@ uid-number@0.0.6: resolved "https://registry.npmjs.org/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" integrity sha1-DqEOgDXo61uOREnwbaHHMGY7qoE= +uid-safe@~2.1.5: + version "2.1.5" + resolved "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz#2b3d5c7240e8fc2e58f8aa269e5ee49c0857bd3a" + integrity sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA== + dependencies: + random-bytes "~1.0.0" + uid2@0.0.3, uid2@0.0.x: version "0.0.3" resolved "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz#483126e11774df2f71b8b639dcd799c376162b82" From 563533dc6c13b35c3e46f3ec6ee5f284e18f4ce5 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Mon, 16 Nov 2020 16:13:01 -0800 Subject: [PATCH 206/259] Moved @types deps to devDependencies --- plugins/auth-backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index d28251badf..f1ad45d9e5 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -25,8 +25,6 @@ "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", - "@types/express-session": "^1.17.2", - "@types/openid-client": "^3.7.0", "compression": "^1.7.4", "cookie-parser": "^1.4.5", "cors": "^2.8.5", @@ -60,7 +58,9 @@ "@backstage/cli": "^0.3.0", "@types/body-parser": "^1.19.0", "@types/cookie-parser": "^1.4.2", + "@types/express-session": "^1.17.2", "@types/jwt-decode": "2.2.1", + "@types/openid-client": "^3.7.0", "@types/passport": "^1.0.3", "@types/passport-github2": "^1.2.4", "@types/passport-google-oauth20": "^2.0.3", From 5d27349000cb702b81258a5a551533798b99daca Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Mon, 16 Nov 2020 16:13:33 -0800 Subject: [PATCH 207/259] Initialized saveUninitialized in express-session --- plugins/auth-backend/src/service/router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index b74f7d8d38..3ae3b6cb95 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -62,7 +62,7 @@ export async function createRouter({ const secret = 'backstage secret'; // TODO: Allow an override here router.use(cookieParser(secret)); - router.use(session({ secret })); + router.use(session({ secret, saveUninitialized: false })); router.use(express.urlencoded({ extended: false })); router.use(express.json()); From b5bb6935223a93bac93158d09a034304d065bca2 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Mon, 16 Nov 2020 17:13:19 -0800 Subject: [PATCH 208/259] Initialized resave in express-session --- plugins/auth-backend/src/service/router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 3ae3b6cb95..e8624d36b3 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -62,7 +62,7 @@ export async function createRouter({ const secret = 'backstage secret'; // TODO: Allow an override here router.use(cookieParser(secret)); - router.use(session({ secret, saveUninitialized: false })); + router.use(session({ secret, saveUninitialized: false, resave: false })); router.use(express.urlencoded({ extended: false })); router.use(express.json()); From 14faf3ad37caa68115f6101f69c6d86b9107bcf6 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Wed, 18 Nov 2020 12:55:11 -0800 Subject: [PATCH 209/259] Changed the oidc prompt parameter value to "none" --- plugins/auth-backend/src/providers/oidc/provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/auth-backend/src/providers/oidc/provider.ts b/plugins/auth-backend/src/providers/oidc/provider.ts index 3217c2a537..ed1488851e 100644 --- a/plugins/auth-backend/src/providers/oidc/provider.ts +++ b/plugins/auth-backend/src/providers/oidc/provider.ts @@ -63,7 +63,7 @@ export class OidcAuthProvider implements OAuthHandlers { const strategy = await this._strategy; return await executeRedirectStrategy(req, strategy, { accessType: 'offline', - prompt: 'consent', + prompt: 'none', scope: `${req.scope} default`, state: encodeState(req.state), }); From 52e79a003238ffd0a7950b89a765787687ffbc53 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Wed, 18 Nov 2020 18:23:59 -0800 Subject: [PATCH 210/259] Addressed PR feedback from @Rugvip --- .../src/apis/implementations/auth/index.ts | 1 - .../implementations/auth/oidc/Oidc.test.ts | 152 --------------- .../apis/implementations/auth/oidc/Oidc.ts | 183 ------------------ .../apis/implementations/auth/oidc/index.ts | 18 -- .../apis/implementations/auth/oidc/types.ts | 28 --- packages/core/src/api-wrappers/defaultApis.ts | 3 +- .../src/providers/oidc/provider.ts | 15 +- 7 files changed, 5 insertions(+), 395 deletions(-) delete mode 100644 packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts delete mode 100644 packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts delete mode 100644 packages/core-api/src/apis/implementations/auth/oidc/index.ts delete mode 100644 packages/core-api/src/apis/implementations/auth/oidc/types.ts diff --git a/packages/core-api/src/apis/implementations/auth/index.ts b/packages/core-api/src/apis/implementations/auth/index.ts index 5adfc6f621..7ef78d19cf 100644 --- a/packages/core-api/src/apis/implementations/auth/index.ts +++ b/packages/core-api/src/apis/implementations/auth/index.ts @@ -23,4 +23,3 @@ export * from './saml'; export * from './auth0'; export * from './microsoft'; export * from './onelogin'; -export * from './oidc'; diff --git a/packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts b/packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts deleted file mode 100644 index 4286fe7e5d..0000000000 --- a/packages/core-api/src/apis/implementations/auth/oidc/Oidc.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import OpenIdConnect from './Oidc'; - -const theFuture = new Date(Date.now() + 3600000); -const thePast = new Date(Date.now() - 10); - -const PREFIX = 'https://www.googleapis.com/auth/'; - -const scopeTransform = (x: string[]) => x; - -describe('OpenIdConnect', () => { - it('should get refreshed access token', async () => { - const getSession = jest.fn().mockResolvedValue({ - providerInfo: { accessToken: 'access-token', expiresAt: theFuture }, - }); - const openIdConnect = new OpenIdConnect({ - sessionManager: { getSession } as any, - scopeTransform, - }); - - expect(await openIdConnect.getAccessToken('my-scope my-scope2')).toBe( - 'access-token', - ); - expect(getSession).toBeCalledTimes(1); - expect(getSession.mock.calls[0][0].scopes).toEqual( - new Set(['my-scope', 'my-scope2']), - ); - }); - - it('should transform scopes', async () => { - const getSession = jest.fn().mockResolvedValue({ - providerInfo: { accessToken: 'access-token', expiresAt: theFuture }, - }); - const openIdConnect = new OpenIdConnect({ - sessionManager: { getSession } as any, - scopeTransform: scopes => scopes.map(scope => `my-prefix/${scope}`), - }); - - expect(await openIdConnect.getAccessToken('my-scope')).toBe('access-token'); - expect(getSession).toBeCalledTimes(1); - expect(getSession.mock.calls[0][0].scopes).toEqual( - new Set(['my-prefix/my-scope']), - ); - }); - - it('should get refreshed id token', async () => { - const getSession = jest.fn().mockResolvedValue({ - providerInfo: { idToken: 'id-token', expiresAt: theFuture }, - }); - const openIdConnect = new OpenIdConnect({ - sessionManager: { getSession } as any, - scopeTransform, - }); - - expect(await openIdConnect.getIdToken()).toBe('id-token'); - expect(getSession).toBeCalledTimes(1); - }); - - it('should get optional id token', async () => { - const getSession = jest.fn().mockResolvedValue({ - providerInfo: { idToken: 'id-token', expiresAt: theFuture }, - }); - const openIdConnect = new OpenIdConnect({ - sessionManager: { getSession } as any, - scopeTransform, - }); - - expect(await openIdConnect.getIdToken({ optional: true })).toBe('id-token'); - expect(getSession).toBeCalledTimes(1); - }); - - it('should share popup closed errors', async () => { - const error = new Error('NOPE'); - error.name = 'RejectedError'; - const getSession = jest - .fn() - .mockResolvedValueOnce({ - providerInfo: { - accessToken: 'access-token', - expiresAt: theFuture, - scopes: new Set([`${PREFIX}not-enough`]), - }, - }) - .mockRejectedValue(error); - const openIdConnect = new OpenIdConnect({ - sessionManager: { getSession } as any, - scopeTransform, - }); - - // Make sure we have a session before we do the double request, so that we get past the !this.currentSession check - await expect(openIdConnect.getAccessToken()).resolves.toBe('access-token'); - - const promise1 = openIdConnect.getAccessToken('more'); - const promise2 = openIdConnect.getAccessToken('more'); - await expect(promise1).rejects.toBe(error); - await expect(promise2).rejects.toBe(error); - expect(getSession).toBeCalledTimes(3); - }); - - it('should wait for all session refreshes', async () => { - const initialSession = { - providerInfo: { - idToken: 'token1', - expiresAt: theFuture, - scopes: new Set(), - }, - }; - const getSession = jest - .fn() - .mockResolvedValueOnce(initialSession) - .mockResolvedValue({ - providerInfo: { - idToken: 'token2', - expiresAt: theFuture, - scopes: new Set(), - }, - }); - const openIdConnect = new OpenIdConnect({ - sessionManager: { getSession } as any, - scopeTransform, - }); - - // Grab the expired session first - await expect(openIdConnect.getIdToken()).resolves.toBe('token1'); - expect(getSession).toBeCalledTimes(1); - - initialSession.providerInfo.expiresAt = thePast; - - const promise1 = openIdConnect.getIdToken(); - const promise2 = openIdConnect.getIdToken(); - const promise3 = openIdConnect.getIdToken(); - await expect(promise1).resolves.toBe('token2'); - await expect(promise2).resolves.toBe('token2'); - await expect(promise3).resolves.toBe('token2'); - expect(getSession).toBeCalledTimes(4); // De-duping of session requests happens in client - }); -}); diff --git a/packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts b/packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts deleted file mode 100644 index deb8331432..0000000000 --- a/packages/core-api/src/apis/implementations/auth/oidc/Oidc.ts +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import OpenIdConnectIcon from '@material-ui/icons/AcUnit'; -import { DefaultAuthConnector } from '../../../../lib/AuthConnector'; -import { RefreshingAuthSessionManager } from '../../../../lib/AuthSessionManager'; -import { SessionManager } from '../../../../lib/AuthSessionManager/types'; -import { Observable } from '../../../../types'; -import { - AuthRequestOptions, - BackstageIdentity, - OAuthApi, - OpenIdConnectApi, - ProfileInfo, - ProfileInfoApi, - SessionState, - SessionApi, - BackstageIdentityApi, -} from '../../../definitions/auth'; -import { OpenIdConnectSession } from './types'; -import { OAuthApiCreateOptions } from '../types'; - -type Options = { - sessionManager: SessionManager; - scopeTransform: (scopes: string[]) => string[]; -}; - -type CreateOptions = OAuthApiCreateOptions & { - scopeTransform?: (scopes: string[]) => string[]; -}; - -export type OpenIdConnectResponse = { - providerInfo: { - accessToken: string; - idToken: string; - scope: string; - expiresInSeconds: number; - }; - profile: ProfileInfo; - backstageIdentity: BackstageIdentity; -}; - -const DEFAULT_PROVIDER = { - id: 'oidc', - title: 'Open ID Connect Identity Provider', - icon: OpenIdConnectIcon, -}; - -class OpenIdConnect - implements - OAuthApi, - OpenIdConnectApi, - ProfileInfoApi, - BackstageIdentityApi, - SessionApi { - static create({ - discoveryApi, - environment = 'development', - provider = DEFAULT_PROVIDER, - oauthRequestApi, - defaultScopes = [], - scopeTransform = x => x, - }: CreateOptions) { - const connector = new DefaultAuthConnector({ - discoveryApi, - environment, - provider, - oauthRequestApi: oauthRequestApi, - sessionTransform(res: OpenIdConnectResponse): OpenIdConnectSession { - return { - ...res, - providerInfo: { - idToken: res.providerInfo.idToken, - accessToken: res.providerInfo.accessToken, - scopes: OpenIdConnect.normalizeScopes( - scopeTransform, - res.providerInfo.scope, - ), - expiresAt: new Date( - Date.now() + res.providerInfo.expiresInSeconds * 1000, - ), - }, - }; - }, - }); - - const sessionManager = new RefreshingAuthSessionManager({ - connector, - defaultScopes: new Set(defaultScopes), - sessionScopes: (session: OpenIdConnectSession) => - session.providerInfo.scopes, - sessionShouldRefresh: (session: OpenIdConnectSession) => { - const expiresInSec = - (session.providerInfo.expiresAt.getTime() - Date.now()) / 1000; - return expiresInSec < 60 * 5; - }, - }); - - return new OpenIdConnect({ sessionManager, scopeTransform }); - } - - private readonly sessionManager: SessionManager; - private readonly scopeTransform: (scopes: string[]) => string[]; - - constructor(options: Options) { - this.sessionManager = options.sessionManager; - this.scopeTransform = options.scopeTransform; - } - - async signIn() { - await this.getAccessToken(); - } - - async signOut() { - await this.sessionManager.removeSession(); - } - - sessionState$(): Observable { - return this.sessionManager.sessionState$(); - } - - async getAccessToken( - scope?: string | string[], - options?: AuthRequestOptions, - ) { - const normalizedScopes = OpenIdConnect.normalizeScopes( - this.scopeTransform, - scope, - ); - const session = await this.sessionManager.getSession({ - ...options, - scopes: normalizedScopes, - }); - return session?.providerInfo.accessToken ?? ''; - } - - async getIdToken(options: AuthRequestOptions = {}) { - const session = await this.sessionManager.getSession(options); - return session?.providerInfo.idToken ?? ''; - } - - async getBackstageIdentity( - options: AuthRequestOptions = {}, - ): Promise { - const session = await this.sessionManager.getSession(options); - return session?.backstageIdentity; - } - - async getProfile(options: AuthRequestOptions = {}) { - const session = await this.sessionManager.getSession(options); - return session?.profile; - } - - private static normalizeScopes( - scopeTransform: (scopes: string[]) => string[], - scopes?: string | string[], - ): Set { - if (!scopes) { - return new Set(); - } - - const scopeList = Array.isArray(scopes) - ? scopes - : scopes.split(/[\s|,]/).filter(Boolean); - - return new Set(scopeTransform(scopeList)); - } -} - -export default OpenIdConnect; diff --git a/packages/core-api/src/apis/implementations/auth/oidc/index.ts b/packages/core-api/src/apis/implementations/auth/oidc/index.ts deleted file mode 100644 index cf9add0757..0000000000 --- a/packages/core-api/src/apis/implementations/auth/oidc/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { default as OpenIdConnect } from './Oidc'; -export * from './types'; diff --git a/packages/core-api/src/apis/implementations/auth/oidc/types.ts b/packages/core-api/src/apis/implementations/auth/oidc/types.ts deleted file mode 100644 index ae9696c229..0000000000 --- a/packages/core-api/src/apis/implementations/auth/oidc/types.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ProfileInfo, BackstageIdentity } from '../../../definitions'; - -export type OpenIdConnectSession = { - providerInfo: { - idToken: string; - accessToken: string; - scopes: Set; - expiresAt: Date; - }; - profile: ProfileInfo; - backstageIdentity: BackstageIdentity; -}; diff --git a/packages/core/src/api-wrappers/defaultApis.ts b/packages/core/src/api-wrappers/defaultApis.ts index adc5fce30a..9b3d26fb4d 100644 --- a/packages/core/src/api-wrappers/defaultApis.ts +++ b/packages/core/src/api-wrappers/defaultApis.ts @@ -47,7 +47,6 @@ import { oneloginAuthApiRef, OneLoginAuth, oidcApiRef, - OpenIdConnect, } from '@backstage/core-api'; export const defaultApis = [ @@ -162,6 +161,6 @@ export const defaultApis = [ oauthRequestApi: oauthRequestApiRef, }, factory: ({ discoveryApi, oauthRequestApi }) => - OpenIdConnect.create({ discoveryApi, oauthRequestApi }), + OAuth2.create({ discoveryApi, oauthRequestApi }), }), ]; diff --git a/plugins/auth-backend/src/providers/oidc/provider.ts b/plugins/auth-backend/src/providers/oidc/provider.ts index ed1488851e..71ea2c4103 100644 --- a/plugins/auth-backend/src/providers/oidc/provider.ts +++ b/plugins/auth-backend/src/providers/oidc/provider.ts @@ -46,9 +46,7 @@ type PrivateInfo = { }; export type OidcAuthProviderOptions = OAuthProviderOptions & { - authorizationUrl: string; - tokenUrl: string; - adUrl: string; + metadataUrl: string; tokenSignedResponseAlg?: string; }; @@ -118,8 +116,7 @@ export class OidcAuthProvider implements OAuthHandlers { private async setupStrategy( options: OidcAuthProviderOptions, ): Promise> { - const issuer = await Issuer.discover(options.adUrl); - // console.log('Discovered issuer %s %O', issuer.issuer, issuer.metadata); + const issuer = await Issuer.discover(options.metadataUrl); const client = new issuer.Client({ client_id: options.clientId, client_secret: options.clientSecret, @@ -191,9 +188,7 @@ export const createOidcProvider: AuthProviderFactory = ({ const clientId = envConfig.getString('clientId'); const clientSecret = envConfig.getString('clientSecret'); const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; - const authorizationUrl = envConfig.getString('authorizationUrl'); - const adUrl = envConfig.getString('adUrl'); - const tokenUrl = envConfig.getString('tokenUrl'); + const metadataUrl = envConfig.getString('metadataUrl'); const tokenSignedResponseAlg = envConfig.getString( 'tokenSignedResponseAlg', ); @@ -202,10 +197,8 @@ export const createOidcProvider: AuthProviderFactory = ({ clientId, clientSecret, callbackUrl, - authorizationUrl, - tokenUrl, tokenSignedResponseAlg, - adUrl, + metadataUrl, }); return OAuthAdapter.fromConfig(globalConfig, provider, { From 5c485ff60ff2c381e7083449c57cea58586b4acc Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Thu, 19 Nov 2020 09:11:13 -0800 Subject: [PATCH 211/259] Added a auth.session.secret config value for setting the express session secret --- app-config.yaml | 2 ++ plugins/auth-backend/package.json | 23 +++++++++++++++++++++- plugins/auth-backend/src/service/router.ts | 4 +++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index cfa47618e8..34a0b09520 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -191,6 +191,8 @@ scaffolder: $env: AZURE_TOKEN auth: + session: + secret: yaml session secret providers: google: development: diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index f1ad45d9e5..94502775e4 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -71,5 +71,26 @@ "files": [ "dist", "migrations" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/auth-backend", + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "session": { + "type": "object", + "properties": { + "secret": { + "type": "string", + "visibility": "secret" + } + } + } + } + } + } + } } diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index e8624d36b3..1baa1d4892 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -60,7 +60,9 @@ export async function createRouter({ }); const catalogApi = new CatalogClient({ discoveryApi: discovery }); - const secret = 'backstage secret'; // TODO: Allow an override here + const secret = + config.getOptionalString('auth.session.secret') ?? 'backstage secret'; + console.log('using secret', secret); router.use(cookieParser(secret)); router.use(session({ secret, saveUninitialized: false, resave: false })); router.use(express.urlencoded({ extended: false })); From 78b390483df0dbc4f4369d4887a157c4df2692d5 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Thu, 19 Nov 2020 11:13:04 -0800 Subject: [PATCH 212/259] Added a oidc provider test --- plugins/auth-backend/package.json | 4 +- .../src/providers/oidc/provider.test.ts | 60 +++++++++++++++++++ .../src/providers/oidc/provider.ts | 2 +- yarn.lock | 22 +++++++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 plugins/auth-backend/src/providers/oidc/provider.test.ts diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 94502775e4..4d1e52aa21 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -60,13 +60,15 @@ "@types/cookie-parser": "^1.4.2", "@types/express-session": "^1.17.2", "@types/jwt-decode": "2.2.1", + "@types/nock": "^11.1.0", "@types/openid-client": "^3.7.0", "@types/passport": "^1.0.3", "@types/passport-github2": "^1.2.4", "@types/passport-google-oauth20": "^2.0.3", "@types/passport-microsoft": "^0.0.0", "@types/passport-saml": "^1.1.2", - "msw": "^0.21.2" + "msw": "^0.21.2", + "nock": "^13.0.5" }, "files": [ "dist", diff --git a/plugins/auth-backend/src/providers/oidc/provider.test.ts b/plugins/auth-backend/src/providers/oidc/provider.test.ts new file mode 100644 index 0000000000..e8b414edf2 --- /dev/null +++ b/plugins/auth-backend/src/providers/oidc/provider.test.ts @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import nock from 'nock'; +import { ClientMetadata, IssuerMetadata } from 'openid-client'; +import { OidcAuthProvider } from './provider'; + +const issuerMetadata = { + issuer: 'https://oidc.test', + authorization_endpoint: 'https://oidc.test/as/authorization.oauth2', + token_endpoint: 'https://oidc.test/as/token.oauth2', + revocation_endpoint: 'https://oidc.test/as/revoke_token.oauth2', + userinfo_endpoint: 'https://oidc.test/idp/userinfo.openid', + introspection_endpoint: 'https://oidc.test/as/introspect.oauth2', + jwks_uri: 'https://oidc.test/pf/JWKS', + scopes_supported: ['openid'], + claims_supported: ['email'], + response_types_supported: ['code'], + id_token_signing_alg_values_supported: ['RS256', 'RS512'], + token_endpoint_auth_signing_alg_values_supported: ['RS256', 'RS512'], + request_object_signing_alg_values_supported: ['RS256', 'RS512'], +}; + +const clientMetadata = { + callbackUrl: 'https://oidc.test/callback', + clientId: 'testclientid', + clientSecret: 'testclientsecret', + metadataUrl: 'https://oidc.test/.well-known/openid-configuration', +}; + +describe('OidcAuthProvider', () => { + it('hit the metadataurl', async () => { + const scope = nock('https://oidc.test') + .get('/.well-known/openid-configuration') + .reply(200, issuerMetadata); + const provider = new OidcAuthProvider(clientMetadata); + const strategy = ((await provider._strategy) as any) as { + _client: ClientMetadata; + _issuer: IssuerMetadata; + }; + // Assert that the expected request to the metadaurl was made. + expect(scope.isDone()).toBeTruthy(); + const { _client, _issuer } = strategy; + expect(_client.client_id).toBe(clientMetadata.clientId); + expect(_issuer.token_endpoint).toBe(issuerMetadata.token_endpoint); + }); +}); diff --git a/plugins/auth-backend/src/providers/oidc/provider.ts b/plugins/auth-backend/src/providers/oidc/provider.ts index 71ea2c4103..5ebb859f2f 100644 --- a/plugins/auth-backend/src/providers/oidc/provider.ts +++ b/plugins/auth-backend/src/providers/oidc/provider.ts @@ -51,7 +51,7 @@ export type OidcAuthProviderOptions = OAuthProviderOptions & { }; export class OidcAuthProvider implements OAuthHandlers { - private readonly _strategy: Promise>; + readonly _strategy: Promise>; constructor(options: OidcAuthProviderOptions) { this._strategy = this.setupStrategy(options); diff --git a/yarn.lock b/yarn.lock index 2c4d23e8d7..ba25713fdc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5521,6 +5521,13 @@ dependencies: "@types/node" "*" +"@types/nock@^11.1.0": + version "11.1.0" + resolved "https://registry.npmjs.org/@types/nock/-/nock-11.1.0.tgz#0a8c1056a31ba32a959843abccf99626dd90a538" + integrity sha512-jI/ewavBQ7X5178262JQR0ewicPAcJhXS/iFaNJl0VHLfyosZ/kwSrsa6VNQNSO8i9d8SqdRgOtZSOKJ/+iNMw== + dependencies: + nock "*" + "@types/node-fetch@2.5.7", "@types/node-fetch@^2.5.4": version "2.5.7" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" @@ -17139,6 +17146,16 @@ no-case@^3.0.3: lower-case "^2.0.1" tslib "^1.10.0" +nock@*, nock@^13.0.5: + version "13.0.5" + resolved "https://registry.npmjs.org/nock/-/nock-13.0.5.tgz#a618c6f86372cb79fac04ca9a2d1e4baccdb2414" + integrity sha512-1ILZl0zfFm2G4TIeJFW0iHknxr2NyA+aGCMTjDVUsBY4CkMRispF1pfIYkTRdAR/3Bg+UzdEuK0B6HczMQZcCg== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + lodash.set "^4.3.2" + propagate "^2.0.0" + node-addon-api@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.0.tgz#f9afb8d777a91525244b01775ea0ddbe1125483b" @@ -19385,6 +19402,11 @@ prop-types@^15.5.10, prop-types@^15.5.8, prop-types@^15.6.0, prop-types@^15.6.1, object-assign "^4.1.1" react-is "^16.8.1" +propagate@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== + property-expr@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/property-expr/-/property-expr-2.0.2.tgz#fff2a43919135553a3bc2fdd94bdb841965b2330" From 3712c47974e67a6a4d4d3e29569ecf470d27a355 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Thu, 19 Nov 2020 17:35:03 -0800 Subject: [PATCH 213/259] Added a test for the OidcAuthProvider handler method --- .../src/providers/oidc/provider.test.ts | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/plugins/auth-backend/src/providers/oidc/provider.test.ts b/plugins/auth-backend/src/providers/oidc/provider.test.ts index e8b414edf2..0ad464f607 100644 --- a/plugins/auth-backend/src/providers/oidc/provider.test.ts +++ b/plugins/auth-backend/src/providers/oidc/provider.test.ts @@ -14,9 +14,12 @@ * limitations under the License. */ +import express from 'express'; +import { Session } from 'express-session'; import nock from 'nock'; import { ClientMetadata, IssuerMetadata } from 'openid-client'; import { OidcAuthProvider } from './provider'; +import { JWT, JWK } from 'jose'; const issuerMetadata = { issuer: 'https://oidc.test', @@ -29,9 +32,9 @@ const issuerMetadata = { scopes_supported: ['openid'], claims_supported: ['email'], response_types_supported: ['code'], - id_token_signing_alg_values_supported: ['RS256', 'RS512'], - token_endpoint_auth_signing_alg_values_supported: ['RS256', 'RS512'], - request_object_signing_alg_values_supported: ['RS256', 'RS512'], + id_token_signing_alg_values_supported: ['RS256', 'RS512', 'HS256'], + token_endpoint_auth_signing_alg_values_supported: ['RS256', 'RS512', 'HS256'], + request_object_signing_alg_values_supported: ['RS256', 'RS512', 'HS256'], }; const clientMetadata = { @@ -39,10 +42,11 @@ const clientMetadata = { clientId: 'testclientid', clientSecret: 'testclientsecret', metadataUrl: 'https://oidc.test/.well-known/openid-configuration', + tokenSignedResponseAlg: 'none', }; describe('OidcAuthProvider', () => { - it('hit the metadataurl', async () => { + it('hit the metadata url', async () => { const scope = nock('https://oidc.test') .get('/.well-known/openid-configuration') .reply(200, issuerMetadata); @@ -57,4 +61,35 @@ describe('OidcAuthProvider', () => { expect(_client.client_id).toBe(clientMetadata.clientId); expect(_issuer.token_endpoint).toBe(issuerMetadata.token_endpoint); }); + + it('OidcAuthProvider#handler successfully invokes the oidc endpoints', async () => { + const jwt = { + sub: 'alice', + iss: 'https://oidc.test', + aud: clientMetadata.clientId, + exp: Date.now() + 10000, + }; + const scope = nock('https://oidc.test') + .get('/.well-known/openid-configuration') + .reply(200, issuerMetadata) + .post('/as/token.oauth2') + .reply(200, { + id_token: JWT.sign(jwt, JWK.None), + access_token: 'test', + authorization_signed_response_alg: 'HS256', + }) + .get('/idp/userinfo.openid') + .reply(200, { + sub: 'alice', + email: 'alice@oidc.test', + }); + const provider = new OidcAuthProvider(clientMetadata); + const req = { + method: 'GET', + url: '/?code=test2', + session: ({ 'oidc:oidc.test': 'test' } as any) as Session, + } as express.Request; + await provider.handler(req); + expect(scope.isDone()).toBeTruthy(); + }); }); From afaaf96eccccd91ea914f3e52a9ea14e1e3ae6e6 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Thu, 19 Nov 2020 23:49:12 -0800 Subject: [PATCH 214/259] Cleanup and fixes after refactoring --- app-config.yaml | 4 ++-- packages/app/src/identityProviders.ts | 4 ++-- packages/core-api/src/apis/definitions/auth.ts | 2 +- packages/core/src/api-wrappers/defaultApis.ts | 16 +++++++++++++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 34a0b09520..8dc870a7cd 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -239,8 +239,8 @@ auth: $env: AUTH_OAUTH2_TOKEN_URL oidc: development: - adUrl: - $env: AUTH_OIDC_AD_URL + metadataUrl: + $env: AUTH_OIDC_METADATA_URL clientId: $env: AUTH_OIDC_CLIENT_ID clientSecret: diff --git a/packages/app/src/identityProviders.ts b/packages/app/src/identityProviders.ts index ba657328f2..01828b1405 100644 --- a/packages/app/src/identityProviders.ts +++ b/packages/app/src/identityProviders.ts @@ -22,7 +22,7 @@ import { samlAuthApiRef, microsoftAuthApiRef, oneloginAuthApiRef, - oidcApiRef, + oidcAuthApiRef, } from '@backstage/core'; export const providers = [ @@ -30,7 +30,7 @@ export const providers = [ id: 'oidc-auth-provider', title: 'Oidc', message: 'Sign In using OpenId Connect', - apiRef: oidcApiRef, + apiRef: oidcAuthApiRef, }, { id: 'google-auth-provider', diff --git a/packages/core-api/src/apis/definitions/auth.ts b/packages/core-api/src/apis/definitions/auth.ts index 5dae5532ad..30b07887ad 100644 --- a/packages/core-api/src/apis/definitions/auth.ts +++ b/packages/core-api/src/apis/definitions/auth.ts @@ -314,7 +314,7 @@ export const oauth2ApiRef: ApiRef< /** * Provides authentication for custom OpenID Connect identity providers. */ -export const oidcApiRef: ApiRef< +export const oidcAuthApiRef: ApiRef< OAuthApi & OpenIdConnectApi & ProfileInfoApi & diff --git a/packages/core/src/api-wrappers/defaultApis.ts b/packages/core/src/api-wrappers/defaultApis.ts index 9b3d26fb4d..1f18f54d2a 100644 --- a/packages/core/src/api-wrappers/defaultApis.ts +++ b/packages/core/src/api-wrappers/defaultApis.ts @@ -46,9 +46,11 @@ import { SamlAuth, oneloginAuthApiRef, OneLoginAuth, - oidcApiRef, + oidcAuthApiRef, } from '@backstage/core-api'; +import OAuth2Icon from '@material-ui/icons/AcUnit'; + export const defaultApis = [ createApiFactory({ api: discoveryApiRef, @@ -155,12 +157,20 @@ export const defaultApis = [ OneLoginAuth.create({ discoveryApi, oauthRequestApi }), }), createApiFactory({ - api: oidcApiRef, + api: oidcAuthApiRef, deps: { discoveryApi: discoveryApiRef, oauthRequestApi: oauthRequestApiRef, }, factory: ({ discoveryApi, oauthRequestApi }) => - OAuth2.create({ discoveryApi, oauthRequestApi }), + OAuth2.create({ + discoveryApi, + oauthRequestApi, + provider: { + id: 'oidc', + title: 'Your Identity Provider', + icon: OAuth2Icon, + }, + }), }), ]; From 68be35912b8e0c52c701e5a9e172a3b1cd9afb6f Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Fri, 20 Nov 2020 10:07:59 -0800 Subject: [PATCH 215/259] Conditionally enable the auth-backend session support --- app-config.yaml | 6 +++--- plugins/auth-backend/src/service/router.ts | 16 +++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 8dc870a7cd..fcc54a1a35 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -189,10 +189,10 @@ scaffolder: api: token: $env: AZURE_TOKEN - auth: - session: - secret: yaml session secret + ### Providing an auth.session.secret will enable session support in the auth-backend + # session: + # secret: custom session secret providers: google: development: diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts index 1baa1d4892..47e0d14caa 100644 --- a/plugins/auth-backend/src/service/router.ts +++ b/plugins/auth-backend/src/service/router.ts @@ -28,6 +28,7 @@ import { Logger } from 'winston'; import { createOidcRouter, DatabaseKeyStore, TokenFactory } from '../identity'; import { createAuthProvider } from '../providers'; import session from 'express-session'; +import passport from 'passport'; export interface RouterOptions { logger: Logger; @@ -60,11 +61,16 @@ export async function createRouter({ }); const catalogApi = new CatalogClient({ discoveryApi: discovery }); - const secret = - config.getOptionalString('auth.session.secret') ?? 'backstage secret'; - console.log('using secret', secret); - router.use(cookieParser(secret)); - router.use(session({ secret, saveUninitialized: false, resave: false })); + const secret = config.getOptionalString('auth.session.secret'); + if (secret) { + router.use(cookieParser(secret)); + // TODO: Configure the server-side session storage. The default MemoryStore is not designed for production + router.use(session({ secret, saveUninitialized: false, resave: false })); + router.use(passport.initialize()); + router.use(passport.session()); + } else { + router.use(cookieParser()); + } router.use(express.urlencoded({ extended: false })); router.use(express.json()); From 40b3036fac715bb7ee81b673ec397b7951876a35 Mon Sep 17 00:00:00 2001 From: Brian Leathem Date: Fri, 20 Nov 2020 11:57:00 -0800 Subject: [PATCH 216/259] Added a test for the createOidcProvider method --- .../src/providers/oidc/provider.test.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/plugins/auth-backend/src/providers/oidc/provider.test.ts b/plugins/auth-backend/src/providers/oidc/provider.test.ts index 0ad464f607..ade69255b7 100644 --- a/plugins/auth-backend/src/providers/oidc/provider.test.ts +++ b/plugins/auth-backend/src/providers/oidc/provider.test.ts @@ -18,8 +18,11 @@ import express from 'express'; import { Session } from 'express-session'; import nock from 'nock'; import { ClientMetadata, IssuerMetadata } from 'openid-client'; -import { OidcAuthProvider } from './provider'; +import { createOidcProvider, OidcAuthProvider } from './provider'; import { JWT, JWK } from 'jose'; +import { AuthProviderFactoryOptions } from '../types'; +import { Config } from '@backstage/config'; +import { OAuthAdapter } from '../../lib/oauth'; const issuerMetadata = { issuer: 'https://oidc.test', @@ -92,4 +95,21 @@ describe('OidcAuthProvider', () => { await provider.handler(req); expect(scope.isDone()).toBeTruthy(); }); + + const options = { + globalConfig: { + appUrl: 'https://oidc.test', + baseUrl: 'https://oidc.test', + }, + config: ({ + keys: jest.fn(() => ['test']), + getConfig: jest.fn(() => ({ getString: () => '' })), + } as any) as Config, + } as AuthProviderFactoryOptions; + + it('createOidcProvider', () => { + const provider = createOidcProvider(options) as OAuthAdapter; + console.log(provider); + expect(provider.start).toBeDefined(); + }); }); From c220f7e35a9159b6540150a6a0f1710275c74aff Mon Sep 17 00:00:00 2001 From: blam Date: Sun, 22 Nov 2020 02:50:35 +0100 Subject: [PATCH 217/259] chore(scaffolder/template): fixing catalog-info.yaml reference for templates with new version --- .../{component-info.yaml => catalog-info.yaml} | 0 .../{component-info.yaml => catalog-info.yaml} | 0 .../{component-info.yaml => catalog-info.yaml} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename plugins/scaffolder-backend/sample-templates/docs-template/{{cookiecutter.component_id}}/{component-info.yaml => catalog-info.yaml} (100%) rename plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/{component-info.yaml => catalog-info.yaml} (100%) rename plugins/scaffolder-backend/sample-templates/springboot-grpc-template/{{cookiecutter.component_id}}/{component-info.yaml => catalog-info.yaml} (100%) diff --git a/plugins/scaffolder-backend/sample-templates/docs-template/{{cookiecutter.component_id}}/component-info.yaml b/plugins/scaffolder-backend/sample-templates/docs-template/{{cookiecutter.component_id}}/catalog-info.yaml similarity index 100% rename from plugins/scaffolder-backend/sample-templates/docs-template/{{cookiecutter.component_id}}/component-info.yaml rename to plugins/scaffolder-backend/sample-templates/docs-template/{{cookiecutter.component_id}}/catalog-info.yaml diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/component-info.yaml b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/catalog-info.yaml similarity index 100% rename from plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/component-info.yaml rename to plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/catalog-info.yaml diff --git a/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/{{cookiecutter.component_id}}/component-info.yaml b/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/{{cookiecutter.component_id}}/catalog-info.yaml similarity index 100% rename from plugins/scaffolder-backend/sample-templates/springboot-grpc-template/{{cookiecutter.component_id}}/component-info.yaml rename to plugins/scaffolder-backend/sample-templates/springboot-grpc-template/{{cookiecutter.component_id}}/catalog-info.yaml From 9075c4816b73716988bdad7daa2fd7a1eb94bd50 Mon Sep 17 00:00:00 2001 From: blam Date: Sun, 22 Nov 2020 02:51:10 +0100 Subject: [PATCH 218/259] docs: added a note about templates should have catalog-info --- docs/features/software-templates/adding-templates.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 75a75003f7..e2b69d6767 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -55,6 +55,10 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. +_NOTE_: When the `publish` step is completed, it is currently assumed by the +scaffolder that the final repository should contain a `catalog-info.yaml` in +order to register this with the Catalog in Backstage. + Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. From ab4794c70272da8deb01df99367a234242cdf3eb Mon Sep 17 00:00:00 2001 From: blam Date: Sun, 22 Nov 2020 02:51:40 +0100 Subject: [PATCH 219/259] chore(scaffolder/cra): needs to create catalog-info yaml --- .../scaffolder-backend/src/scaffolder/stages/templater/cra.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts index da33149c24..045feabc62 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts @@ -69,7 +69,7 @@ export class CreateReactAppTemplater implements TemplaterBase { ); await fs.promises.writeFile( - `${finalDir}/component-info.yaml`, + `${finalDir}/catalog-info.yaml`, yaml.stringify(componentInfo), ); From f12a7cd906fe000263d82d4b37e8803ae9c49bac Mon Sep 17 00:00:00 2001 From: blam Date: Sun, 22 Nov 2020 02:52:00 +0100 Subject: [PATCH 220/259] chore(scaffolder): a little housekeeping and better error handling for the scaffolder frontend. --- .../JobStatusModal/JobStatusModal.tsx | 61 ++++++++-------- .../components/TemplatePage/TemplatePage.tsx | 73 ++++++++++--------- .../useJobPolling.ts | 51 +++++-------- plugins/scaffolder/src/types.ts | 6 +- 4 files changed, 94 insertions(+), 97 deletions(-) rename plugins/scaffolder/src/components/{JobStatusModal => hooks}/useJobPolling.ts (55%) diff --git a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx index d8b194eb41..f0aa5ccd81 100644 --- a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx +++ b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx @@ -13,43 +13,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { Button } from '@backstage/core'; -import { entityRoute, entityRouteParams } from '@backstage/plugin-catalog'; import { + Button as Action, Dialog, DialogActions, DialogContent, DialogTitle, LinearProgress, } from '@material-ui/core'; -import React, { useEffect, useState } from 'react'; -import { generatePath } from 'react-router-dom'; + +import React, { useCallback, useState } from 'react'; import { Job } from '../../types'; import { JobStage } from '../JobStage/JobStage'; -import { useJobPolling } from './useJobPolling'; type Props = { - onComplete: (job: Job) => void; - jobId: string; - entity: TemplateEntityV1alpha1 | null; + job: Job; + toCatalogLink?: string; }; -export const JobStatusModal = ({ jobId, onComplete, entity }: Props) => { - const job = useJobPolling(jobId); - const [dialogTitle, setDialogTitle] = useState('Creating component...'); +export const JobStatusModal = ({ job, toCatalogLink }: Props) => { + const [isOpen, setOpen] = useState(true); - useEffect(() => { - if (job?.status === 'COMPLETED') { - setDialogTitle('Successfully created component'); - onComplete(job); - } else if (job?.status === 'FAILED') - setDialogTitle('Failed to create component'); - }, [job, onComplete, setDialogTitle]); + const renderTitle = () => { + switch (job?.status) { + case 'COMPLETED': + return 'Successfully created component'; + case 'FAILED': + return 'Failed to create component'; + default: + return 'Create component'; + } + }; + + const onClose = useCallback(() => { + if (job.status === 'COMPLETED' || job.status === 'FAILED') { + setOpen(false); + } + }, [job]); return ( - - {dialogTitle} + + {renderTitle()} {!job ? ( @@ -66,16 +71,14 @@ export const JobStatusModal = ({ jobId, onComplete, entity }: Props) => { )) )} - {entity && ( + {job?.status && toCatalogLink && ( - + + + )} + {job?.status === 'FAILED' && ( + + setOpen(false)}>Close )} diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index d8b803f893..0754b64d7f 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -23,18 +23,22 @@ import { Page, useApi, } from '@backstage/core'; -import { catalogApiRef } from '@backstage/plugin-catalog'; +import { + catalogApiRef, + entityRoute, + entityRouteParams, +} from '@backstage/plugin-catalog'; import { LinearProgress } from '@material-ui/core'; import { IChangeEvent } from '@rjsf/core'; import React, { useState, useCallback } from 'react'; -import { Navigate } from 'react-router'; +import { generatePath, Navigate } from 'react-router'; import { useParams } from 'react-router-dom'; import { useAsync } from 'react-use'; import { scaffolderApiRef } from '../../api'; import { rootRoute } from '../../routes'; -import { Job } from '../../types'; import { JobStatusModal } from '../JobStatusModal'; import { MultistepJsonForm } from '../MultistepJsonForm'; +import { useJobPolling } from '../hooks/useJobPolling'; const useTemplate = ( templateName: string, @@ -81,10 +85,10 @@ export const TemplatePage = () => { const catalogApi = useApi(catalogApiRef); const scaffolderApi = useApi(scaffolderApiRef); const { templateName } = useParams(); + const [catalogLink, setCatalogLink] = useState(); const { template, loading } = useTemplate(templateName, catalogApi); - const [formState, setFormState] = useState({}); - + const [modalOpen, setModalOpen] = useState(false); const handleFormReset = () => setFormState({}); const handleChange = useCallback( (e: IChangeEvent) => setFormState({ ...formState, ...e.formData }), @@ -92,35 +96,42 @@ export const TemplatePage = () => { ); const [jobId, setJobId] = useState(null); - - const handleCreate = async () => { - try { - const job = await scaffolderApi.scaffold(templateName, formState); - setJobId(job); - } catch (e) { - errorApi.post(e); - } - }; - - const [entity, setEntity] = React.useState( - null, - ); - - const handleCreateComplete = async (job: Job) => { + const job = useJobPolling(jobId, async job => { if (!job.metadata.catalogInfoUrl) { errorApi.post( - new Error( - `Failed to find catalog-info.yaml file in ${job.metadata.remoteUrl}.`, - ), + new Error(`No catalogInfoUrl returned from the scaffolder`), ); return; } - const { - entities: [createdEntity], - } = await catalogApi.addLocation({ target: job.metadata.catalogInfoUrl }); + try { + const { + entities: [createdEntity], + } = await catalogApi.addLocation({ target: job.metadata.catalogInfoUrl }); - setEntity((createdEntity as any) as TemplateEntityV1alpha1); + const resolvedPath = generatePath( + `/catalog/${entityRoute.path}`, + entityRouteParams(createdEntity), + ); + + setCatalogLink(resolvedPath); + } catch (ex) { + errorApi.post( + new Error( + `Something went wrong trying to add the new 'catalog-info.yaml' to the catalog`, + ), + ); + } + }); + + const handleCreate = async () => { + try { + const jobId = await scaffolderApi.scaffold(templateName, formState); + setJobId(jobId); + setModalOpen(true); + } catch (e) { + errorApi.post(e); + } }; if (!loading && !template) { @@ -150,13 +161,7 @@ export const TemplatePage = () => { /> {loading && } - {jobId && ( - - )} + {modalOpen && } {template && ( Promise, ms: number) => { - let shouldStop = false; - (async () => { - while (!shouldStop) { - await thunk(); - await new Promise(res => setTimeout(res, ms)); - } - })(); - - return () => { - shouldStop = true; - }; -}; export const useJobPolling = ( jobId: string | null, + onFinish?: (j: Job) => void, pollingInterval = DEFAULT_POLLING_INTERVAL, ) => { const scaffolderApi = useApi(scaffolderApiRef); - const [job, setJob] = useState(null); - - useEffect(() => { - if (!jobId) return () => {}; - - const stopPolling = poll(async () => { - const nextJobState = await scaffolderApi.getJob(jobId); - if ( - nextJobState.status === 'FAILED' || - nextJobState.status === 'COMPLETED' - ) { - stopPolling(); + const [currentJob, setCurrentJob] = useState(null); + const shouldBeRunningInterval = + jobId && + currentJob?.status !== 'COMPLETED' && + currentJob?.status !== 'FAILED'; + useInterval( + async () => { + if (jobId) { + const job = await scaffolderApi.getJob(jobId); + if (job?.status === 'COMPLETED' || job?.status === 'FAILED') { + onFinish?.(job); + } + setCurrentJob(job); } - setJob(nextJobState); - }, pollingInterval); - return () => { - stopPolling(); - }; - }, [jobId, setJob, scaffolderApi, pollingInterval]); + }, + shouldBeRunningInterval ? pollingInterval : null, + ); - return job; + return currentJob; }; diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index e07581ad28..45672c603d 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +export type JobStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; export type Job = { id: string; metadata: { @@ -21,7 +23,7 @@ export type Job = { remoteUrl?: string; catalogInfoUrl?: string; }; - status: 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; + status: JobStatus; stages: Stage[]; error?: Error; }; @@ -29,7 +31,7 @@ export type Job = { export type Stage = { name: string; log: string[]; - status: 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; + status: JobStatus; startedAt: string; endedAt?: string; }; From b877f46fde512dd7b7168c5977b65bdd01963982 Mon Sep 17 00:00:00 2001 From: blam Date: Sun, 22 Nov 2020 03:07:23 +0100 Subject: [PATCH 221/259] chore(lint): fixing linting for hooks --- plugins/scaffolder/src/components/hooks/useJobPolling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder/src/components/hooks/useJobPolling.ts b/plugins/scaffolder/src/components/hooks/useJobPolling.ts index 91705c108f..1bfdfc53b8 100644 --- a/plugins/scaffolder/src/components/hooks/useJobPolling.ts +++ b/plugins/scaffolder/src/components/hooks/useJobPolling.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { useState, useEffect } from 'react'; +import { useState } from 'react'; import { Job } from '../../types'; import { useApi } from '@backstage/core'; import { scaffolderApiRef } from '../../api'; From 9cdf4413db1a45c313eaa7a15d0689ff3914a8bd Mon Sep 17 00:00:00 2001 From: blam Date: Sun, 22 Nov 2020 03:20:42 +0100 Subject: [PATCH 222/259] chore(scaffolder): trying to fix the linting and tsc issues --- .../src/components/JobStatusModal/JobStatusModal.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx index f0aa5ccd81..01a6346d24 100644 --- a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx +++ b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx @@ -28,7 +28,7 @@ import { Job } from '../../types'; import { JobStage } from '../JobStage/JobStage'; type Props = { - job: Job; + job: Job | null; toCatalogLink?: string; }; @@ -47,6 +47,9 @@ export const JobStatusModal = ({ job, toCatalogLink }: Props) => { }; const onClose = useCallback(() => { + if (!job) { + return; + } if (job.status === 'COMPLETED' || job.status === 'FAILED') { setOpen(false); } From 031ead5f5a81a8004017327434ac44f4b3bc42bd Mon Sep 17 00:00:00 2001 From: Kevin Kelly Date: Sat, 21 Nov 2020 23:33:32 -0500 Subject: [PATCH 223/259] docs: update local install example for node v14 --- docs/getting-started/running-backstage-locally.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/getting-started/running-backstage-locally.md b/docs/getting-started/running-backstage-locally.md index a30a918fd0..6de9d234e3 100644 --- a/docs/getting-started/running-backstage-locally.md +++ b/docs/getting-started/running-backstage-locally.md @@ -8,19 +8,19 @@ description: Documentation on How to run Backstage Locally - Node.js -First make sure you are using Node.js with an Active LTS Release, currently v12. +First make sure you are using Node.js with an Active LTS Release, currently v14. This is made easy with a version manager such as [nvm](https://github.com/nvm-sh/nvm) which allows for version switching. ```bash # Installing a new version -nvm install 12 -> Downloading and installing node v12.18.3... -> Now using node v12.18.3 (npm v6.14.6) +nvm install 14 +> Downloading and installing node v14.15.1... +> Now using node v14.15.1 (npm v6.14.8) # Checking your version node --version -> v12.18.3 +> v14.15.1 ``` - Yarn From ed08863e17cb02b32242f5474c4d8295d1a94375 Mon Sep 17 00:00:00 2001 From: Kevin Kelly Date: Sat, 21 Nov 2020 23:48:36 -0500 Subject: [PATCH 224/259] docs: create an app, node version update --- docs/getting-started/create-an-app.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/create-an-app.md b/docs/getting-started/create-an-app.md index 552b6c3ec2..6aef4c48c1 100644 --- a/docs/getting-started/create-an-app.md +++ b/docs/getting-started/create-an-app.md @@ -14,7 +14,7 @@ need to run Backstage in your own environment. To create a Backstage app, you will need to have [Node.js](https://nodejs.org/en/download/) Active LTS Release installed -(currently v12). +(currently v14). Backstage provides a utility for creating new apps. It guides you through the initial setup of selecting the name of the app and a database for the backend. From 0145faa6ee2faf8d4938c7d518f053a634009811 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 14:20:13 +0100 Subject: [PATCH 225/259] cli: add Lockfile utility --- packages/cli/package.json | 3 + .../cli/src/lib/versioning/Lockfile.test.ts | 150 +++++++++++ packages/cli/src/lib/versioning/Lockfile.ts | 242 ++++++++++++++++++ packages/cli/src/lib/versioning/index.ts | 17 ++ yarn.lock | 10 + 5 files changed, 422 insertions(+) create mode 100644 packages/cli/src/lib/versioning/Lockfile.test.ts create mode 100644 packages/cli/src/lib/versioning/Lockfile.ts create mode 100644 packages/cli/src/lib/versioning/index.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index ec3aaacdee..eb56b19971 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -51,6 +51,7 @@ "@types/webpack-node-externals": "^2.5.0", "@typescript-eslint/eslint-plugin": "^v3.10.1", "@typescript-eslint/parser": "^v3.10.1", + "@yarnpkg/lockfile": "^1.1.0", "bfj": "^7.0.2", "chalk": "^4.0.0", "chokidar": "^3.3.1", @@ -92,6 +93,7 @@ "rollup-plugin-postcss": "^3.1.1", "rollup-plugin-typescript2": "^0.27.3", "rollup-pluginutils": "^2.8.2", + "semver": "^7.3.2", "start-server-webpack-plugin": "^2.2.5", "style-loader": "^1.2.1", "sucrase": "^3.16.0", @@ -131,6 +133,7 @@ "@types/tar": "^4.0.3", "@types/webpack": "^4.41.7", "@types/webpack-dev-server": "^3.11.0", + "@types/yarnpkg__lockfile": "^1.1.4", "del": "^5.1.0", "mock-fs": "^4.13.0", "nodemon": "^2.0.2", diff --git a/packages/cli/src/lib/versioning/Lockfile.test.ts b/packages/cli/src/lib/versioning/Lockfile.test.ts new file mode 100644 index 0000000000..5459ed53fb --- /dev/null +++ b/packages/cli/src/lib/versioning/Lockfile.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { Lockfile } from './Lockfile'; + +const HEADER = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + +`; + +const mockA = `${HEADER} +a@^1: + version "1.0.1" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz + dependencies: + b "^2" + +b@2.0.x: + version "2.0.1" + +b@^2: + version "2.0.0" +`; + +const mockADedup = `${HEADER} +a@^1: + version "1.0.1" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz + dependencies: + b "^2" + +b@2.0.x, b@^2: + version "2.0.1" +`; + +const mockB = `${HEADER} +"@s/a@*", "@s/a@1 || 2", "@s/a@^1": + version "1.0.1" + +"@s/a@^2.0.x": + version "2.0.0" +`; + +const mockBDedup = `${HEADER} +"@s/a@*", "@s/a@1 || 2", "@s/a@^2.0.x": + version "2.0.0" + +"@s/a@^1": + version "1.0.1" +`; + +describe('Lockfile', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should load and serialize mockA', async () => { + mockFs({ + '/yarn.lock': mockA, + }); + + const lockfile = await Lockfile.load('/yarn.lock'); + expect(lockfile.get('a')).toEqual([{ range: '^1', version: '1.0.1' }]); + expect(lockfile.get('b')).toEqual([ + { range: '2.0.x', version: '2.0.1' }, + { range: '^2', version: '2.0.0' }, + ]); + expect(lockfile.toString()).toBe(mockA); + }); + + it('should deduplicate mockA', async () => { + mockFs({ + '/yarn.lock': mockA, + }); + + const lockfile = await Lockfile.load('/yarn.lock'); + const result = lockfile.analyze(); + expect(result).toEqual({ + invalidRanges: [], + newRanges: [], + newVersions: [ + { + name: 'b', + range: '^2', + oldVersion: '2.0.0', + newVersion: '2.0.1', + }, + ], + }); + + expect(lockfile.toString()).toBe(mockA); + lockfile.replaceVersions(result.newVersions); + expect(lockfile.toString()).toBe(mockADedup); + }); + + it('should deduplicate mockB', async () => { + mockFs({ + '/yarn.lock': mockB, + }); + + const lockfile = await Lockfile.load('/yarn.lock'); + const result = lockfile.analyze(); + expect(result).toEqual({ + invalidRanges: [], + newRanges: [ + { + name: '@s/a', + oldRange: '^1', + newRange: '*', + oldVersion: '1.0.1', + newVersion: '2.0.0', + }, + ], + newVersions: [ + { + name: '@s/a', + range: '*', + oldVersion: '1.0.1', + newVersion: '2.0.0', + }, + { + name: '@s/a', + range: '1 || 2', + oldVersion: '1.0.1', + newVersion: '2.0.0', + }, + ], + }); + + expect(lockfile.toString()).toBe(mockB); + lockfile.replaceVersions(result.newVersions); + expect(lockfile.toString()).toBe(mockBDedup); + }); +}); diff --git a/packages/cli/src/lib/versioning/Lockfile.ts b/packages/cli/src/lib/versioning/Lockfile.ts new file mode 100644 index 0000000000..e03fa8ec3f --- /dev/null +++ b/packages/cli/src/lib/versioning/Lockfile.ts @@ -0,0 +1,242 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import semver from 'semver'; +import { + parse as parseLockfile, + stringify as stringifyLockfile, +} from '@yarnpkg/lockfile'; + +const ENTRY_PATTERN = /^((?:@[^/]+\/)?[^@/]+)@(.+)$/; + +type LockfileData = { + [entry: string]: { + version: string; + resolved?: string; + integrity?: string; + dependencies?: { [name: string]: string }; + }; +}; + +type LockfileQueryEntry = { + range: string; + version: string; +}; + +/** Entries that have an invalid version range, for example an NPM tag */ +type AnalyzeResultInvalidRange = { + name: string; + range: string; +}; + +/** Entries that can be deduplicated by bumping to an existing higher version */ +type AnalyzeResultNewVersion = { + name: string; + range: string; + oldVersion: string; + newVersion: string; +}; + +/** Entries that {would need a dependency update in package.json to be deduplicated */ +type AnalyzeResultNewRange = { + name: string; + oldRange: string; + newRange: string; + oldVersion: string; + newVersion: string; +}; + +type AnalyzeResult = { + invalidRanges: AnalyzeResultInvalidRange[]; + newVersions: AnalyzeResultNewVersion[]; + newRanges: AnalyzeResultNewRange[]; +}; + +export class Lockfile { + static async load(path: string) { + const lockfileContents = await fs.readFile(path, 'utf8'); + const lockfile = parseLockfile(lockfileContents); + if (lockfile.type !== 'success') { + throw new Error(`Failed yarn.lock parse with ${lockfile.type}`); + } + + const data = lockfile.object as LockfileData; + const packages = new Map(); + + for (const [key, value] of Object.entries(data)) { + const [, name, range] = ENTRY_PATTERN.exec(key) ?? []; + if (!name) { + throw new Error(`Failed to parse yarn.lock entry '${key}'`); + } + + let queries = packages.get(name); + if (!queries) { + queries = []; + packages.set(name, queries); + } + queries.push({ range, version: value.version }); + } + + return new Lockfile(packages, data); + } + + constructor( + private readonly packages: Map, + private readonly data: LockfileData, + ) {} + + get(name: string): LockfileQueryEntry[] | undefined { + return this.packages.get(name); + } + + /** Analyzes the lockfile to identify possible actions and warnings for the entries */ + analyze(options?: { filter?: (name: string) => boolean }): AnalyzeResult { + const { filter } = options ?? {}; + const result: AnalyzeResult = { + invalidRanges: [], + newVersions: [], + newRanges: [], + }; + + for (const [name, allEntries] of this.packages) { + if (filter && !filter(name)) { + continue; + } + + // Get rid of an signal any invalid ranges upfront + const invalid = allEntries.filter(e => !semver.validRange(e.range)); + result.invalidRanges.push( + ...invalid.map(({ range }) => ({ name, range })), + ); + + // Grab all valid entries, if there isn't at least 2 different valid ones we're done + const entries = allEntries.filter(e => semver.validRange(e.range)); + if (entries.length < 2) { + continue; + } + + // Find all versions currently in use + const versions = Array.from( + new Set(entries.map(e => e.version)), + ).sort((v1, v2) => semver.rcompare(v1, v2)); + + // If we're not using at least 2 different versions we're done + if (versions.length < 2) { + continue; + } + + const acceptedVersions = new Set(); + for (const { version, range } of entries) { + // Finds the highest matching version from the the known versions + // TODO(Rugvip): We may want to select the version that satisfies the most ranges rather than the highest one + const acceptedVersion = versions.find(v => semver.satisfies(v, range)); + if (!acceptedVersion) { + throw new Error( + `No existing version was accepted for range ${range}, searching through ${versions}`, + ); + } + + if (acceptedVersion !== version) { + result.newVersions.push({ + name, + range, + newVersion: acceptedVersion, + oldVersion: version, + }); + } + + acceptedVersions.add(acceptedVersion); + } + + // If all ranges where able to accept the same version, we're done + if (acceptedVersions.size === 1) { + continue; + } + + // Find the max version and range that we may want bump older packages to + const maxVersion = Array.from(acceptedVersions).sort(semver.rcompare)[0]; + const maxEntry = entries.find(e => semver.satisfies(maxVersion, e.range)); + if (!maxEntry) { + throw new Error( + `No entry found that satisfies max version '${maxVersion}'`, + ); + } + + // Find all entries that don't satisfy the max version + for (const { version, range } of entries) { + if (semver.satisfies(maxVersion, range)) { + continue; + } + + result.newRanges.push({ + name, + oldRange: range, + newRange: maxEntry.range, + oldVersion: version, + newVersion: maxVersion, + }); + } + } + + return result; + } + + /** Modifies the lockfile by bumping packages to the suggested versions */ + replaceVersions(results: AnalyzeResultNewVersion[]) { + for (const { name, range, oldVersion, newVersion } of results) { + const query = `${name}@${range}`; + + // Update the backing data + const entryData = this.data[query]; + if (!entryData) { + throw new Error(`No entry data for ${query}`); + } + if (entryData.version !== oldVersion) { + throw new Error( + `Expected existing version data for ${query} to be ${oldVersion}, was ${entryData.version}`, + ); + } + + // Modifying the data in the entry is not enough, we need to reference an existing version object + const matchingEntry = Object.entries(this.data).find( + ([q, e]) => q.startsWith(`${name}@`) && e.version === newVersion, + ); + if (!matchingEntry) { + throw new Error( + `No matching entry found for ${name} at version ${newVersion}`, + ); + } + this.data[query] = matchingEntry[1]; + + // Update out internal data structure + const entry = this.packages.get(name)?.find(e => e.range === range); + if (!entry) { + throw new Error(`No entry data for ${query}`); + } + if (entry.version !== oldVersion) { + throw new Error( + `Expected existing version data for ${query} to be ${oldVersion}, was ${entryData.version}`, + ); + } + entry.version = newVersion; + } + } + + toString() { + return stringifyLockfile(this.data); + } +} diff --git a/packages/cli/src/lib/versioning/index.ts b/packages/cli/src/lib/versioning/index.ts new file mode 100644 index 0000000000..6f0a649ee4 --- /dev/null +++ b/packages/cli/src/lib/versioning/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { Lockfile } from './Lockfile'; diff --git a/yarn.lock b/yarn.lock index a90d885927..92aecbba36 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6088,6 +6088,11 @@ dependencies: "@types/yargs-parser" "*" +"@types/yarnpkg__lockfile@^1.1.4": + version "1.1.4" + resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.4.tgz#445251eb00bd9c1e751f82c7c6bf4f714edfd464" + integrity sha512-/emrKCfQMQmFCqRqqBJ0JueHBT06jBRM3e8OgnvDUcvuExONujIk2hFA5dNsN9Nt41ljGVDdChvCydATZ+KOZw== + "@types/yup@^0.29.8": version "0.29.8" resolved "https://registry.npmjs.org/@types/yup/-/yup-0.29.8.tgz#83db15735987db9fe5a38772a0fb9500e3c5bf39" @@ -6384,6 +6389,11 @@ resolved "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== +"@yarnpkg/lockfile@^1.1.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31" + integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ== + "@zkochan/cmd-shim@^3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@zkochan/cmd-shim/-/cmd-shim-3.1.0.tgz#2ab8ed81f5bb5452a85f25758eb9b8681982fd2e" From 4c642721376961ecc06c9abde42ba70534d9b63c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 14:48:12 +0100 Subject: [PATCH 226/259] cli: make Lockfile analyze pick the most specific new version range --- packages/cli/src/lib/versioning/Lockfile.test.ts | 2 +- packages/cli/src/lib/versioning/Lockfile.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/lib/versioning/Lockfile.test.ts b/packages/cli/src/lib/versioning/Lockfile.test.ts index 5459ed53fb..20847242d3 100644 --- a/packages/cli/src/lib/versioning/Lockfile.test.ts +++ b/packages/cli/src/lib/versioning/Lockfile.test.ts @@ -122,7 +122,7 @@ describe('Lockfile', () => { { name: '@s/a', oldRange: '^1', - newRange: '*', + newRange: '^2.0.x', oldVersion: '1.0.1', newVersion: '2.0.0', }, diff --git a/packages/cli/src/lib/versioning/Lockfile.ts b/packages/cli/src/lib/versioning/Lockfile.ts index e03fa8ec3f..8acb387704 100644 --- a/packages/cli/src/lib/versioning/Lockfile.ts +++ b/packages/cli/src/lib/versioning/Lockfile.ts @@ -167,9 +167,15 @@ export class Lockfile { continue; } - // Find the max version and range that we may want bump older packages to + // Find the max version that we may want bump older packages to const maxVersion = Array.from(acceptedVersions).sort(semver.rcompare)[0]; - const maxEntry = entries.find(e => semver.satisfies(maxVersion, e.range)); + // Find all existing ranges that satisfy the new max version, and pick the one that + // results in the highest minimum allowed version, usually being the more specific one + const maxEntry = entries + .filter(e => semver.satisfies(maxVersion, e.range)) + .map(e => ({ e, min: semver.minVersion(e.range) })) + .filter(p => p.min) + .sort((a, b) => semver.rcompare(a.min!, b.min!))[0]?.e; if (!maxEntry) { throw new Error( `No entry found that satisfies max version '${maxVersion}'`, From 46a20ad22fbfd2f5cf3965e8eedbf3a2118b6f35 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 15:58:28 +0100 Subject: [PATCH 227/259] cli: add versions:lint command --- packages/cli/src/commands/index.ts | 6 ++ packages/cli/src/commands/versions/lint.ts | 114 +++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 packages/cli/src/commands/versions/lint.ts diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 1ebe518cac..3f3e8bfe83 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -154,6 +154,12 @@ export function registerCommands(program: CommanderStatic) { ) .action(lazy(() => import('./config/validate').then(m => m.default))); + program + .command('versions:lint') + .option('--fix', 'Fix any auto-fixable versioning problems') + .description('Lint Backstage package versioning') + .action(lazy(() => import('./versions/lint').then(m => m.default))); + program .command('prepack') .description('Prepares a package for packaging before publishing') diff --git a/packages/cli/src/commands/versions/lint.ts b/packages/cli/src/commands/versions/lint.ts new file mode 100644 index 0000000000..2f200a75a9 --- /dev/null +++ b/packages/cli/src/commands/versions/lint.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { Command } from 'commander'; +import { Lockfile } from '../../lib/versioning'; +import { paths } from '../../lib/paths'; +import partition from 'lodash/partition'; + +// Packages that we try to avoid duplicates for +const INCLUDED = [/^@backstage\//]; + +// Packages that are not allowed to have any duplicates +const FORBID_DUPLICATES = [ + /^@backstage\/core$/, + /^@backstage\/core-api$/, + /^@backstage\/plugin-/, +]; + +export default async (cmd: Command) => { + const fix = Boolean(cmd.fix); + + let success = true; + + const lockfilePath = paths.resolveTargetRoot('yarn.lock'); + const lockfile = await Lockfile.load(lockfilePath); + const result = lockfile.analyze({ + filter: name => INCLUDED.some(pattern => pattern.test(name)), + }); + + logArray( + result.invalidRanges, + "The following packages versions are invalid and can't be analyzed:", + e => ` ${e.name} @ ${e.range}`, + ); + + if (fix) { + lockfile.replaceVersions(result.newVersions); + + await fs.writeFile(lockfilePath, lockfile.toString(), 'utf8'); + } else { + const [ + newVersionsForbidden, + newVersionsAllowed, + ] = partition(result.newVersions, ({ name }) => + FORBID_DUPLICATES.some(pattern => pattern.test(name)), + ); + if (newVersionsForbidden.length && !fix) { + success = false; + } + + logArray( + newVersionsForbidden, + 'The following packages must be deduplicated, this can be done automatically with --fix', + e => + ` ${e.name} @ ${e.range} bumped from ${e.oldVersion} to ${e.newVersion}`, + ); + logArray( + newVersionsAllowed, + 'The following packages can be deduplicated, this can be done automatically with --fix', + e => + ` ${e.name} @ ${e.range} bumped from ${e.oldVersion} to ${e.newVersion}`, + ); + } + + const [newRangesForbidden, newRangesAllowed] = partition( + result.newRanges, + ({ name }) => FORBID_DUPLICATES.some(pattern => pattern.test(name)), + ); + if (newRangesForbidden.length) { + success = false; + } + + logArray( + newRangesForbidden, + 'The following packages must be deduplicated by updating dependencies in package.json', + e => ` ${e.name} @ ${e.oldRange} should be changed to ${e.newRange}`, + ); + logArray( + newRangesAllowed, + 'The following packages can be deduplicated by updating dependencies in package.json', + e => ` ${e.name} @ ${e.oldRange} should be changed to ${e.newRange}`, + ); + + if (!success) { + throw new Error('Failed versioning check'); + } +}; + +function logArray(arr: T[], header: string, each: (item: T) => void) { + if (arr.length === 0) { + return; + } + + console.log(header); + console.log(); + for (const e of arr) { + console.log(each(e)); + } + console.log(); +} From 06432a0e9973be53a73bd833e1b42eddb1901b5f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 17:17:44 +0100 Subject: [PATCH 228/259] cli: add versions:bump command --- packages/cli/src/commands/index.ts | 5 + packages/cli/src/commands/versions/bump.ts | 199 +++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 packages/cli/src/commands/versions/bump.ts diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 3f3e8bfe83..9f967a2727 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -154,6 +154,11 @@ export function registerCommands(program: CommanderStatic) { ) .action(lazy(() => import('./config/validate').then(m => m.default))); + program + .command('versions:bump') + .description('Bump Backstage packages to the latest versions') + .action(lazy(() => import('./versions/bump').then(m => m.default))); + program .command('versions:lint') .option('--fix', 'Fix any auto-fixable versioning problems') diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts new file mode 100644 index 0000000000..87f6d6c12b --- /dev/null +++ b/packages/cli/src/commands/versions/bump.ts @@ -0,0 +1,199 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import semver from 'semver'; +import { resolve as resolvePath } from 'path'; +import { run, runPlain } from '../../lib/run'; +import { paths } from '../../lib/paths'; +import { Lockfile } from '../../lib/versioning'; + +const PREFIX = '@backstage'; + +const DEP_TYPES = [ + 'dependencies', + 'devDependencies', + 'peerDependencies', + 'optionalDependencies', +]; + +// Package data as returned by `yarn info` +type YarnInfoInspectData = { + name: string; + 'dist-tags': { latest: string }; + versions: string[]; + time: { [version: string]: string }; +}; + +// Possible `yarn info` output +type YarnInfo = { + type: 'inspect'; + data: YarnInfoInspectData | { type: string; data: unknown }; +}; + +type PkgVersionInfo = { + range: string; + name: string; + location: string; +}; + +export default async () => { + const LernaProject = require('@lerna/project'); + const project = new LernaProject(paths.targetDir); + const packages = await project.getPackages(); + + // First we discover all Backstage dependencies within our own repo + const dependencyMap = new Map(); + for (const pkg of packages) { + const deps = DEP_TYPES.flatMap( + t => Object.entries(pkg.get(t) ?? {}) as [string, string][], + ); + + for (const [name, range] of deps) { + if (name.startsWith(PREFIX)) { + dependencyMap.set( + name, + (dependencyMap.get(name) ?? []).concat({ + range, + name: pkg.name, + location: pkg.location, + }), + ); + } + } + } + + // Next check with the package registry what the latest version of all of those dependencies are + const targetVersions = new Map(); + await workerThreads(16, dependencyMap.keys(), async name => { + console.log(`Checking for updates of ${name}`); + const output = await runPlain('yarn', 'info', '--json', name); + const info = JSON.parse(output) as YarnInfo; + if (info.type !== 'inspect') { + throw new Error(`Received unknown yarn info for ${name}, ${output}`); + } + + const data = info.data as YarnInfoInspectData; + const latest = data['dist-tags'].latest; + if (!latest) { + throw new Error(`No latest version found for ${name}`); + } + + targetVersions.set(name, latest); + }); + + // Then figure out which local packages need to have their dependencies bumped + const versionBumps = new Map(); + for (const [name, pkgs] of dependencyMap) { + const targetVersion = targetVersions.get(name)!; + for (const pkg of pkgs) { + if (semver.satisfies(targetVersion, pkg.range)) { + continue; + } + + versionBumps.set( + pkg.name, + (versionBumps.get(pkg.name) ?? []).concat({ + name, + location: pkg.location, + range: `^${targetVersion}`, // TODO(Rugvip): Option to use something else than ^? + }), + ); + } + } + + console.log(); + + // Write all discovered version bumps to package.json in this repo + if (versionBumps.size === 0) { + console.log('All Backstage packages are up to date!'); + } else { + console.log('Some packages are outdated, updating'); + console.log(); + + await workerThreads(16, versionBumps.entries(), async ([name, deps]) => { + const pkgPath = resolvePath(deps[0].location, 'package.json'); + const pkgJson = await fs.readJson(pkgPath); + + for (const dep of deps) { + console.log(`Bumping ${dep.name} in ${name} to ${dep.range}`); + + for (const depType of DEP_TYPES) { + if (depType in pkgJson && dep.name in pkgJson[depType]) { + pkgJson[depType][dep.name] = dep.range; + } + } + } + + await fs.writeJson(pkgPath, pkgJson, { spaces: 2 }); + }); + + console.log(); + console.log("Running 'yarn install' to install new versions"); + console.log(); + await run('yarn', ['install']); + } + + console.log(); + + // Finally we make sure the new lockfile doesn't have any duplicates + const lockfilePath = paths.resolveTargetRoot('yarn.lock'); + const lockfile = await Lockfile.load(lockfilePath); + const result = lockfile.analyze({ + filter: name => dependencyMap.has(name), + }); + + if (result.newVersions.length > 0) { + console.log(); + console.log('Removing duplicate dependencies from yarn.lock'); + lockfile.replaceVersions(result.newVersions); + await fs.writeFile(lockfilePath, lockfile.toString(), 'utf8'); + + console.log( + "Running 'yarn install' to remove duplicates from node_modules", + ); + console.log(); + await run('yarn', ['install']); + + console.log(); + } + + if (result.newRanges.length > 0) { + throw new Error( + `Version bump failed for ${result.newRanges.map(i => i.name).join(', ')}`, + ); + } +}; + +async function workerThreads( + count: number, + items: IterableIterator, + fn: (item: T) => Promise, +) { + const queue = Array.from(items); + + async function pop() { + const item = queue.pop()!; + if (!item) { + return; + } + + await fn(item); + await pop(); + } + + return Promise.all(Array(count).fill(0).map(pop)); +} From b90ad40d1c6c4e3e235fca9c1b5a652eedc625b8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 17:42:52 +0100 Subject: [PATCH 229/259] cli: add Lockfile.save --- packages/cli/src/commands/versions/bump.ts | 11 +++++++---- packages/cli/src/commands/versions/lint.ts | 7 ++----- packages/cli/src/lib/versioning/Lockfile.test.ts | 7 ++++++- packages/cli/src/lib/versioning/Lockfile.ts | 9 +++++++-- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts index 87f6d6c12b..c2db4eff6e 100644 --- a/packages/cli/src/commands/versions/bump.ts +++ b/packages/cli/src/commands/versions/bump.ts @@ -150,8 +150,7 @@ export default async () => { console.log(); // Finally we make sure the new lockfile doesn't have any duplicates - const lockfilePath = paths.resolveTargetRoot('yarn.lock'); - const lockfile = await Lockfile.load(lockfilePath); + const lockfile = await Lockfile.load(paths.resolveTargetRoot('yarn.lock')); const result = lockfile.analyze({ filter: name => dependencyMap.has(name), }); @@ -160,7 +159,7 @@ export default async () => { console.log(); console.log('Removing duplicate dependencies from yarn.lock'); lockfile.replaceVersions(result.newVersions); - await fs.writeFile(lockfilePath, lockfile.toString(), 'utf8'); + await lockfile.save(); console.log( "Running 'yarn install' to remove duplicates from node_modules", @@ -195,5 +194,9 @@ async function workerThreads( await pop(); } - return Promise.all(Array(count).fill(0).map(pop)); + return Promise.all( + Array(count) + .fill(0) + .map(pop), + ); } diff --git a/packages/cli/src/commands/versions/lint.ts b/packages/cli/src/commands/versions/lint.ts index 2f200a75a9..f91d2b74eb 100644 --- a/packages/cli/src/commands/versions/lint.ts +++ b/packages/cli/src/commands/versions/lint.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import fs from 'fs-extra'; import { Command } from 'commander'; import { Lockfile } from '../../lib/versioning'; import { paths } from '../../lib/paths'; @@ -35,8 +34,7 @@ export default async (cmd: Command) => { let success = true; - const lockfilePath = paths.resolveTargetRoot('yarn.lock'); - const lockfile = await Lockfile.load(lockfilePath); + const lockfile = await Lockfile.load(paths.resolveTargetRoot('yarn.lock')); const result = lockfile.analyze({ filter: name => INCLUDED.some(pattern => pattern.test(name)), }); @@ -49,8 +47,7 @@ export default async (cmd: Command) => { if (fix) { lockfile.replaceVersions(result.newVersions); - - await fs.writeFile(lockfilePath, lockfile.toString(), 'utf8'); + await lockfile.save(); } else { const [ newVersionsForbidden, diff --git a/packages/cli/src/lib/versioning/Lockfile.test.ts b/packages/cli/src/lib/versioning/Lockfile.test.ts index 20847242d3..89126c7ce1 100644 --- a/packages/cli/src/lib/versioning/Lockfile.test.ts +++ b/packages/cli/src/lib/versioning/Lockfile.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import fs from 'fs-extra'; import mockFs from 'mock-fs'; import { Lockfile } from './Lockfile'; @@ -84,7 +85,7 @@ describe('Lockfile', () => { expect(lockfile.toString()).toBe(mockA); }); - it('should deduplicate mockA', async () => { + it('should deduplicate and save mockA', async () => { mockFs({ '/yarn.lock': mockA, }); @@ -107,6 +108,10 @@ describe('Lockfile', () => { expect(lockfile.toString()).toBe(mockA); lockfile.replaceVersions(result.newVersions); expect(lockfile.toString()).toBe(mockADedup); + + await expect(fs.readFile('/yarn.lock', 'utf8')).resolves.toBe(mockA); + await expect(lockfile.save()).resolves.toBeUndefined(); + await expect(fs.readFile('/yarn.lock', 'utf8')).resolves.toBe(mockADedup); }); it('should deduplicate mockB', async () => { diff --git a/packages/cli/src/lib/versioning/Lockfile.ts b/packages/cli/src/lib/versioning/Lockfile.ts index 8acb387704..59a734c314 100644 --- a/packages/cli/src/lib/versioning/Lockfile.ts +++ b/packages/cli/src/lib/versioning/Lockfile.ts @@ -91,10 +91,11 @@ export class Lockfile { queries.push({ range, version: value.version }); } - return new Lockfile(packages, data); + return new Lockfile(path, packages, data); } - constructor( + private constructor( + private readonly path: string, private readonly packages: Map, private readonly data: LockfileData, ) {} @@ -242,6 +243,10 @@ export class Lockfile { } } + async save() { + await fs.writeFile(this.path, this.toString(), 'utf8'); + } + toString() { return stringifyLockfile(this.data); } From 99134596986b1e5c7b1594a2c50b60d8a13d3995 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 17:44:25 +0100 Subject: [PATCH 230/259] changesets: add cli versions command change --- .changeset/poor-mails-marry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/poor-mails-marry.md diff --git a/.changeset/poor-mails-marry.md b/.changeset/poor-mails-marry.md new file mode 100644 index 0000000000..586bf91c47 --- /dev/null +++ b/.changeset/poor-mails-marry.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Add new `versions:lint` and `versions:bump` commands to simplify version management and avoid conflicts From c0f7591dd7ab805ef16e1be727636acbcd7110fd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 18:17:55 +0100 Subject: [PATCH 231/259] cli: refactor out parts of versioning:bump + add tests --- packages/cli/src/commands/versions/bump.ts | 58 ++-------- packages/cli/src/lib/versioning/index.ts | 1 + .../cli/src/lib/versioning/packages.test.ts | 109 ++++++++++++++++++ packages/cli/src/lib/versioning/packages.ts | 90 +++++++++++++++ 4 files changed, 209 insertions(+), 49 deletions(-) create mode 100644 packages/cli/src/lib/versioning/packages.test.ts create mode 100644 packages/cli/src/lib/versioning/packages.ts diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts index c2db4eff6e..1d8b9cecf3 100644 --- a/packages/cli/src/commands/versions/bump.ts +++ b/packages/cli/src/commands/versions/bump.ts @@ -17,11 +17,13 @@ import fs from 'fs-extra'; import semver from 'semver'; import { resolve as resolvePath } from 'path'; -import { run, runPlain } from '../../lib/run'; +import { run } from '../../lib/run'; import { paths } from '../../lib/paths'; -import { Lockfile } from '../../lib/versioning'; - -const PREFIX = '@backstage'; +import { + mapDependencies, + fetchPackageInfo, + Lockfile, +} from '../../lib/versioning'; const DEP_TYPES = [ 'dependencies', @@ -30,20 +32,6 @@ const DEP_TYPES = [ 'optionalDependencies', ]; -// Package data as returned by `yarn info` -type YarnInfoInspectData = { - name: string; - 'dist-tags': { latest: string }; - versions: string[]; - time: { [version: string]: string }; -}; - -// Possible `yarn info` output -type YarnInfo = { - type: 'inspect'; - data: YarnInfoInspectData | { type: string; data: unknown }; -}; - type PkgVersionInfo = { range: string; name: string; @@ -51,43 +39,15 @@ type PkgVersionInfo = { }; export default async () => { - const LernaProject = require('@lerna/project'); - const project = new LernaProject(paths.targetDir); - const packages = await project.getPackages(); - // First we discover all Backstage dependencies within our own repo - const dependencyMap = new Map(); - for (const pkg of packages) { - const deps = DEP_TYPES.flatMap( - t => Object.entries(pkg.get(t) ?? {}) as [string, string][], - ); - - for (const [name, range] of deps) { - if (name.startsWith(PREFIX)) { - dependencyMap.set( - name, - (dependencyMap.get(name) ?? []).concat({ - range, - name: pkg.name, - location: pkg.location, - }), - ); - } - } - } + const dependencyMap = await mapDependencies(); // Next check with the package registry what the latest version of all of those dependencies are const targetVersions = new Map(); await workerThreads(16, dependencyMap.keys(), async name => { console.log(`Checking for updates of ${name}`); - const output = await runPlain('yarn', 'info', '--json', name); - const info = JSON.parse(output) as YarnInfo; - if (info.type !== 'inspect') { - throw new Error(`Received unknown yarn info for ${name}, ${output}`); - } - - const data = info.data as YarnInfoInspectData; - const latest = data['dist-tags'].latest; + const info = await fetchPackageInfo(name); + const latest = info['dist-tags'].latest; if (!latest) { throw new Error(`No latest version found for ${name}`); } diff --git a/packages/cli/src/lib/versioning/index.ts b/packages/cli/src/lib/versioning/index.ts index 6f0a649ee4..71fb7647ce 100644 --- a/packages/cli/src/lib/versioning/index.ts +++ b/packages/cli/src/lib/versioning/index.ts @@ -15,3 +15,4 @@ */ export { Lockfile } from './Lockfile'; +export { fetchPackageInfo, mapDependencies } from './packages'; diff --git a/packages/cli/src/lib/versioning/packages.test.ts b/packages/cli/src/lib/versioning/packages.test.ts new file mode 100644 index 0000000000..ba328fa356 --- /dev/null +++ b/packages/cli/src/lib/versioning/packages.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import * as runObj from '../run'; +import { paths } from '../paths'; +import { fetchPackageInfo, mapDependencies } from './packages'; + +describe('fetchPackageInfo', () => { + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should forward info', async () => { + jest + .spyOn(runObj, 'runPlain') + .mockResolvedValue(`{"type":"inspect","data":{"the":"data"}}`); + + await expect(fetchPackageInfo('my-package')).resolves.toEqual({ + the: 'data', + }); + expect(runObj.runPlain).toHaveBeenCalledWith( + 'yarn', + 'info', + '--json', + 'my-package', + ); + }); +}); + +describe('mapDependencies', () => { + afterEach(() => { + mockFs.restore(); + jest.resetAllMocks(); + }); + + it('should derp', async () => { + // Make sure all modules involved in package discovery are in the module cache before we mock fs + const LernaProject = require('@lerna/project'); + const project = new LernaProject(paths.targetDir); + await project.getPackages(); + + mockFs({ + '/lerna.json': JSON.stringify({ + packages: ['pkgs/*'], + }), + '/pkgs/a/package.json': JSON.stringify({ + name: 'a', + dependencies: { + '@backstage/core': '1 || 2', + }, + }), + '/pkgs/b/package.json': JSON.stringify({ + name: 'b', + dependencies: { + '@backstage/core': '3', + '@backstage/cli': '^0', + }, + }), + }); + + const oldDir = paths.targetDir; + paths.targetDir = '/'; + + const dependencyMap = await mapDependencies(); + expect(Array.from(dependencyMap)).toEqual([ + [ + '@backstage/core', + [ + { + name: 'a', + range: '1 || 2', + location: '/pkgs/a', + }, + { + name: 'b', + range: '3', + location: '/pkgs/b', + }, + ], + ], + [ + '@backstage/cli', + [ + { + name: 'b', + range: '^0', + location: '/pkgs/b', + }, + ], + ], + ]); + + paths.targetDir = oldDir; + }); +}); diff --git a/packages/cli/src/lib/versioning/packages.ts b/packages/cli/src/lib/versioning/packages.ts new file mode 100644 index 0000000000..239fe575d6 --- /dev/null +++ b/packages/cli/src/lib/versioning/packages.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { runPlain } from '../../lib/run'; +import { paths } from '../../lib/paths'; + +const PREFIX = '@backstage'; + +const DEP_TYPES = [ + 'dependencies', + 'devDependencies', + 'peerDependencies', + 'optionalDependencies', +]; + +// Package data as returned by `yarn info` +type YarnInfoInspectData = { + name: string; + 'dist-tags': { latest: string }; + versions: string[]; + time: { [version: string]: string }; +}; + +// Possible `yarn info` output +type YarnInfo = { + type: 'inspect'; + data: YarnInfoInspectData | { type: string; data: unknown }; +}; + +type PkgVersionInfo = { + range: string; + name: string; + location: string; +}; + +export async function fetchPackageInfo( + name: string, +): Promise { + const output = await runPlain('yarn', 'info', '--json', name); + const info = JSON.parse(output) as YarnInfo; + if (info.type !== 'inspect') { + throw new Error(`Received unknown yarn info for ${name}, ${output}`); + } + + return info.data as YarnInfoInspectData; +} + +/** Map all dependencies in the repo as dependency => dependents */ +export async function mapDependencies(): Promise< + Map +> { + const LernaProject = require('@lerna/project'); + const project = new LernaProject(paths.targetDir); + const packages = await project.getPackages(); + + const dependencyMap = new Map(); + for (const pkg of packages) { + const deps = DEP_TYPES.flatMap( + t => Object.entries(pkg.get(t) ?? {}) as [string, string][], + ); + + for (const [name, range] of deps) { + if (name.startsWith(PREFIX)) { + dependencyMap.set( + name, + (dependencyMap.get(name) ?? []).concat({ + range, + name: pkg.name, + location: pkg.location, + }), + ); + } + } + } + + return dependencyMap; +} From 1af14032a090d599e6e9fc438350eb18528a21f1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 18:32:32 +0100 Subject: [PATCH 232/259] cli: rename versions:lint to versions:check --- .changeset/poor-mails-marry.md | 2 +- packages/cli/src/commands/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/poor-mails-marry.md b/.changeset/poor-mails-marry.md index 586bf91c47..fdc7e6f1b8 100644 --- a/.changeset/poor-mails-marry.md +++ b/.changeset/poor-mails-marry.md @@ -2,4 +2,4 @@ '@backstage/cli': patch --- -Add new `versions:lint` and `versions:bump` commands to simplify version management and avoid conflicts +Add new `versions:check` and `versions:bump` commands to simplify version management and avoid conflicts diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 9f967a2727..60c1b45cfc 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -160,9 +160,9 @@ export function registerCommands(program: CommanderStatic) { .action(lazy(() => import('./versions/bump').then(m => m.default))); program - .command('versions:lint') + .command('versions:check') .option('--fix', 'Fix any auto-fixable versioning problems') - .description('Lint Backstage package versioning') + .description('Check Backstage package versioning') .action(lazy(() => import('./versions/lint').then(m => m.default))); program From 351539680bcf0eb1406591a0e81590cd89eb8114 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 20:37:06 +0100 Subject: [PATCH 233/259] cli: review feedback on versions:* commands --- packages/cli/src/commands/versions/bump.ts | 2 +- packages/cli/src/commands/versions/lint.ts | 2 +- packages/cli/src/lib/versioning/Lockfile.ts | 10 +++++----- packages/cli/src/lib/versioning/packages.test.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts index 1d8b9cecf3..526ab9d4cf 100644 --- a/packages/cli/src/commands/versions/bump.ts +++ b/packages/cli/src/commands/versions/bump.ts @@ -145,7 +145,7 @@ async function workerThreads( const queue = Array.from(items); async function pop() { - const item = queue.pop()!; + const item = queue.pop(); if (!item) { return; } diff --git a/packages/cli/src/commands/versions/lint.ts b/packages/cli/src/commands/versions/lint.ts index f91d2b74eb..e2c36ba094 100644 --- a/packages/cli/src/commands/versions/lint.ts +++ b/packages/cli/src/commands/versions/lint.ts @@ -97,7 +97,7 @@ export default async (cmd: Command) => { } }; -function logArray(arr: T[], header: string, each: (item: T) => void) { +function logArray(arr: T[], header: string, each: (item: T) => string) { if (arr.length === 0) { return; } diff --git a/packages/cli/src/lib/versioning/Lockfile.ts b/packages/cli/src/lib/versioning/Lockfile.ts index 59a734c314..893beca6e2 100644 --- a/packages/cli/src/lib/versioning/Lockfile.ts +++ b/packages/cli/src/lib/versioning/Lockfile.ts @@ -51,7 +51,7 @@ type AnalyzeResultNewVersion = { newVersion: string; }; -/** Entries that {would need a dependency update in package.json to be deduplicated */ +/** Entries that would need a dependency update in package.json to be deduplicated */ type AnalyzeResultNewRange = { name: string; oldRange: string; @@ -118,13 +118,13 @@ export class Lockfile { continue; } - // Get rid of an signal any invalid ranges upfront + // Get rid of and signal any invalid ranges upfront const invalid = allEntries.filter(e => !semver.validRange(e.range)); result.invalidRanges.push( ...invalid.map(({ range }) => ({ name, range })), ); - // Grab all valid entries, if there isn't at least 2 different valid ones we're done + // Grab all valid entries, if there aren't at least 2 different valid ones we're done const entries = allEntries.filter(e => semver.validRange(e.range)); if (entries.length < 2) { continue; @@ -163,7 +163,7 @@ export class Lockfile { acceptedVersions.add(acceptedVersion); } - // If all ranges where able to accept the same version, we're done + // If all ranges were able to accept the same version, we're done if (acceptedVersions.size === 1) { continue; } @@ -229,7 +229,7 @@ export class Lockfile { } this.data[query] = matchingEntry[1]; - // Update out internal data structure + // Update our internal data structure const entry = this.packages.get(name)?.find(e => e.range === range); if (!entry) { throw new Error(`No entry data for ${query}`); diff --git a/packages/cli/src/lib/versioning/packages.test.ts b/packages/cli/src/lib/versioning/packages.test.ts index ba328fa356..fd37d7af95 100644 --- a/packages/cli/src/lib/versioning/packages.test.ts +++ b/packages/cli/src/lib/versioning/packages.test.ts @@ -47,7 +47,7 @@ describe('mapDependencies', () => { jest.resetAllMocks(); }); - it('should derp', async () => { + it('should read dependencies', async () => { // Make sure all modules involved in package discovery are in the module cache before we mock fs const LernaProject = require('@lerna/project'); const project = new LernaProject(paths.targetDir); From 0c597a8d62b448a152c951bdcee988a84657cba1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 22 Nov 2020 20:18:19 +0100 Subject: [PATCH 234/259] cli: work around flaky formatting in bump.ts --- packages/cli/src/commands/versions/bump.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts index 526ab9d4cf..eec8abed9b 100644 --- a/packages/cli/src/commands/versions/bump.ts +++ b/packages/cli/src/commands/versions/bump.ts @@ -157,6 +157,6 @@ async function workerThreads( return Promise.all( Array(count) .fill(0) - .map(pop), + .map(() => pop()), ); } From 5c8e99ce191b6e5fb9b14771fe5eb64cdbc6ad62 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 22 Nov 2020 20:38:57 -0500 Subject: [PATCH 235/259] Fix entity YAML name --- docs/features/software-catalog/descriptor-format.md | 2 +- plugins/circleci/README.md | 2 +- plugins/jenkins/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 5d7a5b9af0..c3adaabc45 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -43,7 +43,7 @@ software catalog API. "kind": "Component", "metadata": { "annotations": { - "backstage.io/managed-by-location": "file:/tmp/component-info.yaml", + "backstage.io/managed-by-location": "file:/tmp/catalog-info.yaml", "example.com/service-discovery": "artistweb", "circleci.com/project-slug": "github/example-org/artist-website" }, diff --git a/plugins/circleci/README.md b/plugins/circleci/README.md index 00a39ce6c8..c96de56617 100644 --- a/plugins/circleci/README.md +++ b/plugins/circleci/README.md @@ -61,7 +61,7 @@ proxy: ``` 5. Get and provide `CIRCLECI_AUTH_TOKEN` as env variable (https://circleci.com/docs/api/#add-an-api-token) -6. Add `circleci.com/project-slug` annotation to your component-info.yaml file in format // (https://backstage.io/docs/architecture-decisions/adrs-adr002#format) +6. Add `circleci.com/project-slug` annotation to your catalog-info.yaml file in format // (https://backstage.io/docs/architecture-decisions/adrs-adr002#format) ```yaml apiVersion: backstage.io/v1alpha1 diff --git a/plugins/jenkins/README.md b/plugins/jenkins/README.md index 1d3e7f60e5..5e0a701caf 100644 --- a/plugins/jenkins/README.md +++ b/plugins/jenkins/README.md @@ -41,7 +41,7 @@ export JENKINS_BASIC_AUTH_HEADER="Basic $HEADER" ``` 5. Run app with `yarn start` -6. Add the Jenkins folder annotation to your `component-info.yaml`, (note: currently this plugin only supports folders and Git SCM) +6. Add the Jenkins folder annotation to your `catalog-info.yaml`, (note: currently this plugin only supports folders and Git SCM) ```yaml apiVersion: backstage.io/v1alpha1 From da8ca6cd1c13e13390b36da8f9ae84b7d0396317 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 22 Nov 2020 20:51:05 -0500 Subject: [PATCH 236/259] Fix typo --- plugins/circleci/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/circleci/README.md b/plugins/circleci/README.md index c96de56617..4802e8bee2 100644 --- a/plugins/circleci/README.md +++ b/plugins/circleci/README.md @@ -88,4 +88,4 @@ spec: ## Limitations - CircleCI has pretty strict rate limits per token, be careful with opened tabs -- CircelCI doesn't provide a way to auth by 3rd party (e.g. GitHub) token, nor by calling their OAuth endpoints, which currently stands in the way of better auth integration with Backstage (https://discuss.circleci.com/t/circleci-api-authorization-with-github-token/5356) +- CircleCI doesn't provide a way to auth by 3rd party (e.g. GitHub) token, nor by calling their OAuth endpoints, which currently stands in the way of better auth integration with Backstage (https://discuss.circleci.com/t/circleci-api-authorization-with-github-token/5356) From fe3619782bbbbe2cc5d4b2a2b0fca731315d88da Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 22 Nov 2020 20:57:40 -0500 Subject: [PATCH 237/259] Change entity YAML name --- .../scaffolder-backend/src/scaffolder/stages/templater/cra.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts index da33149c24..0fcd8a4f15 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra.ts @@ -48,7 +48,7 @@ export class CreateReactAppTemplater implements TemplaterBase { }, }); - // Need to also make a component-info.yaml to store the data about the service. + // Need to also make a catalog-info.yaml to store the data about the service. const componentInfo = { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', @@ -69,7 +69,7 @@ export class CreateReactAppTemplater implements TemplaterBase { ); await fs.promises.writeFile( - `${finalDir}/component-info.yaml`, + `${finalDir}/catalog-info.yaml`, yaml.stringify(componentInfo), ); From 5a1d8dca38ffa26303e8a9ab2bf02b214bb1eda7 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 22 Nov 2020 21:01:08 -0500 Subject: [PATCH 238/259] Add changeset --- .changeset/tidy-brooms-look.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tidy-brooms-look.md diff --git a/.changeset/tidy-brooms-look.md b/.changeset/tidy-brooms-look.md new file mode 100644 index 0000000000..f2c2a9064e --- /dev/null +++ b/.changeset/tidy-brooms-look.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Fix React entity YAML filename to new standard From af45dc5320457f660b408dfe4b91f0a909de7c2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 06:01:33 +0000 Subject: [PATCH 239/259] build(deps-dev): bump eslint-plugin-cypress from 2.11.1 to 2.11.2 Bumps [eslint-plugin-cypress](https://github.com/cypress-io/eslint-plugin-cypress) from 2.11.1 to 2.11.2. - [Release notes](https://github.com/cypress-io/eslint-plugin-cypress/releases) - [Commits](https://github.com/cypress-io/eslint-plugin-cypress/compare/v2.11.1...v2.11.2) Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index bd1757cea0..bdf52a4bd5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11122,9 +11122,9 @@ eslint-module-utils@^2.1.1, eslint-module-utils@^2.6.0: pkg-dir "^2.0.0" eslint-plugin-cypress@^2.10.3: - version "2.11.1" - resolved "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-2.11.1.tgz#a945e2774b88211e2c706a059d431e262b5c2862" - integrity sha512-MxMYoReSO5+IZMGgpBZHHSx64zYPSPTpXDwsgW7ChlJTF/sA+obqRbHplxD6sBStE+g4Mi0LCLkG4t9liu//mQ== + version "2.11.2" + resolved "https://registry.npmjs.org/eslint-plugin-cypress/-/eslint-plugin-cypress-2.11.2.tgz#a8f3fe7ec840f55e4cea37671f93293e6c3e76a0" + integrity sha512-1SergF1sGbVhsf7MYfOLiBhdOg6wqyeV9pXUAIDIffYTGMN3dTBQS9nFAzhLsHhO+Bn0GaVM1Ecm71XUidQ7VA== dependencies: globals "^11.12.0" From 0e352e981ddcb3619c24360449b0488bae8fae22 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Mon, 23 Nov 2020 09:33:09 +0100 Subject: [PATCH 240/259] Make clicking on the item to execute search work --- packages/core/src/layout/Sidebar/Items.tsx | 35 ++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/core/src/layout/Sidebar/Items.tsx b/packages/core/src/layout/Sidebar/Items.tsx index b8786886cc..929828cee4 100644 --- a/packages/core/src/layout/Sidebar/Items.tsx +++ b/packages/core/src/layout/Sidebar/Items.tsx @@ -14,23 +14,23 @@ * limitations under the License. */ +import { IconComponent } from '@backstage/core-api'; +import { BackstageTheme } from '@backstage/theme'; import { + Badge, makeStyles, styled, TextField, Typography, - Badge, } from '@material-ui/core'; -import { BackstageTheme } from '@backstage/theme'; -import { IconComponent } from '@backstage/core-api'; import SearchIcon from '@material-ui/icons/Search'; import clsx from 'clsx'; import React, { + forwardRef, + KeyboardEventHandler, + ReactNode, useContext, useState, - KeyboardEventHandler, - forwardRef, - ReactNode, } from 'react'; import { NavLink } from 'react-router-dom'; import { sidebarConfig, SidebarContext } from './config'; @@ -120,7 +120,7 @@ type SidebarItemProps = { // If 'to' is set the item will act as a nav link with highlight, otherwise it's just a button to?: string; hasNotifications?: boolean; - onClick?: () => void; + onClick?: (ev: React.MouseEvent) => void; children?: ReactNode; }; @@ -218,10 +218,14 @@ export const SidebarSearchField = (props: SidebarSearchFieldProps) => { const [input, setInput] = useState(''); const classes = useStyles(); + const search = () => { + props.onSearch(input); + setInput(''); + }; + const handleEnter: KeyboardEventHandler = ev => { if (ev.key === 'Enter') { - props.onSearch(input); - setInput(''); + search(); } }; @@ -229,18 +233,25 @@ export const SidebarSearchField = (props: SidebarSearchFieldProps) => { setInput(ev.target.value); }; - const handleClick = (ev: React.MouseEvent) => { + const handleInputClick = (ev: React.MouseEvent) => { // Clicking into the search fields shouldn't navigate to the search page ev.preventDefault(); + ev.stopPropagation(); + }; + + const handleItemClick = (ev: React.MouseEvent) => { + // Clicking on the search icon while should execute a query with the current field content + search(); + ev.preventDefault(); }; return (
- + Date: Mon, 23 Nov 2020 11:25:45 +0100 Subject: [PATCH 241/259] backend-common: make test-utils a dev dep --- packages/backend-common/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index d4f0052d28..b22e4638f5 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -33,7 +33,6 @@ "@backstage/config": "^0.1.1", "@backstage/config-loader": "^0.3.0", "@backstage/integration": "^0.1.1", - "@backstage/test-utils": "^0.1.3", "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "archiver": "^5.0.2", @@ -69,6 +68,7 @@ }, "devDependencies": { "@backstage/cli": "^0.3.0", + "@backstage/test-utils": "^0.1.3", "@types/archiver": "^3.1.1", "@types/compression": "^1.7.0", "@types/concat-stream": "^1.6.0", From 31d8b6979d99f627d678a2cdedde06f32ea1e0c3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 23 Nov 2020 11:29:26 +0100 Subject: [PATCH 242/259] cli: add experimental backend:bundle command --- .changeset/orange-cherries-run.md | 5 +++ .gitignore | 1 + packages/cli/src/commands/backend/bundle.ts | 39 +++++++++++++++++++++ packages/cli/src/commands/index.ts | 5 +++ 4 files changed, 50 insertions(+) create mode 100644 .changeset/orange-cherries-run.md create mode 100644 packages/cli/src/commands/backend/bundle.ts diff --git a/.changeset/orange-cherries-run.md b/.changeset/orange-cherries-run.md new file mode 100644 index 0000000000..3c06c27b47 --- /dev/null +++ b/.changeset/orange-cherries-run.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Add experimental backend:bundle command diff --git a/.gitignore b/.gitignore index 5c27601791..3334bf956d 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ typings/ .nuxt dist dist-types +dist-workspace # Gatsby files .cache/ diff --git a/packages/cli/src/commands/backend/bundle.ts b/packages/cli/src/commands/backend/bundle.ts new file mode 100644 index 0000000000..e604ae9733 --- /dev/null +++ b/packages/cli/src/commands/backend/bundle.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Command } from 'commander'; +import fs from 'fs-extra'; +import { createDistWorkspace } from '../../lib/packager'; +import { paths } from '../../lib/paths'; +import { parseParallel, PARALLEL_ENV_VAR } from '../../lib/parallel'; + +const PKG_PATH = 'package.json'; +const TARGET_DIR = 'dist-workspace'; + +export default async (cmd: Command) => { + const targetDir = paths.resolveTarget(TARGET_DIR); + const pkgPath = paths.resolveTarget(PKG_PATH); + const pkg = await fs.readJson(pkgPath); + + await fs.remove(targetDir); + await fs.mkdir(targetDir); + await createDistWorkspace([pkg.name], { + targetDir: targetDir, + buildDependencies: Boolean(cmd.build), + parallel: parseParallel(process.env[PARALLEL_ENV_VAR]), + skeleton: 'skeleton.tar', + }); +}; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 1ebe518cac..bbd577d069 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -44,6 +44,11 @@ export function registerCommands(program: CommanderStatic) { .description('Build a backend plugin') .action(lazy(() => import('./backend/build').then(m => m.default))); + program + .command('backend:__experimental__bundle__', { hidden: true }) + .description('Bundle all backend packages into dist-workspace') + .action(lazy(() => import('./backend/bundle').then(m => m.default))); + program .command('backend:build-image') .allowUnknownOption(true) From 1185919f399ea9df8340daed9a679d35101a2550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 23 Nov 2020 11:29:37 +0100 Subject: [PATCH 243/259] catalog: mark ancestors and descendants for deprecation (#3392) --- .changeset/forty-buses-watch.md | 14 ++++++ .../software-catalog/descriptor-format.md | 10 ++++ .../src/kinds/GroupEntityV1alpha1.ts | 14 ++++++ .../BuiltinKindsEntityProcessor.test.ts | 47 +++++++++++++++++++ .../processors/BuiltinKindsEntityProcessor.ts | 19 ++++++++ 5 files changed, 104 insertions(+) create mode 100644 .changeset/forty-buses-watch.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts diff --git a/.changeset/forty-buses-watch.md b/.changeset/forty-buses-watch.md new file mode 100644 index 0000000000..4074566fe7 --- /dev/null +++ b/.changeset/forty-buses-watch.md @@ -0,0 +1,14 @@ +--- +'@backstage/catalog-model': patch +'@backstage/plugin-catalog-backend': patch +--- + +Marked the `Group` entity fields `ancestors` and `descendants` for deprecation on Dec 6th, 2020. See https://github.com/backstage/backstage/issues/3049 for details. + +Code that consumes these fields should remove those usages as soon as possible. There is no current or planned replacement for these fields. + +The BuiltinKindsEntityProcessor has been updated to inject these fields as empty arrays if they are missing. Therefore, if you are on a catalog instance that uses the updated version of this code, you can start removing the fields from your source catalog-info.yaml data as well, without breaking validation. + +After Dec 6th, the fields will be removed from types and classes of the Backstage repository. At the first release after that, they will not be present in released packages either. + +If your catalog-info.yaml files still contain these fields after the deletion, they will still be valid and your ingestion will not break, but they won't be visible in the types for consuming code. diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index c3adaabc45..f5cb361b81 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -700,6 +700,11 @@ sufficient to enter only the `metadata.name` field of that group. ### `spec.ancestors` [required] +**NOTE**: This field was marked for deprecation on Nov 22nd, 2020. It will be +removed entirely from the model on Dec 6th, 2020 in the repository and will not +be present in released packages following the next release after that. Please +update your code to not consume this field before the removal date. + The recursive list of parents up the hierarchy, by stepping through parents one by one. The list must be present, but may be empty if `parent` is not present. The first entry in the list is equal to `parent`, and then the following ones @@ -728,6 +733,11 @@ sufficient to enter only the `metadata.name` field of those groups. ### `spec.descendants` [required] +**NOTE**: This field was marked for deprecation on Nov 22nd, 2020. It will be +removed entirely from the model on Dec 6th, 2020 in the repository and will not +be present in released packages following the next release after that. Please +update your code to not consume this field before the removal date. + The immediate and recursive child groups of this group in the hierarchy (children, and children's children, etc.). The list must be present, but may be empty if there are no child groups. The items are not guaranteed to be ordered diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts index f6345741f7..7be432ad1f 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts @@ -57,8 +57,22 @@ export interface GroupEntityV1alpha1 extends Entity { spec: { type: string; parent?: string; + /** + * @deprecated This field will disappear on Dec 6th, 2020. Please remove + * any consuming code. Producers can stop producing this field + * before that date, as long as the catalog backend uses the + * BuiltinKindsEntityProcessor which inserts the fields in the + * mean time. + */ ancestors: string[]; children: string[]; + /** + * @deprecated This field will disappear on Dec 6th, 2020. Please remove + * any consuming code. Producers can stop producing this field + * before that date, as long as the catalog backend uses the + * BuiltinKindsEntityProcessor which inserts the fields in the + * mean time. + */ descendants: string[]; }; } diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts new file mode 100644 index 0000000000..e1328e827a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; + +describe('BuiltinKindsEntityProcessor', () => { + it('fills in fields for #3049', async () => { + const p = new BuiltinKindsEntityProcessor(); + const result = await p.preProcessEntity({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'n', + }, + spec: { + type: 't', + children: [], + } as any, + }); + expect(result).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'n', + }, + spec: { + type: 't', + children: [], + ancestors: [], + descendants: [], + }, + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts index 4d1eb5e583..472dffeb16 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts @@ -35,6 +35,25 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { userEntityV1alpha1Validator, ]; + async preProcessEntity(entity: Entity): Promise { + // NOTE(freben): Part of Group field deprecation on Nov 22nd, 2020. Fields + // scheduled for removal Dec 6th, 2020. This code can be deleted after that + // point. See https://github.com/backstage/backstage/issues/3049 + if ( + entity.apiVersion === 'backstage.io/v1alpha1' && + entity.kind === 'Group' && + entity.spec + ) { + if (!entity.spec.ancestors) { + entity.spec.ancestors = []; + } + if (!entity.spec.descendants) { + entity.spec.descendants = []; + } + } + return entity; + } + async validateEntityKind(entity: Entity): Promise { for (const validator of this.validators) { const result = await validator.check(entity); From da2f4cd41a3034aa26b8c598057ef308b40bde07 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Mon, 23 Nov 2020 12:08:46 +0100 Subject: [PATCH 244/259] Add graphql API type to descriptor format (#3395) --- docs/features/software-catalog/descriptor-format.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index f5cb361b81..60f1901807 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -592,6 +592,9 @@ The current set of well-known and common values for this field is: [OpenAPI](https://swagger.io/specification/) version 2 or version 3 spec. - `asyncapi` - An API definition based on the [AsyncAPI](https://www.asyncapi.com/docs/specifications/latest/) spec. +- `graphql` - An API definition based on + [GraphQL schemas](https://spec.graphql.org/) for consuming + [GraphQL](https://graphql.org/) based APIs. - `grpc` - An API definition based on [Protocol Buffers](https://developers.google.com/protocol-buffers) to use with [gRPC](https://grpc.io/). From e2085293a635a4c4e356f54a7bf213d0958b2447 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 19 Oct 2020 23:22:08 -0400 Subject: [PATCH 245/259] feat(kubernetes): update k8s-backend plugin to use label selectors update plugin to use label selectors when querying backend --- .../src/service/KubernetesFetcher.test.ts | 6 +- .../src/service/KubernetesFetcher.ts | 97 +++++++++---------- ...ubernetesObjectsByServiceIdHandler.test.ts | 10 +- ... getKubernetesObjectsForServiceHandler.ts} | 8 +- .../src/service/router.test.ts | 2 +- .../kubernetes-backend/src/service/router.ts | 12 ++- plugins/kubernetes-backend/src/types/types.ts | 3 +- plugins/kubernetes/README.md | 25 ++++- .../src/api/KubernetesBackendClient.ts | 18 +++- plugins/kubernetes/src/api/types.ts | 4 +- .../KubernetesContent/KubernetesContent.tsx | 21 +++- 11 files changed, 132 insertions(+), 74 deletions(-) rename plugins/kubernetes-backend/src/service/{getKubernetesObjectsByServiceIdHandler.ts => getKubernetesObjectsForServiceHandler.ts} (90%) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 6294b3a7ed..8c562b5200 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -57,7 +57,7 @@ describe('KubernetesClientProvider', () => { clientMock.listServiceForAllNamespaces.mockRejectedValue(errorResponse); - const result = await sut.fetchObjectsByServiceId( + const result = await sut.fetchObjectsForService( 'some-service', { name: 'cluster1', @@ -120,7 +120,7 @@ describe('KubernetesClientProvider', () => { }, }); - const result = await sut.fetchObjectsByServiceId( + const result = await sut.fetchObjectsForService( 'some-service', { name: 'cluster1', @@ -169,7 +169,7 @@ describe('KubernetesClientProvider', () => { }); it('should throw error on unknown type', () => { expect(() => - sut.fetchObjectsByServiceId( + sut.fetchObjectsForService( 'some-service', { name: 'cluster1', diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index f66a45816a..3bfb651d52 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -105,15 +105,19 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { this.logger = logger; } - fetchObjectsByServiceId( + fetchObjectsForService( serviceId: string, clusterDetails: ClusterDetails, objectTypesToFetch: Set, + labelSelector: string, ): Promise { const fetchResults = Array.from(objectTypesToFetch).map(type => { - return this.fetchByObjectType(serviceId, clusterDetails, type).catch( - captureKubernetesErrorsRethrowOthers, - ); + return this.fetchByObjectType( + serviceId, + clusterDetails, + type, + labelSelector, + ).catch(captureKubernetesErrorsRethrowOthers); }); return Promise.all(fetchResults).then(fetchResultsToResponseWrapper); @@ -124,42 +128,53 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { serviceId: string, clusterDetails: ClusterDetails, type: KubernetesObjectTypes, + labelSelector: string, ): Promise { switch (type) { case 'pods': - return this.fetchPodsByServiceId(serviceId, clusterDetails).then(r => ({ + return this.fetchPodsForService( + serviceId, + clusterDetails, + labelSelector, + ).then(r => ({ type: type, resources: r, })); case 'configmaps': - return this.fetchConfigMapsByServiceId( + return this.fetchConfigMapsForService( serviceId, clusterDetails, + labelSelector, ).then(r => ({ type: type, resources: r })); case 'deployments': - return this.fetchDeploymentsByServiceId( + return this.fetchDeploymentsForService( serviceId, clusterDetails, + labelSelector, ).then(r => ({ type: type, resources: r })); case 'replicasets': - return this.fetchReplicaSetsByServiceId( + return this.fetchReplicaSetsForService( serviceId, clusterDetails, + labelSelector, ).then(r => ({ type: type, resources: r })); case 'services': - return this.fetchServicesByServiceId( + return this.fetchServicesForService( serviceId, clusterDetails, + labelSelector, ).then(r => ({ type: type, resources: r })); case 'horizontalpodautoscalers': - return this.fetchHorizontalPodAutoscalersByServiceId( + return this.fetchHorizontalPodAutoscalersForService( serviceId, clusterDetails, + labelSelector, ).then(r => ({ type: type, resources: r })); case 'ingresses': - return this.fetchIngressesByServiceId( + return this.fetchIngressesForService( serviceId, clusterDetails, + labelSelector, ).then(r => ({ type: type, resources: r })); default: // unrecognised type @@ -192,79 +207,60 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { }); } - private fetchServicesByServiceId( + private fetchServicesForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch(clusterDetails, ({ core }) => - core.listServiceForAllNamespaces( - false, - '', - '', - `backstage.io/kubernetes-id=${serviceId}`, - ), + core.listServiceForAllNamespaces(false, '', '', labelSelector), ); } - private fetchPodsByServiceId( + private fetchPodsForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch(clusterDetails, ({ core }) => - core.listPodForAllNamespaces( - false, - '', - '', - `backstage.io/kubernetes-id=${serviceId}`, - ), + core.listPodForAllNamespaces(false, '', '', labelSelector), ); } - private fetchConfigMapsByServiceId( + private fetchConfigMapsForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch(clusterDetails, ({ core }) => - core.listConfigMapForAllNamespaces( - false, - '', - '', - `backstage.io/kubernetes-id=${serviceId}`, - ), + core.listConfigMapForAllNamespaces(false, '', '', labelSelector), ); } - private fetchDeploymentsByServiceId( + private fetchDeploymentsForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch(clusterDetails, ({ apps }) => - apps.listDeploymentForAllNamespaces( - false, - '', - '', - `backstage.io/kubernetes-id=${serviceId}`, - ), + apps.listDeploymentForAllNamespaces(false, '', '', labelSelector), ); } - private fetchReplicaSetsByServiceId( + private fetchReplicaSetsForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch(clusterDetails, ({ apps }) => - apps.listReplicaSetForAllNamespaces( - false, - '', - '', - `backstage.io/kubernetes-id=${serviceId}`, - ), + apps.listReplicaSetForAllNamespaces(false, '', '', labelSelector), ); } - private fetchHorizontalPodAutoscalersByServiceId( + private fetchHorizontalPodAutoscalersForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch( clusterDetails, @@ -273,14 +269,15 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { false, '', '', - `backstage.io/kubernetes-id=${serviceId}`, + labelSelector, ), ); } - private fetchIngressesByServiceId( + private fetchIngressesForService( serviceId: string, clusterDetails: ClusterDetails, + labelSelector: string, ): Promise> { return this.singleClusterFetch( clusterDetails, @@ -289,7 +286,7 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { false, '', '', - `backstage.io/kubernetes-id=${serviceId}`, + labelSelector, ), ); } diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index 8c26ded2d4..f413d080c6 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { handleGetKubernetesObjectsByServiceId } from './getKubernetesObjectsByServiceIdHandler'; +import { handleGetKubernetesObjectsForService } from './getKubernetesObjectsForServiceHandler'; import { getVoidLogger } from '@backstage/backend-common'; import { ClusterDetails } from '..'; @@ -81,10 +81,10 @@ describe('handleGetKubernetesObjectsByServiceId', () => { mockFetch(fetchObjectsByServiceId); - const result = await handleGetKubernetesObjectsByServiceId( + const result = await handleGetKubernetesObjectsForService( TEST_SERVICE_ID, { - fetchObjectsByServiceId, + fetchObjectsForService: fetchObjectsByServiceId, }, { getClustersByServiceId, @@ -155,10 +155,10 @@ describe('handleGetKubernetesObjectsByServiceId', () => { mockFetch(fetchObjectsByServiceId); - const result = await handleGetKubernetesObjectsByServiceId( + const result = await handleGetKubernetesObjectsForService( TEST_SERVICE_ID, { - fetchObjectsByServiceId, + fetchObjectsForService: fetchObjectsByServiceId, }, { getClustersByServiceId, diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts similarity index 90% rename from plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.ts rename to plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts index 6fc6092bd6..5f31248790 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts @@ -26,12 +26,13 @@ import { import { KubernetesAuthTranslator } from '../kubernetes-auth-translator/types'; import { KubernetesAuthTranslatorGenerator } from '../kubernetes-auth-translator/KubernetesAuthTranslatorGenerator'; -export type GetKubernetesObjectsByServiceIdHandler = ( +export type GetKubernetesObjectsForServiceHandler = ( serviceId: string, fetcher: KubernetesFetcher, serviceLocator: KubernetesServiceLocator, logger: Logger, requestBody: AuthRequestBody, + labelSelector: string, objectsToFetch?: Set, ) => Promise; @@ -46,12 +47,13 @@ const DEFAULT_OBJECTS = new Set([ ]); // Fans out the request to all clusters that the service lives in, aggregates their responses together -export const handleGetKubernetesObjectsByServiceId: GetKubernetesObjectsByServiceIdHandler = async ( +export const handleGetKubernetesObjectsForService: GetKubernetesObjectsForServiceHandler = async ( serviceId, fetcher, serviceLocator, logger, requestBody, + labelSelector: string, objectsToFetch = DEFAULT_OBJECTS, ) => { const clusterDetails: ClusterDetails[] = await serviceLocator.getClustersByServiceId( @@ -81,7 +83,7 @@ export const handleGetKubernetesObjectsByServiceId: GetKubernetesObjectsByServic return Promise.all( clusterDetailsDecoratedForAuth.map(cd => { return fetcher - .fetchObjectsByServiceId(serviceId, cd, objectsToFetch) + .fetchObjectsForService(serviceId, cd, objectsToFetch, labelSelector) .then(result => { return { cluster: { diff --git a/plugins/kubernetes-backend/src/service/router.test.ts b/plugins/kubernetes-backend/src/service/router.test.ts index 6f9fbbf70f..b63b63ead6 100644 --- a/plugins/kubernetes-backend/src/service/router.test.ts +++ b/plugins/kubernetes-backend/src/service/router.test.ts @@ -32,7 +32,7 @@ describe('router', () => { beforeAll(async () => { kubernetesFetcher = { - fetchObjectsByServiceId: jest.fn(), + fetchObjectsForService: jest.fn(), }; kubernetesServiceLocator = { diff --git a/plugins/kubernetes-backend/src/service/router.ts b/plugins/kubernetes-backend/src/service/router.ts index b73115e1e4..5462b5c9fb 100644 --- a/plugins/kubernetes-backend/src/service/router.ts +++ b/plugins/kubernetes-backend/src/service/router.ts @@ -22,9 +22,9 @@ import { MultiTenantServiceLocator } from '../service-locator/MultiTenantService import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; import { KubernetesClientProvider } from './KubernetesClientProvider'; import { - GetKubernetesObjectsByServiceIdHandler, - handleGetKubernetesObjectsByServiceId, -} from './getKubernetesObjectsByServiceIdHandler'; + GetKubernetesObjectsForServiceHandler, + handleGetKubernetesObjectsForService, +} from './getKubernetesObjectsForServiceHandler'; import { AuthRequestBody, KubernetesServiceLocator, @@ -64,13 +64,14 @@ export const makeRouter = ( logger: Logger, fetcher: KubernetesFetcher, serviceLocator: KubernetesServiceLocator, - handleGetByServiceId: GetKubernetesObjectsByServiceIdHandler, + handleGetByServiceId: GetKubernetesObjectsForServiceHandler, ): express.Router => { const router = Router(); router.use(express.json()); router.post('/services/:serviceId', async (req, res) => { const serviceId = req.params.serviceId; + const labelSelector = req.query.labelSelector; const requestBody: AuthRequestBody = req.body; try { const response = await handleGetByServiceId( @@ -79,6 +80,7 @@ export const makeRouter = ( serviceLocator, logger, requestBody, + labelSelector, ); res.send(response); } catch (e) { @@ -119,6 +121,6 @@ export async function createRouter( logger, fetcher, serviceLocator, - handleGetKubernetesObjectsByServiceId, + handleGetKubernetesObjectsForService, ); } diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index 1422511c21..0425687f48 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -110,10 +110,11 @@ export interface IngressesFetchResponse { // Fetches information from a kubernetes cluster using the cluster details object // to target a specific cluster export interface KubernetesFetcher { - fetchObjectsByServiceId( + fetchObjectsForService( serviceId: string, clusterDetails: ClusterDetails, objectTypesToFetch: Set, + labelSelector: string, ): Promise; } diff --git a/plugins/kubernetes/README.md b/plugins/kubernetes/README.md index 1ad478bd89..e6f1f2355a 100644 --- a/plugins/kubernetes/README.md +++ b/plugins/kubernetes/README.md @@ -14,7 +14,12 @@ It is only meant for local development, and the setup for it can be found inside ## Surfacing your Kubernetes components as part of an entity -### Adding the entity annotation +There are 2 ways to surface your kubernetes components as part of an entity. +The label selector takes precedence over the annotation/service id. + +### Common `backstage.io/kubernetes-id` label + +#### Adding the entity annotation In order for Backstage to detect that an entity has Kubernetes components, the following annotation should be added to the entity. @@ -24,7 +29,7 @@ annotations: 'backstage.io/kubernetes-id': dice-roller ``` -### Labeling Kubernetes components +#### Labeling Kubernetes components In order for Kubernetes components to show up in the service catalog as a part of an entity, Kubernetes components must be labeled with the following label: @@ -32,3 +37,19 @@ as a part of an entity, Kubernetes components must be labeled with the following ```yaml 'backstage.io/kubernetes-id': ``` + +### Full label selector + +#### Adding the entity label selector to the spec + +In order for Backstage to detect that an entity has kubernetes components the `kubernetes.selector` must have a valid selector. (Currently only matchLabels is supported) + +```yaml +spec: + kubernetes: + selector: + matchLabels: + someKey: someValue + other-key: other-value + app.kubernetes.io/name: dice-roller +``` diff --git a/plugins/kubernetes/src/api/KubernetesBackendClient.ts b/plugins/kubernetes/src/api/KubernetesBackendClient.ts index 86bb830046..faf9aaf024 100644 --- a/plugins/kubernetes/src/api/KubernetesBackendClient.ts +++ b/plugins/kubernetes/src/api/KubernetesBackendClient.ts @@ -20,6 +20,7 @@ import { AuthRequestBody, ObjectsByServiceIdResponse, } from '@backstage/plugin-kubernetes-backend'; +import { V1LabelSelector } from '@kubernetes/client-node'; export class KubernetesBackendClient implements KubernetesApi { private readonly discoveryApi: DiscoveryApi; @@ -50,10 +51,23 @@ export class KubernetesBackendClient implements KubernetesApi { return await response.json(); } - async getObjectsByServiceId( + private parseLabelSelector(params: V1LabelSelector): string { + // TODO: figure out how to convert the selector to the full query param from the yaml + // (as shown here https://github.com/kubernetes/apimachinery/blob/master/pkg/labels/selector.go) + return Object.keys(params.matchLabels) + .map(key => `${key}=${params[key]}`) + .join(','); + } + + async getObjectsByLabelSelector( serviceId: String, + labelSelector: V1LabelSelector, requestBody: AuthRequestBody, ): Promise { - return await this.getRequired(`/services/${serviceId}`, requestBody); + const labelSelectorQueryParams = this.parseLabelSelector(labelSelector); + return await this.getRequired( + `/services/${serviceId}?labelSelector=${labelSelectorQueryParams}`, + requestBody, + ); } } diff --git a/plugins/kubernetes/src/api/types.ts b/plugins/kubernetes/src/api/types.ts index 8e44d58e6a..2f13dcb02b 100644 --- a/plugins/kubernetes/src/api/types.ts +++ b/plugins/kubernetes/src/api/types.ts @@ -19,6 +19,7 @@ import { AuthRequestBody, ObjectsByServiceIdResponse, } from '@backstage/plugin-kubernetes-backend'; +import { V1LabelSelector } from '@kubernetes/client-node'; export const kubernetesApiRef = createApiRef({ id: 'plugin.kubernetes.service', @@ -27,8 +28,9 @@ export const kubernetesApiRef = createApiRef({ }); export interface KubernetesApi { - getObjectsByServiceId( + getObjectsByLabelSelector( serviceId: String, + labelSelector: V1LabelSelector, requestBody: AuthRequestBody, ): Promise; } diff --git a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx index a3e82e79a4..d46a888b39 100644 --- a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx @@ -41,6 +41,7 @@ import { ExtensionsV1beta1Ingress, V1ConfigMap, V1HorizontalPodAutoscaler, + V1LabelSelector, V1Service, } from '@kubernetes/client-node'; import { Services } from '../Services'; @@ -129,9 +130,27 @@ export const KubernetesContent = ({ entity }: KubernetesContentProps) => { ); } + // decide label selector to search by defaulting to this label + let labelSelector: V1LabelSelector = { + matchLabels: { + 'backstage.io/kubernetes-id': entity.metadata.name, + }, + }; + + if ( + entity.spec.kubernetes && + (entity.spec.kubernetes.selector as V1LabelSelector) + ) { + labelSelector = entity.spec.kubernetes.selector; + } + // TODO: Add validation on contents/format of requestBody kubernetesApi - .getObjectsByServiceId(entity.metadata.name, requestBody) + .getObjectsByLabelSelector( + entity.metadata.name, + labelSelector, + requestBody, + ) .then(result => { setKubernetesObjects(result); }) From beaa0a82dbd267193fe7877cf806e977a8b1241e Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Tue, 20 Oct 2020 00:22:44 -0400 Subject: [PATCH 246/259] fix tests, types and cleanup --- .../src/service/KubernetesFetcher.test.ts | 45 +++++++++++++++++++ .../src/service/KubernetesFetcher.ts | 4 +- ...ubernetesObjectsByServiceIdHandler.test.ts | 18 ++++---- .../kubernetes-backend/src/service/router.ts | 2 +- .../src/api/KubernetesBackendClient.ts | 5 ++- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index 8c562b5200..f37a81f9f8 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -66,6 +66,7 @@ describe('KubernetesClientProvider', () => { authProvider: 'serviceAccount', }, new Set(['pods', 'services']), + '', ); expect(result).toStrictEqual({ @@ -129,6 +130,7 @@ describe('KubernetesClientProvider', () => { authProvider: 'serviceAccount', }, new Set(['pods', 'services']), + '', ); expect(result).toStrictEqual({ @@ -178,6 +180,7 @@ describe('KubernetesClientProvider', () => { authProvider: 'serviceAccount', }, new Set(['foo']), + '', ), ).toThrow('unrecognised type=foo'); @@ -254,4 +257,46 @@ describe('KubernetesClientProvider', () => { }, ); }); + it('should always add a labelSelector query', async () => { + clientMock.listPodForAllNamespaces.mockResolvedValueOnce({ + body: { + items: [ + { + metadata: { + name: 'pod-name', + }, + }, + ], + }, + }); + + clientMock.listServiceForAllNamespaces.mockResolvedValueOnce({ + body: { + items: [ + { + metadata: { + name: 'service-name', + }, + }, + ], + }, + }); + + await sut.fetchObjectsForService( + 'some-service', + { + name: 'cluster1', + url: 'http://localhost:9999', + serviceAccountToken: 'token', + authProvider: 'serviceAccount', + }, + new Set(['pods', 'services']), + '', + ); + + const mockCall = clientMock.listPodForAllNamespaces.mock.calls[0]; + const actualSelector = mockCall[mockCall.length - 1]; + const expectedSelector = 'backstage.io/kubernetes-id=some-service'; + expect(actualSelector).toBe(expectedSelector); + }); }); diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index 3bfb651d52..a1ed21552b 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -116,7 +116,9 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { serviceId, clusterDetails, type, - labelSelector, + labelSelector.length !== 0 + ? labelSelector + : `backstage.io/kubernetes-id=${serviceId}`, ).catch(captureKubernetesErrorsRethrowOthers); }); diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index f413d080c6..2f779adfe8 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -20,7 +20,7 @@ import { ClusterDetails } from '..'; const TEST_SERVICE_ID = 'my-service'; -const fetchObjectsByServiceId = jest.fn(); +const fetchObjectsForService = jest.fn(); const getClustersByServiceId = jest.fn(); @@ -64,7 +64,7 @@ const mockFetch = (mock: jest.Mock) => { ); }; -describe('handleGetKubernetesObjectsByServiceId', () => { +describe('handleGetKubernetesObjectsForService', () => { beforeEach(() => { jest.resetAllMocks(); }); @@ -79,22 +79,23 @@ describe('handleGetKubernetesObjectsByServiceId', () => { ]), ); - mockFetch(fetchObjectsByServiceId); + mockFetch(fetchObjectsForService); const result = await handleGetKubernetesObjectsForService( TEST_SERVICE_ID, { - fetchObjectsForService: fetchObjectsByServiceId, + fetchObjectsForService: fetchObjectsForService, }, { getClustersByServiceId, }, getVoidLogger(), {}, + '', ); expect(getClustersByServiceId.mock.calls.length).toBe(1); - expect(fetchObjectsByServiceId.mock.calls.length).toBe(1); + expect(fetchObjectsForService.mock.calls.length).toBe(1); expect(result).toStrictEqual({ items: [ { @@ -153,12 +154,12 @@ describe('handleGetKubernetesObjectsByServiceId', () => { ]), ); - mockFetch(fetchObjectsByServiceId); + mockFetch(fetchObjectsForService); const result = await handleGetKubernetesObjectsForService( TEST_SERVICE_ID, { - fetchObjectsForService: fetchObjectsByServiceId, + fetchObjectsForService: fetchObjectsForService, }, { getClustersByServiceId, @@ -169,10 +170,11 @@ describe('handleGetKubernetesObjectsByServiceId', () => { google: 'google_token_123', }, }, + '', ); expect(getClustersByServiceId.mock.calls.length).toBe(1); - expect(fetchObjectsByServiceId.mock.calls.length).toBe(2); + expect(fetchObjectsForService.mock.calls.length).toBe(2); expect(result).toStrictEqual({ items: [ { diff --git a/plugins/kubernetes-backend/src/service/router.ts b/plugins/kubernetes-backend/src/service/router.ts index 5462b5c9fb..fada4c7888 100644 --- a/plugins/kubernetes-backend/src/service/router.ts +++ b/plugins/kubernetes-backend/src/service/router.ts @@ -80,7 +80,7 @@ export const makeRouter = ( serviceLocator, logger, requestBody, - labelSelector, + labelSelector ? labelSelector.toString() : '', ); res.send(response); } catch (e) { diff --git a/plugins/kubernetes/src/api/KubernetesBackendClient.ts b/plugins/kubernetes/src/api/KubernetesBackendClient.ts index faf9aaf024..c6d0153538 100644 --- a/plugins/kubernetes/src/api/KubernetesBackendClient.ts +++ b/plugins/kubernetes/src/api/KubernetesBackendClient.ts @@ -54,8 +54,9 @@ export class KubernetesBackendClient implements KubernetesApi { private parseLabelSelector(params: V1LabelSelector): string { // TODO: figure out how to convert the selector to the full query param from the yaml // (as shown here https://github.com/kubernetes/apimachinery/blob/master/pkg/labels/selector.go) - return Object.keys(params.matchLabels) - .map(key => `${key}=${params[key]}`) + const { matchLabels } = params; + return Object.keys(matchLabels) + .map(key => `${key}=${matchLabels[key]}`) .join(','); } From 61a552fac6133af81be6ee1d59aeec29240ef53e Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Tue, 20 Oct 2020 00:49:23 -0400 Subject: [PATCH 247/259] fix type checks --- .../src/service/KubernetesFetcher.ts | 29 ++++--------------- .../src/api/KubernetesBackendClient.ts | 5 +++- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index a1ed21552b..11917efc52 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -113,7 +113,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { ): Promise { const fetchResults = Array.from(objectTypesToFetch).map(type => { return this.fetchByObjectType( - serviceId, clusterDetails, type, labelSelector.length !== 0 @@ -127,54 +126,45 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { // TODO could probably do with a tidy up private fetchByObjectType( - serviceId: string, clusterDetails: ClusterDetails, type: KubernetesObjectTypes, labelSelector: string, ): Promise { switch (type) { case 'pods': - return this.fetchPodsForService( - serviceId, - clusterDetails, - labelSelector, - ).then(r => ({ - type: type, - resources: r, - })); + return this.fetchPodsForService(clusterDetails, labelSelector).then( + r => ({ + type: type, + resources: r, + }), + ); case 'configmaps': return this.fetchConfigMapsForService( - serviceId, clusterDetails, labelSelector, ).then(r => ({ type: type, resources: r })); case 'deployments': return this.fetchDeploymentsForService( - serviceId, clusterDetails, labelSelector, ).then(r => ({ type: type, resources: r })); case 'replicasets': return this.fetchReplicaSetsForService( - serviceId, clusterDetails, labelSelector, ).then(r => ({ type: type, resources: r })); case 'services': return this.fetchServicesForService( - serviceId, clusterDetails, labelSelector, ).then(r => ({ type: type, resources: r })); case 'horizontalpodautoscalers': return this.fetchHorizontalPodAutoscalersForService( - serviceId, clusterDetails, labelSelector, ).then(r => ({ type: type, resources: r })); case 'ingresses': return this.fetchIngressesForService( - serviceId, clusterDetails, labelSelector, ).then(r => ({ type: type, resources: r })); @@ -210,7 +200,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchServicesForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { @@ -220,7 +209,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchPodsForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { @@ -230,7 +218,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchConfigMapsForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { @@ -240,7 +227,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchDeploymentsForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { @@ -250,7 +236,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchReplicaSetsForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { @@ -260,7 +245,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchHorizontalPodAutoscalersForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { @@ -277,7 +261,6 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } private fetchIngressesForService( - serviceId: string, clusterDetails: ClusterDetails, labelSelector: string, ): Promise> { diff --git a/plugins/kubernetes/src/api/KubernetesBackendClient.ts b/plugins/kubernetes/src/api/KubernetesBackendClient.ts index c6d0153538..088196b0c8 100644 --- a/plugins/kubernetes/src/api/KubernetesBackendClient.ts +++ b/plugins/kubernetes/src/api/KubernetesBackendClient.ts @@ -55,8 +55,11 @@ export class KubernetesBackendClient implements KubernetesApi { // TODO: figure out how to convert the selector to the full query param from the yaml // (as shown here https://github.com/kubernetes/apimachinery/blob/master/pkg/labels/selector.go) const { matchLabels } = params; + if (!matchLabels) { + return ''; + } return Object.keys(matchLabels) - .map(key => `${key}=${matchLabels[key]}`) + .map(key => `${key}=${matchLabels[key.toString()]}`) .join(','); } From 99fffed216b5f2f0030627b45cea7c20e9c72fbd Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Sat, 7 Nov 2020 19:44:19 -0500 Subject: [PATCH 248/259] fix types --- .../src/kinds/ComponentEntityV1alpha1.ts | 16 ++++++++++++++++ .../KubernetesContent/KubernetesContent.tsx | 10 ++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts index 9d46dea03f..0e1e369e44 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts @@ -30,6 +30,15 @@ const schema = yup.object>({ lifecycle: yup.string().required().min(1), owner: yup.string().required().min(1), implementsApis: yup.array(yup.string().required()).notRequired(), + kubernetes: yup + .object({ + selector: yup + .object({ + matchLabels: yup.object().required(), + }) + .required(), + }) + .notRequired(), }) .required(), }); @@ -42,6 +51,13 @@ export interface ComponentEntityV1alpha1 extends Entity { lifecycle: string; owner: string; implementsApis?: string[]; + kubernetes?: { + selector: { + matchLabels: { + [key: string]: string; + }; + }; + }; }; } diff --git a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx index d46a888b39..3f9c1a866f 100644 --- a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx @@ -26,7 +26,7 @@ import { TabbedCard, useApi, } from '@backstage/core'; -import { Entity } from '@backstage/catalog-model'; +import { ComponentEntityV1alpha1, Entity } from '@backstage/catalog-model'; import { kubernetesApiRef } from '../../api/types'; import { AuthRequestBody, @@ -137,11 +137,13 @@ export const KubernetesContent = ({ entity }: KubernetesContentProps) => { }, }; + const componentEntity = entity as ComponentEntityV1alpha1; if ( - entity.spec.kubernetes && - (entity.spec.kubernetes.selector as V1LabelSelector) + componentEntity.spec && + componentEntity.spec.kubernetes && + (componentEntity.spec.kubernetes.selector as V1LabelSelector) ) { - labelSelector = entity.spec.kubernetes.selector; + labelSelector = componentEntity.spec.kubernetes.selector; } // TODO: Add validation on contents/format of requestBody From 4639d20720b0ba3c4fa48d0f1fbc2b417b90e6bf Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 01:42:08 -0500 Subject: [PATCH 249/259] modify approach to post entire entity to k8s backend --- .../GoogleKubernetesAuthTranslator.ts | 4 +-- .../ServiceAccountKubernetesAuthTranslator.ts | 4 +-- .../src/kubernetes-auth-translator/types.ts | 4 +-- ...ubernetesObjectsByServiceIdHandler.test.ts | 1 - .../getKubernetesObjectsForServiceHandler.ts | 33 +++++++++++++++---- .../src/service/router.test.ts | 4 +-- .../kubernetes-backend/src/service/router.ts | 10 +++--- plugins/kubernetes-backend/src/types/types.ts | 6 ++-- .../src/api/KubernetesBackendClient.ts | 28 ++++------------ plugins/kubernetes/src/api/types.ts | 10 +++--- .../KubernetesContent/KubernetesContent.tsx | 32 ++++-------------- .../GoogleKubernetesAuthProvider.ts | 6 ++-- .../KubernetesAuthProviders.ts | 6 ++-- .../ServiceAccountKubernetesAuthProvider.ts | 6 ++-- .../src/kubernetes-auth-provider/types.ts | 10 +++--- 15 files changed, 74 insertions(+), 90 deletions(-) diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/GoogleKubernetesAuthTranslator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/GoogleKubernetesAuthTranslator.ts index 7f9b2f0bae..0dd2a4bcc2 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/GoogleKubernetesAuthTranslator.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/GoogleKubernetesAuthTranslator.ts @@ -15,13 +15,13 @@ */ import { KubernetesAuthTranslator } from './types'; -import { AuthRequestBody, ClusterDetails } from '../types/types'; +import { KubernetesRequestBody, ClusterDetails } from '../types/types'; export class GoogleKubernetesAuthTranslator implements KubernetesAuthTranslator { async decorateClusterDetailsWithAuth( clusterDetails: ClusterDetails, - requestBody: AuthRequestBody, + requestBody: KubernetesRequestBody, ): Promise { const clusterDetailsWithAuthToken: ClusterDetails = Object.assign( {}, diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/ServiceAccountKubernetesAuthTranslator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/ServiceAccountKubernetesAuthTranslator.ts index ecf2f12b72..6433e41546 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/ServiceAccountKubernetesAuthTranslator.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/ServiceAccountKubernetesAuthTranslator.ts @@ -15,7 +15,7 @@ */ import { KubernetesAuthTranslator } from './types'; -import { AuthRequestBody, ClusterDetails } from '../types/types'; +import { KubernetesRequestBody, ClusterDetails } from '../types/types'; export class ServiceAccountKubernetesAuthTranslator implements KubernetesAuthTranslator { @@ -23,7 +23,7 @@ export class ServiceAccountKubernetesAuthTranslator clusterDetails: ClusterDetails, // To ignore TS6133 linting error where it detects 'requestBody' is declared but its value is never read. // @ts-ignore-start - requestBody: AuthRequestBody, // eslint-disable-line @typescript-eslint/no-unused-vars + requestBody: KubernetesRequestBody, // eslint-disable-line @typescript-eslint/no-unused-vars // @ts-ignore-end ): Promise { return clusterDetails; diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/types.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/types.ts index f89e04456f..c01a57889c 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/types.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/types.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { AuthRequestBody, ClusterDetails } from '../types/types'; +import { KubernetesRequestBody, ClusterDetails } from '../types/types'; export interface KubernetesAuthTranslator { decorateClusterDetailsWithAuth( clusterDetails: ClusterDetails, - requestBody: AuthRequestBody, + requestBody: KubernetesRequestBody, ): Promise; } diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index 2f779adfe8..f95b9a7242 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -91,7 +91,6 @@ describe('handleGetKubernetesObjectsForService', () => { }, getVoidLogger(), {}, - '', ); expect(getClustersByServiceId.mock.calls.length).toBe(1); diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts index 5f31248790..46f3f27c73 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts @@ -16,25 +16,25 @@ import { Logger } from 'winston'; import { - AuthRequestBody, + KubernetesRequestBody, ClusterDetails, KubernetesServiceLocator, KubernetesFetcher, KubernetesObjectTypes, - ObjectsByServiceIdResponse, + ObjectsByEntityResponse, } from '../types/types'; import { KubernetesAuthTranslator } from '../kubernetes-auth-translator/types'; import { KubernetesAuthTranslatorGenerator } from '../kubernetes-auth-translator/KubernetesAuthTranslatorGenerator'; +import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; export type GetKubernetesObjectsForServiceHandler = ( serviceId: string, fetcher: KubernetesFetcher, serviceLocator: KubernetesServiceLocator, logger: Logger, - requestBody: AuthRequestBody, - labelSelector: string, + requestBody: KubernetesRequestBody, objectsToFetch?: Set, -) => Promise; +) => Promise; const DEFAULT_OBJECTS = new Set([ 'pods', @@ -46,6 +46,26 @@ const DEFAULT_OBJECTS = new Set([ 'ingresses', ]); +function parseLabelSelector(entity: ComponentEntityV1alpha1): string { + if ( + entity && + entity.spec && + entity.spec.kubernetes && + entity.spec.kubernetes.selector + ) { + // TODO: figure out how to convert the selector to the full query param from the yaml + // (as shown here https://github.com/kubernetes/apimachinery/blob/master/pkg/labels/selector.go) + const { matchLabels } = entity.spec.kubernetes.selector; + if (!matchLabels) { + return ''; + } + return Object.keys(matchLabels) + .map(key => `${key}=${matchLabels[key.toString()]}`) + .join(','); + } + return ''; +} + // Fans out the request to all clusters that the service lives in, aggregates their responses together export const handleGetKubernetesObjectsForService: GetKubernetesObjectsForServiceHandler = async ( serviceId, @@ -53,7 +73,6 @@ export const handleGetKubernetesObjectsForService: GetKubernetesObjectsForServic serviceLocator, logger, requestBody, - labelSelector: string, objectsToFetch = DEFAULT_OBJECTS, ) => { const clusterDetails: ClusterDetails[] = await serviceLocator.getClustersByServiceId( @@ -80,6 +99,8 @@ export const handleGetKubernetesObjectsForService: GetKubernetesObjectsForServic .join(', ')}]`, ); + const labelSelector = parseLabelSelector(requestBody.entity); + return Promise.all( clusterDetailsDecoratedForAuth.map(cd => { return fetcher diff --git a/plugins/kubernetes-backend/src/service/router.test.ts b/plugins/kubernetes-backend/src/service/router.test.ts index b63b63ead6..a407ce9693 100644 --- a/plugins/kubernetes-backend/src/service/router.test.ts +++ b/plugins/kubernetes-backend/src/service/router.test.ts @@ -21,14 +21,14 @@ import { makeRouter } from './router'; import { KubernetesServiceLocator, KubernetesFetcher, - ObjectsByServiceIdResponse, + ObjectsByEntityResponse, } from '..'; describe('router', () => { let app: express.Express; let kubernetesFetcher: jest.Mocked; let kubernetesServiceLocator: jest.Mocked; - let handleGetByServiceId: jest.Mock>; + let handleGetByServiceId: jest.Mock>; beforeAll(async () => { kubernetesFetcher = { diff --git a/plugins/kubernetes-backend/src/service/router.ts b/plugins/kubernetes-backend/src/service/router.ts index fada4c7888..38a02ad970 100644 --- a/plugins/kubernetes-backend/src/service/router.ts +++ b/plugins/kubernetes-backend/src/service/router.ts @@ -26,7 +26,7 @@ import { handleGetKubernetesObjectsForService, } from './getKubernetesObjectsForServiceHandler'; import { - AuthRequestBody, + KubernetesRequestBody, KubernetesServiceLocator, KubernetesFetcher, ServiceLocatorMethod, @@ -64,23 +64,21 @@ export const makeRouter = ( logger: Logger, fetcher: KubernetesFetcher, serviceLocator: KubernetesServiceLocator, - handleGetByServiceId: GetKubernetesObjectsForServiceHandler, + handleGetByEntity: GetKubernetesObjectsForServiceHandler, ): express.Router => { const router = Router(); router.use(express.json()); router.post('/services/:serviceId', async (req, res) => { const serviceId = req.params.serviceId; - const labelSelector = req.query.labelSelector; - const requestBody: AuthRequestBody = req.body; + const requestBody: KubernetesRequestBody = req.body; try { - const response = await handleGetByServiceId( + const response = await handleGetByEntity( serviceId, fetcher, serviceLocator, logger, requestBody, - labelSelector ? labelSelector.toString() : '', ); res.send(response); } catch (e) { diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index 0425687f48..738bc89b21 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -23,6 +23,7 @@ import { V1ReplicaSet, V1Service, } from '@kubernetes/client-node'; +import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; export interface ClusterDetails { name: string; @@ -31,10 +32,11 @@ export interface ClusterDetails { serviceAccountToken?: string | undefined; } -export interface AuthRequestBody { +export interface KubernetesRequestBody { auth?: { google?: string; }; + entity: ComponentEntityV1alpha1; } export interface ClusterObjects { @@ -43,7 +45,7 @@ export interface ClusterObjects { errors: KubernetesFetchError[]; } -export interface ObjectsByServiceIdResponse { +export interface ObjectsByEntityResponse { items: ClusterObjects[]; } diff --git a/plugins/kubernetes/src/api/KubernetesBackendClient.ts b/plugins/kubernetes/src/api/KubernetesBackendClient.ts index 088196b0c8..d8fa800ab6 100644 --- a/plugins/kubernetes/src/api/KubernetesBackendClient.ts +++ b/plugins/kubernetes/src/api/KubernetesBackendClient.ts @@ -17,10 +17,9 @@ import { DiscoveryApi } from '@backstage/core'; import { KubernetesApi } from './types'; import { - AuthRequestBody, - ObjectsByServiceIdResponse, + KubernetesRequestBody, + ObjectsByEntityResponse, } from '@backstage/plugin-kubernetes-backend'; -import { V1LabelSelector } from '@kubernetes/client-node'; export class KubernetesBackendClient implements KubernetesApi { private readonly discoveryApi: DiscoveryApi; @@ -31,7 +30,7 @@ export class KubernetesBackendClient implements KubernetesApi { private async getRequired( path: string, - requestBody: AuthRequestBody, + requestBody: KubernetesRequestBody, ): Promise { const url = `${await this.discoveryApi.getBaseUrl('kubernetes')}${path}`; const response = await fetch(url, { @@ -51,26 +50,11 @@ export class KubernetesBackendClient implements KubernetesApi { return await response.json(); } - private parseLabelSelector(params: V1LabelSelector): string { - // TODO: figure out how to convert the selector to the full query param from the yaml - // (as shown here https://github.com/kubernetes/apimachinery/blob/master/pkg/labels/selector.go) - const { matchLabels } = params; - if (!matchLabels) { - return ''; - } - return Object.keys(matchLabels) - .map(key => `${key}=${matchLabels[key.toString()]}`) - .join(','); - } - async getObjectsByLabelSelector( - serviceId: String, - labelSelector: V1LabelSelector, - requestBody: AuthRequestBody, - ): Promise { - const labelSelectorQueryParams = this.parseLabelSelector(labelSelector); + requestBody: KubernetesRequestBody, + ): Promise { return await this.getRequired( - `/services/${serviceId}?labelSelector=${labelSelectorQueryParams}`, + `/services/${requestBody.entity.metadata.name}`, requestBody, ); } diff --git a/plugins/kubernetes/src/api/types.ts b/plugins/kubernetes/src/api/types.ts index 2f13dcb02b..663841ddf9 100644 --- a/plugins/kubernetes/src/api/types.ts +++ b/plugins/kubernetes/src/api/types.ts @@ -16,8 +16,8 @@ import { createApiRef } from '@backstage/core'; import { - AuthRequestBody, - ObjectsByServiceIdResponse, + KubernetesRequestBody, + ObjectsByEntityResponse, } from '@backstage/plugin-kubernetes-backend'; import { V1LabelSelector } from '@kubernetes/client-node'; @@ -29,8 +29,6 @@ export const kubernetesApiRef = createApiRef({ export interface KubernetesApi { getObjectsByLabelSelector( - serviceId: String, - labelSelector: V1LabelSelector, - requestBody: AuthRequestBody, - ): Promise; + requestBody: KubernetesRequestBody, + ): Promise; } diff --git a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx index 3f9c1a866f..4e1560be70 100644 --- a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx @@ -29,10 +29,10 @@ import { import { ComponentEntityV1alpha1, Entity } from '@backstage/catalog-model'; import { kubernetesApiRef } from '../../api/types'; import { - AuthRequestBody, + KubernetesRequestBody, ClusterObjects, FetchResponse, - ObjectsByServiceIdResponse, + ObjectsByEntityResponse, } from '@backstage/plugin-kubernetes-backend'; import { kubernetesAuthProvidersApiRef } from '../../kubernetes-auth-provider/types'; import { DeploymentTables } from '../DeploymentTables'; @@ -105,7 +105,7 @@ export const KubernetesContent = ({ entity }: KubernetesContentProps) => { const kubernetesApi = useApi(kubernetesApiRef); const [kubernetesObjects, setKubernetesObjects] = useState< - ObjectsByServiceIdResponse | undefined + ObjectsByEntityResponse | undefined >(undefined); const [error, setError] = useState(undefined); @@ -121,7 +121,9 @@ export const KubernetesContent = ({ entity }: KubernetesContentProps) => { useEffect(() => { (async () => { // For each auth type, invoke decorateRequestBodyForAuth on corresponding KubernetesAuthProvider - let requestBody: AuthRequestBody = {}; + let requestBody: KubernetesRequestBody = { + entity: entity as ComponentEntityV1alpha1, + }; for (const authProviderStr of authProviders) { // Multiple asyncs done sequentially instead of all at once to prevent same requestBody from being modified simultaneously requestBody = await kubernetesAuthProvidersApi.decorateRequestBodyForAuth( @@ -130,29 +132,9 @@ export const KubernetesContent = ({ entity }: KubernetesContentProps) => { ); } - // decide label selector to search by defaulting to this label - let labelSelector: V1LabelSelector = { - matchLabels: { - 'backstage.io/kubernetes-id': entity.metadata.name, - }, - }; - - const componentEntity = entity as ComponentEntityV1alpha1; - if ( - componentEntity.spec && - componentEntity.spec.kubernetes && - (componentEntity.spec.kubernetes.selector as V1LabelSelector) - ) { - labelSelector = componentEntity.spec.kubernetes.selector; - } - // TODO: Add validation on contents/format of requestBody kubernetesApi - .getObjectsByLabelSelector( - entity.metadata.name, - labelSelector, - requestBody, - ) + .getObjectsByLabelSelector(requestBody) .then(result => { setKubernetesObjects(result); }) diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/GoogleKubernetesAuthProvider.ts b/plugins/kubernetes/src/kubernetes-auth-provider/GoogleKubernetesAuthProvider.ts index 7de5c7c0a9..570806a1a4 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/GoogleKubernetesAuthProvider.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/GoogleKubernetesAuthProvider.ts @@ -16,7 +16,7 @@ import { OAuthApi } from '@backstage/core'; import { KubernetesAuthProvider } from './types'; -import { AuthRequestBody } from '@backstage/plugin-kubernetes-backend'; +import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-backend'; export class GoogleKubernetesAuthProvider implements KubernetesAuthProvider { authProvider: OAuthApi; @@ -26,8 +26,8 @@ export class GoogleKubernetesAuthProvider implements KubernetesAuthProvider { } async decorateRequestBodyForAuth( - requestBody: AuthRequestBody, - ): Promise { + requestBody: KubernetesRequestBody, + ): Promise { const googleAuthToken: string = await this.authProvider.getAccessToken( 'https://www.googleapis.com/auth/cloud-platform', ); diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts b/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts index ac909d6695..9f64af9c2e 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/KubernetesAuthProviders.ts @@ -15,7 +15,7 @@ */ import { OAuthApi } from '@backstage/core'; -import { AuthRequestBody } from '@backstage/plugin-kubernetes-backend'; +import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-backend'; import { KubernetesAuthProvider, KubernetesAuthProvidersApi } from './types'; import { GoogleKubernetesAuthProvider } from './GoogleKubernetesAuthProvider'; import { ServiceAccountKubernetesAuthProvider } from './ServiceAccountKubernetesAuthProvider'; @@ -40,8 +40,8 @@ export class KubernetesAuthProviders implements KubernetesAuthProvidersApi { async decorateRequestBodyForAuth( authProvider: string, - requestBody: AuthRequestBody, - ): Promise { + requestBody: KubernetesRequestBody, + ): Promise { const kubernetesAuthProvider: | KubernetesAuthProvider | undefined = this.kubernetesAuthProviderMap.get(authProvider); diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts b/plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts index 3ac5a9494b..5d33521ae4 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/ServiceAccountKubernetesAuthProvider.ts @@ -15,13 +15,13 @@ */ import { KubernetesAuthProvider } from './types'; -import { AuthRequestBody } from '@backstage/plugin-kubernetes-backend'; +import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-backend'; export class ServiceAccountKubernetesAuthProvider implements KubernetesAuthProvider { async decorateRequestBodyForAuth( - requestBody: AuthRequestBody, - ): Promise { + requestBody: KubernetesRequestBody, + ): Promise { // No-op, with service account for auth, cluster config/details should already have serviceAccountToken return requestBody; } diff --git a/plugins/kubernetes/src/kubernetes-auth-provider/types.ts b/plugins/kubernetes/src/kubernetes-auth-provider/types.ts index 24fee0f94c..5bd253d2c1 100644 --- a/plugins/kubernetes/src/kubernetes-auth-provider/types.ts +++ b/plugins/kubernetes/src/kubernetes-auth-provider/types.ts @@ -15,12 +15,12 @@ */ import { createApiRef } from '@backstage/core'; -import { AuthRequestBody } from '@backstage/plugin-kubernetes-backend'; +import { KubernetesRequestBody } from '@backstage/plugin-kubernetes-backend'; export interface KubernetesAuthProvider { decorateRequestBodyForAuth( - requestBody: AuthRequestBody, - ): Promise; + requestBody: KubernetesRequestBody, + ): Promise; } export const kubernetesAuthProvidersApiRef = createApiRef< @@ -33,6 +33,6 @@ export const kubernetesAuthProvidersApiRef = createApiRef< export interface KubernetesAuthProvidersApi { decorateRequestBodyForAuth( authProvider: string, - requestBody: AuthRequestBody, - ): Promise; + requestBody: KubernetesRequestBody, + ): Promise; } From 1166fcc36e22c7e4de184ca9e02479d84a14d72c Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 01:44:15 -0500 Subject: [PATCH 250/259] add changeset --- .changeset/chatty-radios-beam.md | 7 +++++++ plugins/kubernetes-backend/package.json | 1 + plugins/kubernetes/src/api/types.ts | 1 - .../src/components/KubernetesContent/KubernetesContent.tsx | 1 - 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changeset/chatty-radios-beam.md diff --git a/.changeset/chatty-radios-beam.md b/.changeset/chatty-radios-beam.md new file mode 100644 index 0000000000..91ad76ee88 --- /dev/null +++ b/.changeset/chatty-radios-beam.md @@ -0,0 +1,7 @@ +--- +'@backstage/catalog-model': minor +'@backstage/plugin-kubernetes': minor +'@backstage/plugin-kubernetes-backend': minor +--- + +add kubernetes selector to component model diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index e58bb4f047..be7b86dd52 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.3.0", + "@backstage/catalog-model": "^0.2.0", "@backstage/config": "^0.1.1", "@kubernetes/client-node": "^0.12.1", "@types/express": "^4.17.6", diff --git a/plugins/kubernetes/src/api/types.ts b/plugins/kubernetes/src/api/types.ts index 663841ddf9..90c9e33504 100644 --- a/plugins/kubernetes/src/api/types.ts +++ b/plugins/kubernetes/src/api/types.ts @@ -19,7 +19,6 @@ import { KubernetesRequestBody, ObjectsByEntityResponse, } from '@backstage/plugin-kubernetes-backend'; -import { V1LabelSelector } from '@kubernetes/client-node'; export const kubernetesApiRef = createApiRef({ id: 'plugin.kubernetes.service', diff --git a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx index 4e1560be70..f87ab04ece 100644 --- a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx @@ -41,7 +41,6 @@ import { ExtensionsV1beta1Ingress, V1ConfigMap, V1HorizontalPodAutoscaler, - V1LabelSelector, V1Service, } from '@kubernetes/client-node'; import { Services } from '../Services'; From a2f791836e80386fca7fb7613cb3e073fb8e6ab5 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 01:45:54 -0500 Subject: [PATCH 251/259] refactor name of api method --- plugins/kubernetes/src/api/KubernetesBackendClient.ts | 2 +- plugins/kubernetes/src/api/types.ts | 2 +- .../src/components/KubernetesContent/KubernetesContent.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/kubernetes/src/api/KubernetesBackendClient.ts b/plugins/kubernetes/src/api/KubernetesBackendClient.ts index d8fa800ab6..b81a799d6b 100644 --- a/plugins/kubernetes/src/api/KubernetesBackendClient.ts +++ b/plugins/kubernetes/src/api/KubernetesBackendClient.ts @@ -50,7 +50,7 @@ export class KubernetesBackendClient implements KubernetesApi { return await response.json(); } - async getObjectsByLabelSelector( + async getObjectsByEntity( requestBody: KubernetesRequestBody, ): Promise { return await this.getRequired( diff --git a/plugins/kubernetes/src/api/types.ts b/plugins/kubernetes/src/api/types.ts index 90c9e33504..498e74a626 100644 --- a/plugins/kubernetes/src/api/types.ts +++ b/plugins/kubernetes/src/api/types.ts @@ -27,7 +27,7 @@ export const kubernetesApiRef = createApiRef({ }); export interface KubernetesApi { - getObjectsByLabelSelector( + getObjectsByEntity( requestBody: KubernetesRequestBody, ): Promise; } diff --git a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx index f87ab04ece..14cda3b5f7 100644 --- a/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx +++ b/plugins/kubernetes/src/components/KubernetesContent/KubernetesContent.tsx @@ -133,7 +133,7 @@ export const KubernetesContent = ({ entity }: KubernetesContentProps) => { // TODO: Add validation on contents/format of requestBody kubernetesApi - .getObjectsByLabelSelector(requestBody) + .getObjectsByEntity(requestBody) .then(result => { setKubernetesObjects(result); }) From 75d63ee0a08cd07169dae07eae4bda871de4d718 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 01:58:33 -0500 Subject: [PATCH 252/259] fix test --- ...ubernetesObjectsByServiceIdHandler.test.ts | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index f95b9a7242..4fdd65f474 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -16,6 +16,7 @@ import { handleGetKubernetesObjectsForService } from './getKubernetesObjectsForServiceHandler'; import { getVoidLogger } from '@backstage/backend-common'; +import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; import { ClusterDetails } from '..'; const TEST_SERVICE_ID = 'my-service'; @@ -24,6 +25,26 @@ const fetchObjectsForService = jest.fn(); const getClustersByServiceId = jest.fn(); +const goodEntity: ComponentEntityV1alpha1 = { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { + name: 'test-component', + }, + spec: { + type: 'service', + lifecycle: 'production', + owner: 'joe', + kubernetes: { + selector: { + matchLabels: { + 'backstage.io/test-label': 'test-component', + }, + }, + }, + }, +}; + const mockFetch = (mock: jest.Mock) => { mock.mockImplementation((serviceId: string, clusterDetails: ClusterDetails) => Promise.resolve({ @@ -90,7 +111,7 @@ describe('handleGetKubernetesObjectsForService', () => { getClustersByServiceId, }, getVoidLogger(), - {}, + { entity: goodEntity }, ); expect(getClustersByServiceId.mock.calls.length).toBe(1); @@ -165,11 +186,11 @@ describe('handleGetKubernetesObjectsForService', () => { }, getVoidLogger(), { + entity: goodEntity, auth: { google: 'google_token_123', }, }, - '', ); expect(getClustersByServiceId.mock.calls.length).toBe(1); From 728767e3d39a0fd0c149357ecf96da2275f91809 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 14:00:13 -0500 Subject: [PATCH 253/259] cleanup --- .../src/service/KubernetesFetcher.test.ts | 49 ++++++++++--------- .../src/service/KubernetesFetcher.ts | 15 +++--- ...ubernetesObjectsByServiceIdHandler.test.ts | 2 +- .../getKubernetesObjectsForServiceHandler.ts | 28 +++++------ plugins/kubernetes-backend/src/types/types.ts | 12 +++-- plugins/kubernetes/README.md | 36 +++++++------- 6 files changed, 71 insertions(+), 71 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts index f37a81f9f8..29233cf7fd 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts @@ -16,6 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { KubernetesClientBasedFetcher } from './KubernetesFetcher'; +import { ObjectFetchParams } from '..'; describe('KubernetesClientProvider', () => { let clientMock: any; @@ -57,17 +58,17 @@ describe('KubernetesClientProvider', () => { clientMock.listServiceForAllNamespaces.mockRejectedValue(errorResponse); - const result = await sut.fetchObjectsForService( - 'some-service', - { + const result = await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { name: 'cluster1', url: 'http://localhost:9999', serviceAccountToken: 'token', authProvider: 'serviceAccount', }, - new Set(['pods', 'services']), - '', - ); + objectTypesToFetch: new Set(['pods', 'services']), + labelSelector: '', + }); expect(result).toStrictEqual({ errors: [expectedResult], @@ -121,17 +122,17 @@ describe('KubernetesClientProvider', () => { }, }); - const result = await sut.fetchObjectsForService( - 'some-service', - { + const result = await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { name: 'cluster1', url: 'http://localhost:9999', serviceAccountToken: 'token', authProvider: 'serviceAccount', }, - new Set(['pods', 'services']), - '', - ); + objectTypesToFetch: new Set(['pods', 'services']), + labelSelector: '', + }); expect(result).toStrictEqual({ errors: [], @@ -171,17 +172,17 @@ describe('KubernetesClientProvider', () => { }); it('should throw error on unknown type', () => { expect(() => - sut.fetchObjectsForService( - 'some-service', - { + sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { name: 'cluster1', url: 'http://localhost:9999', serviceAccountToken: 'token', authProvider: 'serviceAccount', }, - new Set(['foo']), - '', - ), + objectTypesToFetch: new Set(['foo']), + labelSelector: '', + }), ).toThrow('unrecognised type=foo'); expect(clientMock.listPodForAllNamespaces.mock.calls.length).toBe(0); @@ -282,17 +283,17 @@ describe('KubernetesClientProvider', () => { }, }); - await sut.fetchObjectsForService( - 'some-service', - { + await sut.fetchObjectsForService({ + serviceId: 'some-service', + clusterDetails: { name: 'cluster1', url: 'http://localhost:9999', serviceAccountToken: 'token', authProvider: 'serviceAccount', }, - new Set(['pods', 'services']), - '', - ); + objectTypesToFetch: new Set(['pods', 'services']), + labelSelector: '', + }); const mockCall = clientMock.listPodForAllNamespaces.mock.calls[0]; const actualSelector = mockCall[mockCall.length - 1]; diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index 11917efc52..d9c6ee6d1a 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -38,6 +38,7 @@ import { FetchResponseWrapper, KubernetesFetchError, KubernetesErrorTypes, + ObjectFetchParams, } from '..'; import lodash, { Dictionary } from 'lodash'; @@ -106,18 +107,14 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { } fetchObjectsForService( - serviceId: string, - clusterDetails: ClusterDetails, - objectTypesToFetch: Set, - labelSelector: string, + params: ObjectFetchParams, ): Promise { - const fetchResults = Array.from(objectTypesToFetch).map(type => { + const fetchResults = Array.from(params.objectTypesToFetch).map(type => { return this.fetchByObjectType( - clusterDetails, + params.clusterDetails, type, - labelSelector.length !== 0 - ? labelSelector - : `backstage.io/kubernetes-id=${serviceId}`, + params.labelSelector || + `backstage.io/kubernetes-id=${params.serviceId}`, ).catch(captureKubernetesErrorsRethrowOthers); }); diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index 4fdd65f474..789c7062b9 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -25,7 +25,7 @@ const fetchObjectsForService = jest.fn(); const getClustersByServiceId = jest.fn(); -const goodEntity: ComponentEntityV1alpha1 = { +const goodEntity: ComponentEntityV1alpha1 = { apiVersion: 'backstage.io/v1beta1', kind: 'Component', metadata: { diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts index 46f3f27c73..cec4fa8b66 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts @@ -15,6 +15,7 @@ */ import { Logger } from 'winston'; +import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; import { KubernetesRequestBody, ClusterDetails, @@ -22,10 +23,10 @@ import { KubernetesFetcher, KubernetesObjectTypes, ObjectsByEntityResponse, + ObjectFetchParams, } from '../types/types'; import { KubernetesAuthTranslator } from '../kubernetes-auth-translator/types'; import { KubernetesAuthTranslatorGenerator } from '../kubernetes-auth-translator/KubernetesAuthTranslatorGenerator'; -import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; export type GetKubernetesObjectsForServiceHandler = ( serviceId: string, @@ -47,18 +48,10 @@ const DEFAULT_OBJECTS = new Set([ ]); function parseLabelSelector(entity: ComponentEntityV1alpha1): string { - if ( - entity && - entity.spec && - entity.spec.kubernetes && - entity.spec.kubernetes.selector - ) { + const matchLabels = entity?.spec?.kubernetes?.selector?.matchLabels; + if (matchLabels) { // TODO: figure out how to convert the selector to the full query param from the yaml // (as shown here https://github.com/kubernetes/apimachinery/blob/master/pkg/labels/selector.go) - const { matchLabels } = entity.spec.kubernetes.selector; - if (!matchLabels) { - return ''; - } return Object.keys(matchLabels) .map(key => `${key}=${matchLabels[key.toString()]}`) .join(','); @@ -73,7 +66,7 @@ export const handleGetKubernetesObjectsForService: GetKubernetesObjectsForServic serviceLocator, logger, requestBody, - objectsToFetch = DEFAULT_OBJECTS, + objectTypesToFetch = DEFAULT_OBJECTS, ) => { const clusterDetails: ClusterDetails[] = await serviceLocator.getClustersByServiceId( serviceId, @@ -102,13 +95,18 @@ export const handleGetKubernetesObjectsForService: GetKubernetesObjectsForServic const labelSelector = parseLabelSelector(requestBody.entity); return Promise.all( - clusterDetailsDecoratedForAuth.map(cd => { + clusterDetailsDecoratedForAuth.map(clusterDetails => { return fetcher - .fetchObjectsForService(serviceId, cd, objectsToFetch, labelSelector) + .fetchObjectsForService({ + serviceId, + clusterDetails, + objectTypesToFetch, + labelSelector, + }) .then(result => { return { cluster: { - name: cd.name, + name: clusterDetails.name, }, resources: result.responses, errors: result.errors, diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index 738bc89b21..b2a5ef432c 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -109,14 +109,18 @@ export interface IngressesFetchResponse { resources: Array; } +export interface ObjectFetchParams { + serviceId: string; + clusterDetails: ClusterDetails; + objectTypesToFetch: Set; + labelSelector: string; +} + // Fetches information from a kubernetes cluster using the cluster details object // to target a specific cluster export interface KubernetesFetcher { fetchObjectsForService( - serviceId: string, - clusterDetails: ClusterDetails, - objectTypesToFetch: Set, - labelSelector: string, + params: ObjectFetchParams, ): Promise; } diff --git a/plugins/kubernetes/README.md b/plugins/kubernetes/README.md index e6f1f2355a..8f3542a46e 100644 --- a/plugins/kubernetes/README.md +++ b/plugins/kubernetes/README.md @@ -14,10 +14,26 @@ It is only meant for local development, and the setup for it can be found inside ## Surfacing your Kubernetes components as part of an entity -There are 2 ways to surface your kubernetes components as part of an entity. +There are two ways to surface your kubernetes components as part of an entity. The label selector takes precedence over the annotation/service id. -### Common `backstage.io/kubernetes-id` label +### Full label selector + +#### Adding the entity label selector to the spec + +In order for Backstage to detect that an entity has kubernetes components the `kubernetes.selector` must have a valid selector. (Currently only matchLabels is supported) + +```yaml +spec: + kubernetes: + selector: + matchLabels: + someKey: someValue + other-key: other-value + app.kubernetes.io/name: dice-roller +``` + +### Common `backstage.io/kubernetes-id` label on objects #### Adding the entity annotation @@ -37,19 +53,3 @@ as a part of an entity, Kubernetes components must be labeled with the following ```yaml 'backstage.io/kubernetes-id': ``` - -### Full label selector - -#### Adding the entity label selector to the spec - -In order for Backstage to detect that an entity has kubernetes components the `kubernetes.selector` must have a valid selector. (Currently only matchLabels is supported) - -```yaml -spec: - kubernetes: - selector: - matchLabels: - someKey: someValue - other-key: other-value - app.kubernetes.io/name: dice-roller -``` From 74102d917635bf59c3062d64b846151844c0d8cd Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 14:20:32 -0500 Subject: [PATCH 254/259] fix test after refactor --- .../getKubernetesObjectsByServiceIdHandler.test.ts | 8 ++++---- .../src/service/getKubernetesObjectsForServiceHandler.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index 789c7062b9..fd76cab662 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -46,7 +46,7 @@ const goodEntity: ComponentEntityV1alpha1 = { }; const mockFetch = (mock: jest.Mock) => { - mock.mockImplementation((serviceId: string, clusterDetails: ClusterDetails) => + mock.mockImplementation((params: ObjectFetchParams) => Promise.resolve({ errors: [], responses: [ @@ -55,7 +55,7 @@ const mockFetch = (mock: jest.Mock) => { resources: [ { metadata: { - name: `my-pods-${serviceId}-${clusterDetails.name}`, + name: `my-pods-${params.serviceId}-${params.clusterDetails.name}`, }, }, ], @@ -65,7 +65,7 @@ const mockFetch = (mock: jest.Mock) => { resources: [ { metadata: { - name: `my-configmaps-${serviceId}-${clusterDetails.name}`, + name: `my-configmaps-${params.serviceId}-${params.clusterDetails.name}`, }, }, ], @@ -75,7 +75,7 @@ const mockFetch = (mock: jest.Mock) => { resources: [ { metadata: { - name: `my-services-${serviceId}-${clusterDetails.name}`, + name: `my-services-${params.serviceId}-${params.clusterDetails.name}`, }, }, ], diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts index cec4fa8b66..1b0cf4f783 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsForServiceHandler.ts @@ -34,7 +34,7 @@ export type GetKubernetesObjectsForServiceHandler = ( serviceLocator: KubernetesServiceLocator, logger: Logger, requestBody: KubernetesRequestBody, - objectsToFetch?: Set, + objectTypesToFetch?: Set, ) => Promise; const DEFAULT_OBJECTS = new Set([ From 44ea3101a7e6659a6dbae7e20dd1e953078b9367 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 14:23:40 -0500 Subject: [PATCH 255/259] remove unneeded import --- .../src/service/getKubernetesObjectsByServiceIdHandler.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index fd76cab662..bb0ffddc26 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -17,7 +17,6 @@ import { handleGetKubernetesObjectsForService } from './getKubernetesObjectsForServiceHandler'; import { getVoidLogger } from '@backstage/backend-common'; import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; -import { ClusterDetails } from '..'; const TEST_SERVICE_ID = 'my-service'; From 18d0fd10a03c941ed87b734eada71a284586f8f3 Mon Sep 17 00:00:00 2001 From: Moustafa Baiou Date: Mon, 9 Nov 2020 14:49:12 -0500 Subject: [PATCH 256/259] add needed import --- .../src/service/getKubernetesObjectsByServiceIdHandler.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts index bb0ffddc26..adebebe1c5 100644 --- a/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/getKubernetesObjectsByServiceIdHandler.test.ts @@ -17,6 +17,7 @@ import { handleGetKubernetesObjectsForService } from './getKubernetesObjectsForServiceHandler'; import { getVoidLogger } from '@backstage/backend-common'; import { ComponentEntityV1alpha1 } from '@backstage/catalog-model'; +import { ObjectFetchParams } from '..'; const TEST_SERVICE_ID = 'my-service'; From d20f605082f3f2c1a0812c9d8172aafb298d6fd1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 23 Nov 2020 12:43:54 +0100 Subject: [PATCH 257/259] cli: add test for versions:bump command --- .../cli/src/commands/versions/bump.test.ts | 161 ++++++++++++++++++ packages/cli/src/commands/versions/bump.ts | 2 +- .../cli/src/lib/versioning/packages.test.ts | 2 +- packages/cli/src/lib/versioning/packages.ts | 9 +- 4 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 packages/cli/src/commands/versions/bump.test.ts diff --git a/packages/cli/src/commands/versions/bump.test.ts b/packages/cli/src/commands/versions/bump.test.ts new file mode 100644 index 0000000000..687708b074 --- /dev/null +++ b/packages/cli/src/commands/versions/bump.test.ts @@ -0,0 +1,161 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { resolve as resolvePath } from 'path'; +import { paths } from '../../lib/paths'; +import { mapDependencies } from '../../lib/versioning'; +import * as runObj from '../../lib/run'; +import bump from './bump'; +import { withLogCollector } from '@backstage/test-utils'; + +const REGISTRY_VERSIONS: { [name: string]: string } = { + '@backstage/core': '1.0.7', + '@backstage/theme': '2.0.0', +}; + +const HEADER = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + +`; + +const lockfileMock = `${HEADER} +"@backstage/core@^1.0.5": + version "1.0.6" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz + +"@backstage/core@^1.0.3": + version "1.0.3" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz + +"@backstage/theme@^1.0.0": + version "1.0.0" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz +`; + +const lockfileMockResult = `${HEADER} +"@backstage/core@^1.0.3", "@backstage/core@^1.0.5": + version "1.0.6" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz + +"@backstage/theme@^1.0.0": + version "1.0.0" + resolved "https://my-registry/a-1.0.01.tgz#abc123" + integrity sha512-xyz +`; + +describe('bump', () => { + afterEach(() => { + mockFs.restore(); + jest.resetAllMocks(); + }); + + it('should bump backstage dependencies', async () => { + // Make sure all modules involved in package discovery are in the module cache before we mock fs + await mapDependencies(paths.targetDir); + + mockFs({ + '/yarn.lock': lockfileMock, + '/lerna.json': JSON.stringify({ + packages: ['packages/*'], + }), + '/packages/a/package.json': JSON.stringify({ + name: 'a', + dependencies: { + '@backstage/core': '^1.0.5', + }, + }), + '/packages/b/package.json': JSON.stringify({ + name: 'b', + dependencies: { + '@backstage/core': '^1.0.3', + '@backstage/theme': '^1.0.0', + }, + }), + }); + + paths.targetDir = '/'; + jest + .spyOn(paths, 'resolveTargetRoot') + .mockImplementation((...paths) => resolvePath('/', ...paths)); + jest.spyOn(runObj, 'runPlain').mockImplementation(async (...[, , , name]) => + JSON.stringify({ + type: 'inspect', + data: { + name: name, + 'dist-tags': { + latest: REGISTRY_VERSIONS[name], + }, + }, + }), + ); + jest.spyOn(runObj, 'run').mockResolvedValue(undefined); + + const { log: logs } = await withLogCollector(['log'], async () => { + await bump(); + }); + expect(logs.filter(Boolean)).toEqual([ + 'Checking for updates of @backstage/theme', + 'Checking for updates of @backstage/core', + 'Some packages are outdated, updating', + 'Bumping @backstage/theme in b to ^2.0.0', + "Running 'yarn install' to install new versions", + 'Removing duplicate dependencies from yarn.lock', + "Running 'yarn install' to remove duplicates from node_modules", + ]); + + expect(runObj.runPlain).toHaveBeenCalledTimes(2); + expect(runObj.runPlain).toHaveBeenCalledWith( + 'yarn', + 'info', + '--json', + '@backstage/core', + ); + expect(runObj.runPlain).toHaveBeenCalledWith( + 'yarn', + 'info', + '--json', + '@backstage/theme', + ); + + expect(runObj.run).toHaveBeenCalledTimes(2); + expect(runObj.run).toHaveBeenCalledWith('yarn', ['install']); + + const lockfileContents = await fs.readFile('/yarn.lock', 'utf8'); + expect(lockfileContents).toBe(lockfileMockResult); + + const packageA = await fs.readJson('/packages/a/package.json'); + expect(packageA).toEqual({ + name: 'a', + dependencies: { + '@backstage/core': '^1.0.5', // not bumped since new version is within range + }, + }); + const packageB = await fs.readJson('/packages/b/package.json'); + expect(packageB).toEqual({ + name: 'b', + dependencies: { + '@backstage/core': '^1.0.3', // not bumped + '@backstage/theme': '^2.0.0', // bumped since newer + }, + }); + }); +}); diff --git a/packages/cli/src/commands/versions/bump.ts b/packages/cli/src/commands/versions/bump.ts index eec8abed9b..00944c5db9 100644 --- a/packages/cli/src/commands/versions/bump.ts +++ b/packages/cli/src/commands/versions/bump.ts @@ -40,7 +40,7 @@ type PkgVersionInfo = { export default async () => { // First we discover all Backstage dependencies within our own repo - const dependencyMap = await mapDependencies(); + const dependencyMap = await mapDependencies(paths.targetDir); // Next check with the package registry what the latest version of all of those dependencies are const targetVersions = new Map(); diff --git a/packages/cli/src/lib/versioning/packages.test.ts b/packages/cli/src/lib/versioning/packages.test.ts index fd37d7af95..8155d4b5c0 100644 --- a/packages/cli/src/lib/versioning/packages.test.ts +++ b/packages/cli/src/lib/versioning/packages.test.ts @@ -75,7 +75,7 @@ describe('mapDependencies', () => { const oldDir = paths.targetDir; paths.targetDir = '/'; - const dependencyMap = await mapDependencies(); + const dependencyMap = await mapDependencies(paths.targetDir); expect(Array.from(dependencyMap)).toEqual([ [ '@backstage/core', diff --git a/packages/cli/src/lib/versioning/packages.ts b/packages/cli/src/lib/versioning/packages.ts index 239fe575d6..76e5dc49b0 100644 --- a/packages/cli/src/lib/versioning/packages.ts +++ b/packages/cli/src/lib/versioning/packages.ts @@ -15,7 +15,6 @@ */ import { runPlain } from '../../lib/run'; -import { paths } from '../../lib/paths'; const PREFIX = '@backstage'; @@ -59,11 +58,11 @@ export async function fetchPackageInfo( } /** Map all dependencies in the repo as dependency => dependents */ -export async function mapDependencies(): Promise< - Map -> { +export async function mapDependencies( + targetDir: string, +): Promise> { const LernaProject = require('@lerna/project'); - const project = new LernaProject(paths.targetDir); + const project = new LernaProject(targetDir); const packages = await project.getPackages(); const dependencyMap = new Map(); From eeda19c7cefb9ab9650c1c0546cdc69a0c313f1f Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 23 Nov 2020 17:07:38 +0100 Subject: [PATCH 258/259] [Search] name in search result clickable (#3349) * fix(search): wrap value in link component for rows in component id column * fix(search result): change name of column from component id to name * fix(search): construct url in search api instead of search result component * fix(search): use entity default constant from catalog model as fallback to namespace, add entity type * fix(search): replace unknown with undefined if owner or lifecycle doesn't exist on entity, lowercase namespace and kind in url * fix owner fallback and delete description fallback to show empty cells instead --- plugins/search/package.json | 1 + plugins/search/src/apis.ts | 27 ++++++++++++------- .../components/SearchResult/SearchResult.tsx | 11 +++++--- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/plugins/search/package.json b/plugins/search/package.json index 0b1941d9e0..df5c6c25b9 100644 --- a/plugins/search/package.json +++ b/plugins/search/package.json @@ -22,6 +22,7 @@ "dependencies": { "@backstage/core": "^0.3.1", "@backstage/plugin-catalog": "^0.2.2", + "@backstage/catalog-model": "^0.2.0", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index f3a050b2a2..e74741e27b 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -15,13 +15,15 @@ */ import { CatalogApi } from '@backstage/plugin-catalog'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; export type Result = { name: string; - description: string; - owner: string; + description: string | undefined; + owner: string | undefined; kind: string; - lifecycle: string; + lifecycle: string | undefined; + url: string; }; export type SearchResults = Array; @@ -35,12 +37,19 @@ class SearchApi { private async entities() { const entities = await this.catalogApi.getEntities(); - return entities.items.map((result: any) => ({ - name: result.metadata.name, - description: result.metadata.description, - owner: result.spec.owner, - kind: result.kind, - lifecycle: result.spec.lifecycle, + return entities.items.map((entity: Entity) => ({ + name: entity.metadata.name, + description: entity.metadata.description, + owner: + typeof entity.spec?.owner === 'string' ? entity.spec?.owner : undefined, + kind: entity.kind, + lifecycle: + typeof entity.spec?.lifecycle === 'string' + ? entity.spec?.lifecycle + : undefined, + url: `/catalog/${ + entity.metadata.namespace?.toLowerCase() || ENTITY_DEFAULT_NAMESPACE + }/${entity.kind.toLowerCase()}/${entity.metadata.name}`, })); } diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index a5dc11aa89..f1e160e792 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -19,6 +19,7 @@ import { useAsync } from 'react-use'; import { makeStyles, Typography, Grid, Divider } from '@material-ui/core'; import { Alert } from '@material-ui/lab'; import { + Link, EmptyState, Progress, Table, @@ -66,9 +67,12 @@ type Filters = { // TODO: move out column to make the search result component more generic const columns: TableColumn[] = [ { - title: 'Component Id', + title: 'Name', field: 'name', highlight: true, + render: (result: Partial) => ( + {result.name} + ), }, { title: 'Description', @@ -150,8 +154,9 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => { // filter on checked if (filters.checked.length > 0) { - withFilters = withFilters.filter((result: Result) => - filters.checked.includes(result.lifecycle), + withFilters = withFilters.filter( + (result: Result) => + result.lifecycle && filters.checked.includes(result.lifecycle), ); } From b47dce06fc8538da931abef8ef740abaff119c34 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 23 Nov 2020 17:14:28 +0100 Subject: [PATCH 259/259] backend-common: make integrations host and urls visibile in the frontend --- .changeset/tame-rockets-explode.md | 5 +++++ packages/backend-common/config.d.ts | 35 +++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 .changeset/tame-rockets-explode.md diff --git a/.changeset/tame-rockets-explode.md b/.changeset/tame-rockets-explode.md new file mode 100644 index 0000000000..123472701e --- /dev/null +++ b/.changeset/tame-rockets-explode.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Make integration host and url configurations visible in the frontend diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 7ebcd01e8c..7a134ac506 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -87,7 +87,10 @@ export interface Config { integrations?: { /** Integration configuration for Azure */ azure?: Array<{ - /** The hostname of the given Azure instance */ + /** + * The hostname of the given Azure instance + * @visibility frontend + */ host: string; /** * Token used to authenticate requests. @@ -98,14 +101,20 @@ export interface Config { /** Integration configuration for BitBucket */ bitbucket?: Array<{ - /** The hostname of the given Bitbucket instance */ + /** + * The hostname of the given Bitbucket instance + * @visibility frontend + */ host: string; /** * Token used to authenticate requests. * @visibility secret */ token?: string; - /** The base url for the BitBucket API, for example https://api.bitbucket.org/2.0 */ + /** + * The base url for the BitBucket API, for example https://api.bitbucket.org/2.0 + * @visibility frontend + */ apiBaseUrl?: string; /** * The username to use for authenticated requests. @@ -121,22 +130,34 @@ export interface Config { /** Integration configuration for GitHub */ github?: Array<{ - /** The hostname of the given GitHub instance */ + /** + * The hostname of the given GitHub instance + * @visibility frontend + */ host: string; /** * Token used to authenticate requests. * @visibility secret */ token?: string; - /** The base url for the GitHub API, for example https://api.github.com */ + /** + * The base url for the GitHub API, for example https://api.github.com + * @visibility frontend + */ apiBaseUrl?: string; - /** The base url for GitHub raw resources, for example https://raw.githubusercontent.com */ + /** + * The base url for GitHub raw resources, for example https://raw.githubusercontent.com + * @visibility frontend + */ rawBaseUrl?: string; }>; /** Integration configuration for GitLab */ gitlab?: Array<{ - /** The hostname of the given GitLab instance */ + /** + * The hostname of the given GitLab instance + * @visibility frontend + */ host: string; /** * Token used to authenticate requests.