From 38b17fe361dadd9a98a7ca95bd5c6e39bfb3f2ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2020 04:45:38 +0000 Subject: [PATCH 01/51] chore(deps-dev): bump @testing-library/cypress from 6.0.1 to 7.0.1 Bumps [@testing-library/cypress](https://github.com/kentcdodds/cypress-testing-library) from 6.0.1 to 7.0.1. - [Release notes](https://github.com/kentcdodds/cypress-testing-library/releases) - [Changelog](https://github.com/testing-library/cypress-testing-library/blob/master/CHANGELOG.md) - [Commits](https://github.com/kentcdodds/cypress-testing-library/compare/v6.0.1...v7.0.1) Signed-off-by: dependabot[bot] --- packages/app/package.json | 2 +- yarn.lock | 21 ++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index e8dfb94046..a6875ba86f 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -46,7 +46,7 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@testing-library/cypress": "^6.0.0", + "@testing-library/cypress": "^7.0.1", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", diff --git a/yarn.lock b/yarn.lock index a3b64f0650..d17878a727 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4467,16 +4467,15 @@ dependencies: defer-to-connect "^2.0.0" -"@testing-library/cypress@^6.0.0": - version "6.0.1" - resolved "https://registry.npmjs.org/@testing-library/cypress/-/cypress-6.0.1.tgz#c924a69617db6a403c498abcb63759b79318fc76" - integrity sha512-hcPu2OnVuSTX1yDubEe7TBRD6mP2VgyopP1WTBIrJP79INPZdgOAX+TMNH68uZ/r5b4C7100IB4hjPIEQ+/Xog== +"@testing-library/cypress@^7.0.1": + version "7.0.1" + resolved "https://registry.npmjs.org/@testing-library/cypress/-/cypress-7.0.1.tgz#2843033acaefe96cb4cf789f16e98d957383e59d" + integrity sha512-LtggqG/7Hdc1EiKdmqXQwxWOO3ET1dkZtq0S8mIe8o+xaOtaVLrdCn0dE8Bi4Aj7z3w51w6wN9STdYymnUPlnQ== dependencies: "@babel/runtime" "^7.11.2" "@testing-library/dom" "^7.22.2" - "@types/testing-library__cypress" "^5.0.6" -"@testing-library/dom@^7.11.0", "@testing-library/dom@^7.17.1", "@testing-library/dom@^7.22.2": +"@testing-library/dom@^7.17.1", "@testing-library/dom@^7.22.2": version "7.23.0" resolved "https://registry.npmjs.org/@testing-library/dom/-/dom-7.23.0.tgz#c54c0fa53705ad867bcefb52fc0c96487fbc10f6" integrity sha512-H5m090auYH+obdZmsaYLrSWC5OauWD2CvNbz88KBxQJoXgkJzbU0DpAG8BS7Evj5WqCC3nAAKrLS6vw0ljUYLg== @@ -5600,14 +5599,6 @@ dependencies: "@types/estree" "*" -"@types/testing-library__cypress@^5.0.6": - version "5.0.6" - resolved "https://registry.npmjs.org/@types/testing-library__cypress/-/testing-library__cypress-5.0.6.tgz#9015f575c1a98f05996a4fe769071134ee488c26" - integrity sha512-TUp5wfanU7zUZigKqIeQDChnHQ1MEzbYqrI5iCQMFiesWNOASWm/el1lFBh1JPqmd6GkdDdDiHYJnkqd9le2ww== - dependencies: - "@testing-library/dom" "^7.11.0" - cypress "*" - "@types/testing-library__jest-dom@^5.9.1": version "5.9.1" resolved "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.9.1.tgz#aba5ee062b7880f69c212ef769389f30752806e5" @@ -9338,7 +9329,7 @@ cyclist@^1.0.1: resolved "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= -cypress@*, cypress@^4.2.0: +cypress@^4.2.0: version "4.12.1" resolved "https://registry.npmjs.org/cypress/-/cypress-4.12.1.tgz#0ead1b9f4c0917d69d8b57f996b6e01fe693b6ec" integrity sha512-9SGIPEmqU8vuRA6xst2CMTYd9sCFCxKSzrHt0wr+w2iAQMCIIsXsQ5Gplns1sT6LDbZcmLv6uehabAOl3fhc9Q== From a6a61f192c89c65c1293b07ac3398237d2a6a957 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Tue, 17 Nov 2020 17:28:30 -0500 Subject: [PATCH 02/51] fix tooltip for long period separated titles --- .../src/components/BarChart/BarChartTooltip.tsx | 7 ++++--- plugins/cost-insights/src/utils/styles.ts | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx index 2cb09b724e..8ab787a443 100644 --- a/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx +++ b/plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx @@ -35,7 +35,6 @@ export const BarChartTooltip = ({ children, }: PropsWithChildren) => { const classes = useStyles(); - return ( - {title} + + {title} + {subtitle && ( {subtitle} )} - {topRight} + {topRight && {topRight}} {content && ( diff --git a/plugins/cost-insights/src/utils/styles.ts b/plugins/cost-insights/src/utils/styles.ts index de743d8459..1e8fd50899 100644 --- a/plugins/cost-insights/src/utils/styles.ts +++ b/plugins/cost-insights/src/utils/styles.ts @@ -386,7 +386,7 @@ export const useTooltipStyles = makeStyles( boxShadow: theme.shadows[1], color: theme.palette.tooltip.color, fontSize: theme.typography.fontSize, - width: 250, + maxWidth: 300, }, actions: { padding: theme.spacing(2), @@ -403,6 +403,12 @@ export const useTooltipStyles = makeStyles( divider: { backgroundColor: emphasize(theme.palette.divider, 1), }, + truncate: { + maxWidth: 200, + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + }, subtitle: { fontStyle: 'italic', }, From 9023404510fc9579a8f755482d695cf49a67a5c9 Mon Sep 17 00:00:00 2001 From: Joel Low Date: Wed, 18 Nov 2020 14:11:15 +0800 Subject: [PATCH 03/51] feat(cli): Support specifying listen host/port for frontend --- .changeset/modern-boats-knock.md | 5 +++++ packages/cli/src/lib/bundler/server.ts | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 .changeset/modern-boats-knock.md diff --git a/.changeset/modern-boats-knock.md b/.changeset/modern-boats-knock.md new file mode 100644 index 0000000000..16f80fe041 --- /dev/null +++ b/.changeset/modern-boats-knock.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Support specifying listen host/port for frontend diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index e1fb560e22..2710145d83 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -25,7 +25,12 @@ import { resolveBundlingPaths } from './paths'; export async function serveBundle(options: ServeOptions) { const url = resolveBaseUrl(options.config); - const port = Number(url.port) || (url.protocol === 'https:' ? 443 : 80); + const host = + options.config.getOptionalString('app.listen.host') || url.hostname; + const port = + options.config.getOptionalNumber('app.listen.port') || + Number(url.port) || + (url.protocol === 'https:' ? 443 : 80); const paths = resolveBundlingPaths(options); const pkgPath = paths.targetPackageJson; @@ -50,7 +55,7 @@ export async function serveBundle(options: ServeOptions) { clientLogLevel: 'warning', stats: 'errors-warnings', https: url.protocol === 'https:', - host: url.hostname, + host, port, proxy: pkg.proxy, }); From 7d680c075e9fd75122a4a3ac3dc8cb50e7f7bcdc Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Wed, 18 Nov 2020 11:20:37 +0100 Subject: [PATCH 04/51] [Search] Search frontend plugin (#3086) * feat(search) search plugin * feat(search): search page components * feat(search): api * feat(search): filters wip * feat(search): wip filters * fix(search): delete unused useParams hook * fix(search): update docs * fix(search): use latest versions of dependencies * fix(search): change version of catalog plugin * fix(search api): pass instance of catalog api to search api * fix(filters): rename component from FilterButton to FiltersButton * fixup * fix(filters): use list of checkboxes to match style of catalog page filters * fix(styles): use theme spacing for margins and paddings, delete unused styles * fix(search): change terminology of search input to be more consistent and clear * fix(search): restructure component exports according to ADR * fix(search): replace sm with xs on Grid components to support smaller screens * fix(search): add types * fixup * fix(search): bump backstage core * change versions of backstage theme and dev-utils --- .github/CODEOWNERS | 1 + packages/app/package.json | 1 + packages/app/src/plugins.ts | 1 + plugins/search/.eslintrc.js | 3 + plugins/search/README.md | 9 + plugins/search/dev/index.tsx | 19 ++ plugins/search/package.json | 49 ++++ plugins/search/src/apis.ts | 52 ++++ .../search/src/components/Filters/Filters.tsx | 132 ++++++++++ .../src/components/Filters/FiltersButton.tsx | 56 +++++ .../search/src/components/Filters/index.tsx | 19 ++ .../src/components/SearchBar/SearchBar.tsx | 69 ++++++ .../search/src/components/SearchBar/index.tsx | 17 ++ .../src/components/SearchPage/SearchPage.tsx | 55 +++++ .../src/components/SearchPage/index.tsx | 17 ++ .../components/SearchResult/SearchResult.tsx | 227 ++++++++++++++++++ .../src/components/SearchResult/index.tsx | 17 ++ plugins/search/src/components/index.tsx | 20 ++ plugins/search/src/index.ts | 16 ++ plugins/search/src/plugin.test.ts | 22 ++ plugins/search/src/plugin.ts | 29 +++ plugins/search/src/setupTests.ts | 17 ++ 22 files changed, 848 insertions(+) create mode 100644 plugins/search/.eslintrc.js create mode 100644 plugins/search/README.md create mode 100644 plugins/search/dev/index.tsx create mode 100644 plugins/search/package.json create mode 100644 plugins/search/src/apis.ts create mode 100644 plugins/search/src/components/Filters/Filters.tsx create mode 100644 plugins/search/src/components/Filters/FiltersButton.tsx create mode 100644 plugins/search/src/components/Filters/index.tsx create mode 100644 plugins/search/src/components/SearchBar/SearchBar.tsx create mode 100644 plugins/search/src/components/SearchBar/index.tsx create mode 100644 plugins/search/src/components/SearchPage/SearchPage.tsx create mode 100644 plugins/search/src/components/SearchPage/index.tsx create mode 100644 plugins/search/src/components/SearchResult/SearchResult.tsx create mode 100644 plugins/search/src/components/SearchResult/index.tsx create mode 100644 plugins/search/src/components/index.tsx create mode 100644 plugins/search/src/index.ts create mode 100644 plugins/search/src/plugin.test.ts create mode 100644 plugins/search/src/plugin.ts create mode 100644 plugins/search/src/setupTests.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d818c0d6b5..b3c9bc653d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,5 +9,6 @@ /plugins/cost-insights @backstage/silver-lining /plugins/cloudbuild @trivago/ebarrios /plugins/techdocs @backstage/techdocs-core +/plugins/search @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /.changeset/cost-insights-* @backstage/silver-lining diff --git a/packages/app/package.json b/packages/app/package.json index f013200fbb..dea63e3a4c 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -25,6 +25,7 @@ "@backstage/plugin-rollbar": "^0.2.1", "@backstage/plugin-scaffolder": "^0.3.0", "@backstage/plugin-sentry": "^0.2.1", + "@backstage/plugin-search": "^0.2.0", "@backstage/plugin-tech-radar": "^0.3.0", "@backstage/plugin-techdocs": "^0.2.1", "@backstage/plugin-user-settings": "^0.2.1", diff --git a/packages/app/src/plugins.ts b/packages/app/src/plugins.ts index c61a16e59c..d6577b4ce4 100644 --- a/packages/app/src/plugins.ts +++ b/packages/app/src/plugins.ts @@ -39,3 +39,4 @@ export { plugin as CostInsights } from '@backstage/plugin-cost-insights'; export { plugin as GitHubInsights } from '@roadiehq/backstage-plugin-github-insights'; export { plugin as UserSettings } from '@backstage/plugin-user-settings'; export { plugin as BuildKite } from '@roadiehq/backstage-plugin-buildkite'; +export { plugin as Search } from '@backstage/plugin-search'; diff --git a/plugins/search/.eslintrc.js b/plugins/search/.eslintrc.js new file mode 100644 index 0000000000..13573efa9c --- /dev/null +++ b/plugins/search/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint')], +}; diff --git a/plugins/search/README.md b/plugins/search/README.md new file mode 100644 index 0000000000..fb4b047811 --- /dev/null +++ b/plugins/search/README.md @@ -0,0 +1,9 @@ +# Backstage Search + +**This plugin is still under development.** + +You can follow the progress under the Global search in Backstage [milestone](https://github.com/backstage/backstage/milestone/21) or reach out to us in the #search Discord channel. + +## Getting started + +Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search)to check out the plugin. diff --git a/plugins/search/dev/index.tsx b/plugins/search/dev/index.tsx new file mode 100644 index 0000000000..264d6f801f --- /dev/null +++ b/plugins/search/dev/index.tsx @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createDevApp } from '@backstage/dev-utils'; +import { plugin } from '../src/plugin'; + +createDevApp().registerPlugin(plugin).render(); diff --git a/plugins/search/package.json b/plugins/search/package.json new file mode 100644 index 0000000000..6f356d1bf4 --- /dev/null +++ b/plugins/search/package.json @@ -0,0 +1,49 @@ +{ + "name": "@backstage/plugin-search", + "version": "0.2.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "build": "backstage-cli plugin:build", + "start": "backstage-cli plugin:serve", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "diff": "backstage-cli plugin:diff", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/core": "^0.3.0", + "@backstage/theme": "^0.2.1", + "@material-ui/core": "^4.11.0", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "4.0.0-alpha.45", + "@backstage/plugin-catalog": "^0.2.0", + "react-router-dom": "6.0.0-beta.0", + "react": "^16.13.1", + "react-dom": "^16.13.1", + "react-use": "^15.3.3" + }, + "devDependencies": { + "@backstage/cli": "^0.2.0", + "@backstage/dev-utils": "^0.1.3", + "@backstage/test-utils": "^0.1.2", + "@testing-library/jest-dom": "^5.10.1", + "@testing-library/react": "^10.4.1", + "@testing-library/user-event": "^12.0.7", + "@types/jest": "^26.0.7", + "@types/node": "^12.0.0", + "msw": "^0.21.2", + "cross-fetch": "^3.0.6" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts new file mode 100644 index 0000000000..1783b91d2a --- /dev/null +++ b/plugins/search/src/apis.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogApi } from '@backstage/plugin-catalog'; + +export type Result = { + name: string; + description: string; + owner: string; + kind: string; + lifecycle: string; +}; + +export type SearchResults = Array; + +class SearchApi { + private catalogApi: CatalogApi; + + constructor(catalogApi: CatalogApi) { + this.catalogApi = catalogApi; + } + + private async entities() { + const entities = await this.catalogApi.getEntities(); + return entities.map((result: any) => ({ + name: result.metadata.name, + description: result.metadata.description, + owner: result.spec.owner, + kind: result.kind, + lifecycle: result.spec.lifecycle, + })); + } + + public getSearchResult(): Promise { + return this.entities(); + } +} + +export default SearchApi; diff --git a/plugins/search/src/components/Filters/Filters.tsx b/plugins/search/src/components/Filters/Filters.tsx new file mode 100644 index 0000000000..19d4e53d72 --- /dev/null +++ b/plugins/search/src/components/Filters/Filters.tsx @@ -0,0 +1,132 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { + makeStyles, + Typography, + Divider, + Card, + CardHeader, + Button, + CardContent, + Select, + Checkbox, + List, + ListItem, + ListItemText, + MenuItem, +} from '@material-ui/core'; + +const useStyles = makeStyles(theme => ({ + filters: { + background: 'transparent', + boxShadow: '0px 0px 0px 0px', + }, + checkbox: { + padding: theme.spacing(0, 1, 0, 1), + }, + dropdown: { + width: '100%', + }, +})); + +export type FiltersState = { + selected: string; + checked: Array; +}; + +type FiltersProps = { + filters: FiltersState; + resetFilters: () => void; + updateSelected: (filter: string) => void; + updateChecked: (filter: string) => void; +}; + +export const Filters = ({ + filters, + resetFilters, + updateSelected, + updateChecked, +}: FiltersProps) => { + const classes = useStyles(); + + // TODO: move mocked filters out of filters component to make it more generic + const filter1 = ['All', 'API', 'Component', 'Location', 'Template']; + const filter2 = ['deprecated', 'recommended', 'experimental', 'production']; + + return ( + + Filters} + action={ + + } + /> + + + Kind + + + + Lifecycle + + {filter2.map(filter => ( + updateChecked(filter)} + > + + + + ))} + + + + ); +}; diff --git a/plugins/search/src/components/Filters/FiltersButton.tsx b/plugins/search/src/components/Filters/FiltersButton.tsx new file mode 100644 index 0000000000..4775e9d8b8 --- /dev/null +++ b/plugins/search/src/components/Filters/FiltersButton.tsx @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import FilterListIcon from '@material-ui/icons/FilterList'; +import { makeStyles, IconButton, Typography } from '@material-ui/core'; + +const useStyles = makeStyles(theme => ({ + filters: { + width: '250px', + display: 'flex', + }, + icon: { + margin: theme.spacing(-1, 0, 0, 0), + }, +})); + +type FiltersButtonProps = { + numberOfSelectedFilters: number; + handleToggleFilters: () => void; +}; + +export const FiltersButton = ({ + numberOfSelectedFilters, + handleToggleFilters, +}: FiltersButtonProps) => { + const classes = useStyles(); + + return ( +
+ + + + + Filters ({numberOfSelectedFilters ? numberOfSelectedFilters : 0}) + +
+ ); +}; diff --git a/plugins/search/src/components/Filters/index.tsx b/plugins/search/src/components/Filters/index.tsx new file mode 100644 index 0000000000..ea431a3c01 --- /dev/null +++ b/plugins/search/src/components/Filters/index.tsx @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { FiltersButton } from './FiltersButton'; +export { Filters } from './Filters'; +export type { FiltersState } from './Filters'; diff --git a/plugins/search/src/components/SearchBar/SearchBar.tsx b/plugins/search/src/components/SearchBar/SearchBar.tsx new file mode 100644 index 0000000000..9aa48e7284 --- /dev/null +++ b/plugins/search/src/components/SearchBar/SearchBar.tsx @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { makeStyles } from '@material-ui/core/styles'; +import { Paper } from '@material-ui/core'; +import InputBase from '@material-ui/core/InputBase'; +import IconButton from '@material-ui/core/IconButton'; +import SearchIcon from '@material-ui/icons/Search'; +import ClearButton from '@material-ui/icons/Clear'; + +const useStyles = makeStyles(() => ({ + root: { + display: 'flex', + alignItems: 'center', + }, + input: { + flex: 1, + }, +})); + +type SearchBarProps = { + searchQuery: string; + handleSearch: any; + handleClearSearchBar: any; +}; + +export const SearchBar = ({ + searchQuery, + handleSearch, + handleClearSearchBar, +}: SearchBarProps) => { + const classes = useStyles(); + + return ( + handleSearch(e)} + className={classes.root} + > + + + + handleSearch(e)} + inputProps={{ 'aria-label': 'search backstage' }} + /> + handleClearSearchBar()}> + + + + ); +}; diff --git a/plugins/search/src/components/SearchBar/index.tsx b/plugins/search/src/components/SearchBar/index.tsx new file mode 100644 index 0000000000..e2b8af1946 --- /dev/null +++ b/plugins/search/src/components/SearchBar/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchBar } from './SearchBar'; diff --git a/plugins/search/src/components/SearchPage/SearchPage.tsx b/plugins/search/src/components/SearchPage/SearchPage.tsx new file mode 100644 index 0000000000..b5779a3d6e --- /dev/null +++ b/plugins/search/src/components/SearchPage/SearchPage.tsx @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState } from 'react'; + +import { Header, Content, Page } from '@backstage/core'; +import { Grid } from '@material-ui/core'; + +import { SearchBar } from '../SearchBar'; +import { SearchResult } from '../SearchResult'; + +export const SearchPage = () => { + const [searchQuery, setSearchQuery] = useState(''); + + const handleSearch = (event: React.ChangeEvent) => { + event.preventDefault(); + setSearchQuery(event.target.value); + }; + + const handleClearSearchBar = () => { + setSearchQuery(''); + }; + + return ( + +
+ + + + + + + + + + + + ); +}; diff --git a/plugins/search/src/components/SearchPage/index.tsx b/plugins/search/src/components/SearchPage/index.tsx new file mode 100644 index 0000000000..acdc0967ab --- /dev/null +++ b/plugins/search/src/components/SearchPage/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchPage } from './SearchPage'; diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx new file mode 100644 index 0000000000..a4985c20fb --- /dev/null +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -0,0 +1,227 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState, useEffect } from 'react'; +import { useAsync } from 'react-use'; + +import { makeStyles, Typography, Grid, Divider } from '@material-ui/core'; +import { Table, TableColumn, useApi } from '@backstage/core'; +import { catalogApiRef } from '@backstage/plugin-catalog'; + +import { FiltersButton, Filters, FiltersState } from '../Filters'; +import SearchApi, { Result, SearchResults } from '../../apis'; + +const useStyles = makeStyles(theme => ({ + searchTerm: { + background: '#eee', + borderRadius: '10%', + }, + tableHeader: { + margin: theme.spacing(1, 0, 0, 0), + display: 'flex', + }, + divider: { + width: '1px', + margin: theme.spacing(0, 2), + padding: theme.spacing(2, 0), + }, +})); + +type SearchResultProps = { + searchQuery?: string; +}; + +type TableHeaderProps = { + searchQuery?: string; + numberOfSelectedFilters: number; + numberOfResults: number; + handleToggleFilters: () => void; +}; + +type Filters = { + selected: string; + checked: Array; +}; + +// TODO: move out column to make the search result component more generic +const columns: TableColumn[] = [ + { + title: 'Component Id', + field: 'name', + highlight: true, + }, + { + title: 'Description', + field: 'description', + }, + { + title: 'Owner', + field: 'owner', + }, + { + title: 'Kind', + field: 'kind', + }, + { + title: 'LifeCycle', + field: 'lifecycle', + }, +]; + +const TableHeader = ({ + searchQuery, + numberOfSelectedFilters, + numberOfResults, + handleToggleFilters, +}: TableHeaderProps) => { + const classes = useStyles(); + + return ( +
+ + + + {searchQuery ? ( + + {`${numberOfResults} `} + {numberOfResults > 1 ? `results for ` : `result for `} + "{searchQuery}"{' '} + + ) : ( + {`${numberOfResults} results`} + )} + +
+ ); +}; + +export const SearchResult = ({ searchQuery }: SearchResultProps) => { + const catalogApi = useApi(catalogApiRef); + + const [showFilters, toggleFilters] = useState(false); + const [filters, setFilters] = useState({ + selected: 'All', + checked: [], + }); + + const [filteredResults, setFilteredResults] = useState([]); + + const searchApi = new SearchApi(catalogApi); + + const { loading, error, value: results } = useAsync(() => { + return searchApi.getSearchResult(); + }, []); + + useEffect(() => { + if (results) { + let withFilters = results; + + // apply filters + + // filter on selected + if (filters.selected !== 'All') { + withFilters = results.filter((result: Result) => + filters.selected.includes(result.kind), + ); + } + + // filter on checked + if (filters.checked.length > 0) { + withFilters = withFilters.filter((result: Result) => + filters.checked.includes(result.lifecycle), + ); + } + + // filter on searchQuery + if (searchQuery) { + withFilters = withFilters.filter( + (result: Result) => + result.name?.toLowerCase().includes(searchQuery) || + result.description?.toLowerCase().includes(searchQuery), + ); + } + + setFilteredResults(withFilters); + } + }, [filters, searchQuery, results]); + + if (loading || error || !results) return null; + + const resetFilters = () => { + setFilters({ + selected: 'All', + checked: [], + }); + }; + + const updateSelected = (filter: string) => { + setFilters(prevState => ({ + ...prevState, + selected: filter, + })); + }; + + const updateChecked = (filter: string) => { + if (filters.checked.includes(filter)) { + setFilters(prevState => ({ + ...prevState, + checked: prevState.checked.filter(item => item !== filter), + })); + return; + } + + setFilters(prevState => ({ + ...prevState, + checked: [...prevState.checked, filter], + })); + }; + + return ( + <> + + {showFilters && ( + + + + )} + + toggleFilters(!showFilters)} + /> + } + /> + + + + ); +}; diff --git a/plugins/search/src/components/SearchResult/index.tsx b/plugins/search/src/components/SearchResult/index.tsx new file mode 100644 index 0000000000..cf10135fd0 --- /dev/null +++ b/plugins/search/src/components/SearchResult/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchResult } from './SearchResult'; diff --git a/plugins/search/src/components/index.tsx b/plugins/search/src/components/index.tsx new file mode 100644 index 0000000000..f8e6a5a09e --- /dev/null +++ b/plugins/search/src/components/index.tsx @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Filters'; +export * from './SearchBar'; +export * from './SearchPage'; +export * from './SearchResult'; diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts new file mode 100644 index 0000000000..224e293890 --- /dev/null +++ b/plugins/search/src/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { plugin } from './plugin'; diff --git a/plugins/search/src/plugin.test.ts b/plugins/search/src/plugin.test.ts new file mode 100644 index 0000000000..92b8d5bcbf --- /dev/null +++ b/plugins/search/src/plugin.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { plugin } from './plugin'; + +describe('search', () => { + it('should export plugin', () => { + expect(plugin).toBeDefined(); + }); +}); diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts new file mode 100644 index 0000000000..44c7bcb042 --- /dev/null +++ b/plugins/search/src/plugin.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createPlugin, createRouteRef } from '@backstage/core'; +import { SearchPage } from './components/SearchPage'; + +export const rootRouteRef = createRouteRef({ + path: '/search', + title: 'search', +}); + +export const plugin = createPlugin({ + id: 'search', + register({ router }) { + router.addRoute(rootRouteRef, SearchPage); + }, +}); diff --git a/plugins/search/src/setupTests.ts b/plugins/search/src/setupTests.ts new file mode 100644 index 0000000000..43b8421558 --- /dev/null +++ b/plugins/search/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; From ddb5a38f7782089da0f0875bbe0339daa6adfb4a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 7 Nov 2020 17:51:16 +0100 Subject: [PATCH 05/51] backend-common,core,auth-backend: added initial config schemas --- packages/backend-common/config-schema.json | 80 ++++++++++++++++++++++ packages/backend-common/package.json | 3 +- packages/core/package.json | 51 +++++++++++++- plugins/auth-backend/package.json | 35 +++++++++- 4 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 packages/backend-common/config-schema.json diff --git a/packages/backend-common/config-schema.json b/packages/backend-common/config-schema.json new file mode 100644 index 0000000000..2012447a11 --- /dev/null +++ b/packages/backend-common/config-schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/backend-common", + "type": "object", + "properties": { + "integrations": { + "type": "object", + "additionalProperties": false, + "properties": { + "github": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "token": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "gitlab": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "token": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "bitbucket": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "username": { + "type": "string" + }, + "appPassword": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "azure": { + "type": "array", + "items": { + "type": "object", + "required": ["host"], + "properties": { + "host": { + "type": "string" + }, + "token": { + "type": "string", + "visibility": "secret" + } + } + } + } + } + } + } +} diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index ea84d1a383..2f58bd0536 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -90,5 +90,6 @@ }, "files": [ "dist" - ] + ], + "configSchema": "config-schema.json" } diff --git a/packages/core/package.json b/packages/core/package.json index 848f0847bd..3d4b53d35b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -80,5 +80,54 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/core", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "title": { + "type": "string", + "visibility": "frontend" + }, + "baseUrl": { + "type": "string", + "visibility": "frontend" + } + }, + "required": [ + "title", + "baseUrl" + ] + }, + "backend": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "visibility": "frontend" + } + }, + "required": [ + "baseUrl" + ] + }, + "organization": { + "type": "object", + "properties": { + "name": { + "type": "string", + "visibility": "frontend" + } + } + } + }, + "required": [ + "app", + "backend" + ] + } } diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index af3d056b28..2bc00f56a2 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -66,5 +66,38 @@ "files": [ "dist", "migrations" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/plugin-auth-backend", + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "providers": { + "type": "object", + "properties": { + "google": { + "type": "object", + "visibility": "frontend", + "additionalProperties": { + "type": "object", + "properties": { + "clientId": { + "type": "string" + }, + "clientSecret": { + "type": "string", + "visibility": "secret" + } + } + } + } + } + } + } + } + } + } } From 6fec95079a18d2da1c8036d16809dac0d67388dc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 7 Nov 2020 18:08:19 +0100 Subject: [PATCH 06/51] config-loader: initial incomplete config schema implementation --- packages/config-loader/package.json | 6 + packages/config-loader/src/lib/index.ts | 1 + packages/config-loader/src/lib/schema.ts | 216 +++++++++++++++++++++++ packages/config-loader/src/loader.ts | 10 +- yarn.lock | 32 +++- 5 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 packages/config-loader/src/lib/schema.ts diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 2d3474f3af..39735ff2fd 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -30,8 +30,14 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", + "@types/json-schema": "^7.0.6", + "@types/json-schema-merge-allof": "^0.6.0", + "ajv": "^7.0.0-beta.2", "fs-extra": "^9.0.0", + "json-schema": "^0.2.5", + "json-schema-merge-allof": "^0.7.0", "yaml": "^1.9.2", "yup": "^0.29.3" }, diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 1b5ad2ef50..29812053bc 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; +export { loadSchema } from './schema'; diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts new file mode 100644 index 0000000000..c3441bce38 --- /dev/null +++ b/packages/config-loader/src/lib/schema.ts @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; +import Ajv from 'ajv'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; +import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; +import { AppConfig, ConfigReader } from '@backstage/config'; + +const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; + +type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; + +type ConfigSchema = { + load(appConfigs: AppConfig[]): AppConfig[]; +}; + +type ConfigSchemaPackageEntry = { + value: JSONSchema; + path: string; +}; + +type Options = { + dependencies: string[]; +}; + +type ValidationError = string; + +type ValidationResult = { + errors?: ValidationError[]; + visibilities: Map; +}; + +type ValidationFunc = (configs: AppConfig[]) => ValidationResult; + +export async function loadSchema(options: Options): Promise { + const start = process.hrtime(); + + const schemas = await collectConfigSchemas(options.dependencies[0]); + + const [durS, durNs] = process.hrtime(start); + const dur = (durS + durNs / 10 ** 9).toFixed(3); + console.log(`DEBUG: collected config schemas in ${dur}s`); + console.log('DEBUG: schemas =', schemas); + + const validate = compileConfigSchemas(schemas); + + return { + load(configs: AppConfig[]): AppConfig[] { + const result = validate(configs); + console.log('DEBUG: result =', result); + if (result.errors) { + throw new Error( + `Config validation failed, ${result.errors.join('; ')}`, + ); + } + + return configs; + }, + }; +} + +function compileConfigSchemas( + schemas: ConfigSchemaPackageEntry[], +): ValidationFunc { + const visibilities = new Map(); + + const ajv = new Ajv({ + strict: true, + allErrors: true, + defaultMeta: 'http://json-schema.org/draft-07/schema#', + schemas: { + 'https://backstage.io/schema/config-v1': true, + }, + keywords: [ + { + keyword: 'visibility', + schemaType: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, ctx) => { + if (!ctx) { + return false; + } + if (visibility === 'frontend') { + visibilities.set(ctx.dataPath, visibility); + } + return true; + }; + }, + }, + ], + }); + + const merged = mergeAllOf( + { allOf: schemas.map(_ => _.value) }, + { + ignoreAdditionalProperties: true, + resolvers: { + visibility(values: string[], path: string[]) { + const hasApp = values.some(_ => _ === 'frontend'); + const hasSecret = values.some(_ => _ === 'secret'); + if (hasApp && hasSecret) { + throw new Error( + `Config schema visibility is both 'frontend' and 'secret' for ${path.join( + '/', + )}`, + ); + } else if (hasApp) { + return 'frontend'; + } else if (hasSecret) { + return 'secret'; + } + + return 'backend'; + }, + } as Partial>, + }, + ); + + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilities.clear(); + const valid = validate(config); + if (!valid) { + const errors = ajv.errorsText(validate.errors); + return { + errors: [errors], + visibilities: new Map(), + }; + } + + return { + visibilities: new Map(visibilities), + }; + }; +} + +const req = + typeof __non_webpack_require__ === 'undefined' + ? require + : __non_webpack_require__; + +async function collectConfigSchemas( + name: string, + opts: unknown = {}, + visited: Set = new Set(), + schemas: ConfigSchemaPackageEntry[] = [], +): Promise { + const pkgPath = req.resolve(`${name}/package.json`, opts); + if (visited.has(pkgPath)) { + return schemas; + } + visited.add(pkgPath); + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return schemas; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + for (const depName of depNames) { + await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); + } + + return schemas; +} diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index e7eb2a38fa..318eaa7915 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname, isAbsolute } from 'path'; import { AppConfig, JsonObject } from '@backstage/config'; -import { readConfigFile, readEnvConfig, readSecret } from './lib'; +import { readConfigFile, readEnvConfig, readSecret, loadSchema } from './lib'; export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -71,13 +71,17 @@ class Context { } } +type LoadedConfig = AppConfig[]; + export async function loadConfig( options: LoadConfigOptions, -): Promise { +): Promise { const configs = []; const { configRoot } = options; const configPaths = options.configPaths.slice(); + const schema = await loadSchema({ dependencies: ['example-backend'] }); + // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` if (configPaths.length === 0) { @@ -126,5 +130,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return configs; + return schema.load(configs); } diff --git a/yarn.lock b/yarn.lock index 10e70f52fb..98e1b2476f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5369,7 +5369,14 @@ resolved "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-3.12.5.tgz#136d5e6a57a931e1cce6f9d8126aa98a9c92a6bb" integrity sha512-JCcp6J0GV66Y4ZMDAQCXot4xprYB+Zfd3meK9+INSJeVZwJmHAW30BBEEkPzXswMXuiyReUGOP3GxrADc9wPww== -"@types/json-schema@*", "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5": +"@types/json-schema-merge-allof@^0.6.0": + version "0.6.0" + resolved "https://registry.npmjs.org/@types/json-schema-merge-allof/-/json-schema-merge-allof-0.6.0.tgz#0f587d8a3bcb41a55ef2e91d3ba96430c9bc1813" + integrity sha512-v6iCEk4Sxy1twlCTtrRxMqSHX0vuLJ7Ql4hiIUZRMOswxtlUeybIfYaZIj7pX747RBczG2YtBm4Fn6sqKzeY2Q== + dependencies: + "@types/json-schema" "*" + +"@types/json-schema@*", "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.4", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6": version "7.0.6" resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz#f4c7ec43e81b319a9815115031709f26987891f0" integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== @@ -6562,6 +6569,15 @@ ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.1, ajv@^6.10.2, ajv@^6.12.4, ajv@^6.12.5, ajv json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ajv@^7.0.0-beta.2: + version "7.0.0-beta.3" + resolved "https://registry.npmjs.org/ajv/-/ajv-7.0.0-beta.3.tgz#d34861ccfbdebb55bf9f49a08b29f76bf656fc5c" + integrity sha512-gUGVvM4NmyqrFvCNAQnP4P7FC0RjxMQyRnrXpozNglBkDJnTysVbvycyOZUy5n6yLKSqVDUqWZBXj7dXINrSqw== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^0.5.0" + uri-js "^4.2.2" + alphanum-sort@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3" @@ -15022,11 +15038,25 @@ json-schema-merge-allof@^0.6.0: json-schema-compare "^0.2.2" lodash "^4.17.4" +json-schema-merge-allof@^0.7.0: + version "0.7.0" + resolved "https://registry.npmjs.org/json-schema-merge-allof/-/json-schema-merge-allof-0.7.0.tgz#84d3e8c3e03d3060014286958eb8834fa9d76304" + integrity sha512-kvsuSVnl1n5xnNEu5ed4o8r8ujSA4/IgRtHmpgfMfa7FOMIRAzN4F9qbuklouTn5J8bi83y6MQ11n+ERMMTXZg== + dependencies: + compute-lcm "^1.1.0" + json-schema-compare "^0.2.2" + lodash "^4.17.4" + json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== +json-schema-traverse@^0.5.0: + version "0.5.0" + resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.5.0.tgz#1069a6097f9e9a567bfc6cc215e85f606ebf6480" + integrity sha512-x+TRJIQFskrNnFKE2Viz9FCSjK1vIh+H/uaBiOYszh/IcZmAFneQ35H4osWDJp1NPXccuV2I0RMXmi2ZS6Kqcg== + json-schema@0.2.3: version "0.2.3" resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" From 47952780392b522827707c6ffbcf5a58771c89c2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:05:56 +0100 Subject: [PATCH 07/51] config-loader: add config filtering by visibility --- packages/config-loader/src/lib/schema.ts | 97 +++++++++++++++++++++--- packages/config-loader/src/loader.ts | 2 +- 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts index c3441bce38..0d820ab173 100644 --- a/packages/config-loader/src/lib/schema.ts +++ b/packages/config-loader/src/lib/schema.ts @@ -19,14 +19,24 @@ import { resolve as resolvePath, dirname } from 'path'; import Ajv from 'ajv'; import { JSONSchema7 as JSONSchema } from 'json-schema'; import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; -import { AppConfig, ConfigReader } from '@backstage/config'; +import { + AppConfig, + ConfigReader, + JsonObject, + JsonValue, +} from '@backstage/config'; const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; +const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; + type ConfigSchema = { - load(appConfigs: AppConfig[]): AppConfig[]; + process( + appConfigs: AppConfig[], + options?: ConfigProcessingOptions, + ): AppConfig[]; }; type ConfigSchemaPackageEntry = { @@ -42,11 +52,61 @@ type ValidationError = string; type ValidationResult = { errors?: ValidationError[]; - visibilities: Map; + visibilityByPath: Map; }; type ValidationFunc = (configs: AppConfig[]) => ValidationResult; +type ConfigProcessingOptions = { + visibilities?: ConfigVisibility[]; +}; + +export function filterByVisibility( + data: JsonObject, + includeVisibilities: ConfigVisibility[], + visibilityByPath: Map, +): JsonObject { + function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + if (typeof jsonVal !== 'object') { + const visibility = + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + if (includeVisibilities.includes(visibility)) { + return jsonVal; + } + return undefined; + } else if (jsonVal === null) { + return undefined; + } else if (Array.isArray(jsonVal)) { + const arr = new Array(); + + for (const [index, value] of jsonVal.entries()) { + const out = transform(value, `${path}/${index}`); + if (out !== undefined) { + arr.push(out); + } + } + + return arr.length === 0 ? undefined : arr; + } + + const outObj: JsonObject = {}; + + for (const [key, value] of Object.entries(jsonVal)) { + if (value === undefined) { + continue; + } + const out = transform(value, `${path}/${key}`); + if (out !== undefined) { + outObj[key] = out; + } + } + + return Object.keys(outObj).length === 0 ? undefined : outObj; + } + + return (transform(data, '') as JsonObject) ?? {}; +} + export async function loadSchema(options: Options): Promise { const start = process.hrtime(); @@ -60,16 +120,28 @@ export async function loadSchema(options: Options): Promise { const validate = compileConfigSchemas(schemas); return { - load(configs: AppConfig[]): AppConfig[] { + process( + configs: AppConfig[], + { visibilities }: ConfigProcessingOptions = {}, + ): AppConfig[] { const result = validate(configs); - console.log('DEBUG: result =', result); if (result.errors) { throw new Error( `Config validation failed, ${result.errors.join('; ')}`, ); } - return configs; + let processedConfigs = configs; + + if (visibilities) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility(data, visibilities, result.visibilityByPath), + })); + } + console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); + + return processedConfigs; }, }; } @@ -77,7 +149,7 @@ export async function loadSchema(options: Options): Promise { function compileConfigSchemas( schemas: ConfigSchemaPackageEntry[], ): ValidationFunc { - const visibilities = new Map(); + const visibilityByPath = new Map(); const ajv = new Ajv({ strict: true, @@ -99,8 +171,8 @@ function compileConfigSchemas( if (!ctx) { return false; } - if (visibility === 'frontend') { - visibilities.set(ctx.dataPath, visibility); + if (visibility) { + visibilityByPath.set(ctx.dataPath, visibility); } return true; }; @@ -140,18 +212,19 @@ function compileConfigSchemas( return configs => { const config = ConfigReader.fromConfigs(configs).get(); - visibilities.clear(); + visibilityByPath.clear(); + const valid = validate(config); if (!valid) { const errors = ajv.errorsText(validate.errors); return { errors: [errors], - visibilities: new Map(), + visibilityByPath: new Map(), }; } return { - visibilities: new Map(visibilities), + visibilityByPath: new Map(visibilityByPath), }; }; } diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 318eaa7915..2d3c2afd9c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -130,5 +130,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return schema.load(configs); + return schema.process(configs, { visibilities: ['frontend', 'secret'] }); } From 8c65cec932d2818914b0ef124059e832016952a9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:18:58 +0100 Subject: [PATCH 08/51] config-loader: split up config schema implementation and move into separate lib --- packages/config-loader/src/index.ts | 2 +- packages/config-loader/src/lib/index.ts | 2 +- packages/config-loader/src/lib/schema.ts | 289 ------------------ .../config-loader/src/lib/schema/collect.ts | 78 +++++ .../config-loader/src/lib/schema/compile.ts | 109 +++++++ .../config-loader/src/lib/schema/filtering.ts | 64 ++++ .../config-loader/src/lib/schema/index.ts | 17 ++ packages/config-loader/src/lib/schema/load.ts | 63 ++++ .../config-loader/src/lib/schema/types.ts | 49 +++ packages/config-loader/src/loader.ts | 6 +- 10 files changed, 384 insertions(+), 295 deletions(-) delete mode 100644 packages/config-loader/src/lib/schema.ts create mode 100644 packages/config-loader/src/lib/schema/collect.ts create mode 100644 packages/config-loader/src/lib/schema/compile.ts create mode 100644 packages/config-loader/src/lib/schema/filtering.ts create mode 100644 packages/config-loader/src/lib/schema/index.ts create mode 100644 packages/config-loader/src/lib/schema/load.ts create mode 100644 packages/config-loader/src/lib/schema/types.ts diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 02db134fe5..415b6d4022 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export { readEnvConfig } from './lib'; +export { readEnvConfig, loadConfigSchema } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 29812053bc..4da0c97788 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,4 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; -export { loadSchema } from './schema'; +export { loadConfigSchema } from './schema'; diff --git a/packages/config-loader/src/lib/schema.ts b/packages/config-loader/src/lib/schema.ts deleted file mode 100644 index 0d820ab173..0000000000 --- a/packages/config-loader/src/lib/schema.ts +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { resolve as resolvePath, dirname } from 'path'; -import Ajv from 'ajv'; -import { JSONSchema7 as JSONSchema } from 'json-schema'; -import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; -import { - AppConfig, - ConfigReader, - JsonObject, - JsonValue, -} from '@backstage/config'; - -const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; - -type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; - -const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; - -type ConfigSchema = { - process( - appConfigs: AppConfig[], - options?: ConfigProcessingOptions, - ): AppConfig[]; -}; - -type ConfigSchemaPackageEntry = { - value: JSONSchema; - path: string; -}; - -type Options = { - dependencies: string[]; -}; - -type ValidationError = string; - -type ValidationResult = { - errors?: ValidationError[]; - visibilityByPath: Map; -}; - -type ValidationFunc = (configs: AppConfig[]) => ValidationResult; - -type ConfigProcessingOptions = { - visibilities?: ConfigVisibility[]; -}; - -export function filterByVisibility( - data: JsonObject, - includeVisibilities: ConfigVisibility[], - visibilityByPath: Map, -): JsonObject { - function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { - if (typeof jsonVal !== 'object') { - const visibility = - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; - if (includeVisibilities.includes(visibility)) { - return jsonVal; - } - return undefined; - } else if (jsonVal === null) { - return undefined; - } else if (Array.isArray(jsonVal)) { - const arr = new Array(); - - for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}/${index}`); - if (out !== undefined) { - arr.push(out); - } - } - - return arr.length === 0 ? undefined : arr; - } - - const outObj: JsonObject = {}; - - for (const [key, value] of Object.entries(jsonVal)) { - if (value === undefined) { - continue; - } - const out = transform(value, `${path}/${key}`); - if (out !== undefined) { - outObj[key] = out; - } - } - - return Object.keys(outObj).length === 0 ? undefined : outObj; - } - - return (transform(data, '') as JsonObject) ?? {}; -} - -export async function loadSchema(options: Options): Promise { - const start = process.hrtime(); - - const schemas = await collectConfigSchemas(options.dependencies[0]); - - const [durS, durNs] = process.hrtime(start); - const dur = (durS + durNs / 10 ** 9).toFixed(3); - console.log(`DEBUG: collected config schemas in ${dur}s`); - console.log('DEBUG: schemas =', schemas); - - const validate = compileConfigSchemas(schemas); - - return { - process( - configs: AppConfig[], - { visibilities }: ConfigProcessingOptions = {}, - ): AppConfig[] { - const result = validate(configs); - if (result.errors) { - throw new Error( - `Config validation failed, ${result.errors.join('; ')}`, - ); - } - - let processedConfigs = configs; - - if (visibilities) { - processedConfigs = processedConfigs.map(({ data, context }) => ({ - context, - data: filterByVisibility(data, visibilities, result.visibilityByPath), - })); - } - console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); - - return processedConfigs; - }, - }; -} - -function compileConfigSchemas( - schemas: ConfigSchemaPackageEntry[], -): ValidationFunc { - const visibilityByPath = new Map(); - - const ajv = new Ajv({ - strict: true, - allErrors: true, - defaultMeta: 'http://json-schema.org/draft-07/schema#', - schemas: { - 'https://backstage.io/schema/config-v1': true, - }, - keywords: [ - { - keyword: 'visibility', - schemaType: 'string', - metaSchema: { - type: 'string', - enum: CONFIG_VISIBILITIES, - }, - compile(visibility: ConfigVisibility) { - return (_data, ctx) => { - if (!ctx) { - return false; - } - if (visibility) { - visibilityByPath.set(ctx.dataPath, visibility); - } - return true; - }; - }, - }, - ], - }); - - const merged = mergeAllOf( - { allOf: schemas.map(_ => _.value) }, - { - ignoreAdditionalProperties: true, - resolvers: { - visibility(values: string[], path: string[]) { - const hasApp = values.some(_ => _ === 'frontend'); - const hasSecret = values.some(_ => _ === 'secret'); - if (hasApp && hasSecret) { - throw new Error( - `Config schema visibility is both 'frontend' and 'secret' for ${path.join( - '/', - )}`, - ); - } else if (hasApp) { - return 'frontend'; - } else if (hasSecret) { - return 'secret'; - } - - return 'backend'; - }, - } as Partial>, - }, - ); - - const validate = ajv.compile(merged); - - return configs => { - const config = ConfigReader.fromConfigs(configs).get(); - - visibilityByPath.clear(); - - const valid = validate(config); - if (!valid) { - const errors = ajv.errorsText(validate.errors); - return { - errors: [errors], - visibilityByPath: new Map(), - }; - } - - return { - visibilityByPath: new Map(visibilityByPath), - }; - }; -} - -const req = - typeof __non_webpack_require__ === 'undefined' - ? require - : __non_webpack_require__; - -async function collectConfigSchemas( - name: string, - opts: unknown = {}, - visited: Set = new Set(), - schemas: ConfigSchemaPackageEntry[] = [], -): Promise { - const pkgPath = req.resolve(`${name}/package.json`, opts); - if (visited.has(pkgPath)) { - return schemas; - } - visited.add(pkgPath); - const pkg = await fs.readJson(pkgPath); - const depNames = [ - ...Object.keys(pkg.dependencies ?? {}), - ...Object.keys(pkg.peerDependencies ?? {}), - ]; - - // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, - // since that's pretty slow. We probably need a better way to determine when - // we've left the Backstage ecosystem, but this will do for now. - const hasSchema = 'configSchema' in pkg; - const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); - if (!hasSchema && !hasBackstageDep) { - return schemas; - } - if (hasSchema) { - if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { - throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, - ); - } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: pkgPath, - }); - } else { - schemas.push({ - value: pkg.configSchema, - path: pkgPath, - }); - } - } - - for (const depName of depNames) { - await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); - } - - return schemas; -} diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts new file mode 100644 index 0000000000..b6d0bffaef --- /dev/null +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; +import { ConfigSchemaPackageEntry } from './types'; + +const req = + typeof __non_webpack_require__ === 'undefined' + ? require + : __non_webpack_require__; + +export async function collectConfigSchemas( + name: string, + opts: unknown = {}, + visited: Set = new Set(), + schemas: ConfigSchemaPackageEntry[] = [], +): Promise { + const pkgPath = req.resolve(`${name}/package.json`, opts); + if (visited.has(pkgPath)) { + return schemas; + } + visited.add(pkgPath); + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return schemas; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + for (const depName of depNames) { + await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); + } + + return schemas; +} diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts new file mode 100644 index 0000000000..9ab8627fce --- /dev/null +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Ajv from 'ajv'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; +import mergeAllOf, { Resolvers } from 'json-schema-merge-allof'; +import { ConfigReader } from '@backstage/config'; +import { + ConfigSchemaPackageEntry, + ValidationFunc, + CONFIG_VISIBILITIES, + ConfigVisibility, +} from './types'; + +export function compileConfigSchemas( + schemas: ConfigSchemaPackageEntry[], +): ValidationFunc { + const visibilityByPath = new Map(); + + const ajv = new Ajv({ + strict: true, + allErrors: true, + defaultMeta: 'http://json-schema.org/draft-07/schema#', + schemas: { + 'https://backstage.io/schema/config-v1': true, + }, + keywords: [ + { + keyword: 'visibility', + schemaType: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, ctx) => { + if (!ctx) { + return false; + } + if (visibility) { + visibilityByPath.set(ctx.dataPath, visibility); + } + return true; + }; + }, + }, + ], + }); + + const merged = mergeAllOf( + { allOf: schemas.map(_ => _.value) }, + { + ignoreAdditionalProperties: true, + resolvers: { + visibility(values: string[], path: string[]) { + const hasApp = values.some(_ => _ === 'frontend'); + const hasSecret = values.some(_ => _ === 'secret'); + if (hasApp && hasSecret) { + throw new Error( + `Config schema visibility is both 'frontend' and 'secret' for ${path.join( + '/', + )}`, + ); + } else if (hasApp) { + return 'frontend'; + } else if (hasSecret) { + return 'secret'; + } + + return 'backend'; + }, + } as Partial>, + }, + ); + + const validate = ajv.compile(merged); + + return configs => { + const config = ConfigReader.fromConfigs(configs).get(); + + visibilityByPath.clear(); + + const valid = validate(config); + if (!valid) { + const errors = ajv.errorsText(validate.errors); + return { + errors: [errors], + visibilityByPath: new Map(), + }; + } + + return { + visibilityByPath: new Map(visibilityByPath), + }; + }; +} diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts new file mode 100644 index 0000000000..64ccfa5e24 --- /dev/null +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject, JsonValue } from '@backstage/config'; +import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; + +export function filterByVisibility( + data: JsonObject, + includeVisibilities: ConfigVisibility[], + visibilityByPath: Map, +): JsonObject { + function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + if (typeof jsonVal !== 'object') { + const visibility = + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + if (includeVisibilities.includes(visibility)) { + return jsonVal; + } + return undefined; + } else if (jsonVal === null) { + return undefined; + } else if (Array.isArray(jsonVal)) { + const arr = new Array(); + + for (const [index, value] of jsonVal.entries()) { + const out = transform(value, `${path}/${index}`); + if (out !== undefined) { + arr.push(out); + } + } + + return arr.length === 0 ? undefined : arr; + } + + const outObj: JsonObject = {}; + + for (const [key, value] of Object.entries(jsonVal)) { + if (value === undefined) { + continue; + } + const out = transform(value, `${path}/${key}`); + if (out !== undefined) { + outObj[key] = out; + } + } + + return Object.keys(outObj).length === 0 ? undefined : outObj; + } + + return (transform(data, '') as JsonObject) ?? {}; +} diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts new file mode 100644 index 0000000000..990c0dc182 --- /dev/null +++ b/packages/config-loader/src/lib/schema/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { loadConfigSchema } from './load'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts new file mode 100644 index 0000000000..75fcb86ba7 --- /dev/null +++ b/packages/config-loader/src/lib/schema/load.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { compileConfigSchemas } from './compile'; +import { collectConfigSchemas } from './collect'; +import { filterByVisibility } from './filtering'; +import { ConfigSchema } from './types'; + +type Options = { + dependencies: string[]; +}; + +export async function loadConfigSchema( + options: Options, +): Promise { + const start = process.hrtime(); + + const schemas = await collectConfigSchemas(options.dependencies[0]); + + const [durS, durNs] = process.hrtime(start); + const dur = (durS + durNs / 10 ** 9).toFixed(3); + console.log(`DEBUG: collected config schemas in ${dur}s`); + console.log('DEBUG: schemas =', schemas); + + const validate = compileConfigSchemas(schemas); + + return { + process(configs: AppConfig[], { visibilities } = {}): AppConfig[] { + const result = validate(configs); + if (result.errors) { + throw new Error( + `Config validation failed, ${result.errors.join('; ')}`, + ); + } + + let processedConfigs = configs; + + if (visibilities) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility(data, visibilities, result.visibilityByPath), + })); + } + console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); + + return processedConfigs; + }, + }; +} diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts new file mode 100644 index 0000000000..9e5624ca6a --- /dev/null +++ b/packages/config-loader/src/lib/schema/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppConfig } from '@backstage/config'; +import { JSONSchema7 as JSONSchema } from 'json-schema'; + +export type ConfigSchemaPackageEntry = { + value: JSONSchema; + path: string; +}; + +export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; + +export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; + +export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; + +type ValidationError = string; + +type ValidationResult = { + errors?: ValidationError[]; + visibilityByPath: Map; +}; + +export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; + +type ConfigProcessingOptions = { + visibilities?: ConfigVisibility[]; +}; + +export type ConfigSchema = { + process( + appConfigs: AppConfig[], + options?: ConfigProcessingOptions, + ): AppConfig[]; +}; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 2d3c2afd9c..00414f86eb 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname, isAbsolute } from 'path'; import { AppConfig, JsonObject } from '@backstage/config'; -import { readConfigFile, readEnvConfig, readSecret, loadSchema } from './lib'; +import { readConfigFile, readEnvConfig, readSecret } from './lib'; export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. @@ -80,8 +80,6 @@ export async function loadConfig( const { configRoot } = options; const configPaths = options.configPaths.slice(); - const schema = await loadSchema({ dependencies: ['example-backend'] }); - // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` if (configPaths.length === 0) { @@ -130,5 +128,5 @@ export async function loadConfig( configs.push(...readEnvConfig(process.env)); - return schema.process(configs, { visibilities: ['frontend', 'secret'] }); + return configs; } From b652ebc4e7e276d963804f3d1a6e5c98c81e0a62 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:39:07 +0100 Subject: [PATCH 09/51] config-loader: add visibility filtering tests + fix filtering logic --- .../src/lib/schema/filtering.test.ts | 105 ++++++++++++++++++ .../config-loader/src/lib/schema/filtering.ts | 23 ++-- 2 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 packages/config-loader/src/lib/schema/filtering.test.ts diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts new file mode 100644 index 0000000000..8cc907417b --- /dev/null +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject } from '@backstage/config'; +import { ConfigVisibility } from './types'; +import { filterByVisibility } from './filtering'; + +const data = { + arr: ['f', 'b', 's'], + objArr: [ + { f: 1, b: 2, s: 3 }, + { f: 4, b: 5, s: 6 }, + ], + obj: { + f: 'a', + b: { + s: true, + }, + }, + arrF: [{ never: 'here' }], + arrB: [{ never: 'here' }], + arrS: [{ never: 'here' }], + objF: { never: 'here' }, + objB: { never: 'here' }, + objS: { never: 'here' }, +}; + +const visibilities = new Map( + Object.entries({ + '/arr/0': 'frontend', + '/arr/1': 'backend', + '/arr/2': 'secret', + '/obj/f': 'frontend', + '/obj/b': 'backend', + '/obj/b/s': 'secret', + '/objArr/0/f': 'frontend', + '/objArr/0/b': 'backend', + '/objArr/0/s': 'secret', + '/objArr/1/f': 'frontend', + '/objArr/1/b': 'backend', + '/objArr/1/s': 'secret', + '/arrF': 'frontend', + '/arrB': 'backend', + '/arrS': 'secret', + '/objF': 'frontend', + '/objB': 'backend', + '/objS': 'secret', + }), +); + +describe('filterByVisibility', () => { + test.each<[ConfigVisibility[], JsonObject]>([ + [[], {}], + [ + ['frontend'], + { + arr: ['f'], + objArr: [{ f: 1 }, { f: 4 }], + obj: { f: 'a' }, + arrF: [], + objF: {}, + }, + ], + [ + ['backend'], + { + arr: ['b'], + objArr: [{ b: 2 }, { b: 5 }], + obj: { b: {} }, + arrF: [{ never: 'here' }], + arrB: [{ never: 'here' }], + arrS: [{ never: 'here' }], + objF: { never: 'here' }, + objB: { never: 'here' }, + objS: { never: 'here' }, + }, + ], + [ + ['secret'], + { + arr: ['s'], + objArr: [{ s: 3 }, { s: 6 }], + obj: { b: { s: true } }, + arrS: [], + objS: {}, + }, + ], + [['frontend', 'backend', 'secret'], data], + ])('should filter correctly with %p', (filter, expected) => { + expect(filterByVisibility(data, filter, visibilities)).toEqual(expected); + }); +}); diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 64ccfa5e24..3ed4c958b8 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -23,13 +23,12 @@ export function filterByVisibility( visibilityByPath: Map, ): JsonObject { function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { + const isVisible = includeVisibilities.includes( + visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY, + ); + if (typeof jsonVal !== 'object') { - const visibility = - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; - if (includeVisibilities.includes(visibility)) { - return jsonVal; - } - return undefined; + return isVisible ? jsonVal : undefined; } else if (jsonVal === null) { return undefined; } else if (Array.isArray(jsonVal)) { @@ -42,10 +41,14 @@ export function filterByVisibility( } } - return arr.length === 0 ? undefined : arr; + if (arr.length > 0 || isVisible) { + return arr; + } + return undefined; } const outObj: JsonObject = {}; + let hasOutput = false; for (const [key, value] of Object.entries(jsonVal)) { if (value === undefined) { @@ -54,10 +57,14 @@ export function filterByVisibility( const out = transform(value, `${path}/${key}`); if (out !== undefined) { outObj[key] = out; + hasOutput = true; } } - return Object.keys(outObj).length === 0 ? undefined : outObj; + if (hasOutput || isVisible) { + return outObj; + } + return undefined; } return (transform(data, '') as JsonObject) ?? {}; From a533c15b0979745739a86456742f6ff7ba06aa2f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:57:50 +0100 Subject: [PATCH 10/51] config-loader: document schema lib --- .../config-loader/src/lib/schema/collect.ts | 3 ++ .../config-loader/src/lib/schema/compile.ts | 16 +++++++ .../config-loader/src/lib/schema/filtering.ts | 4 ++ packages/config-loader/src/lib/schema/load.ts | 3 ++ .../config-loader/src/lib/schema/types.ts | 45 +++++++++++++++++++ 5 files changed, 71 insertions(+) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index b6d0bffaef..25bdbe7f55 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -23,6 +23,9 @@ const req = ? require : __non_webpack_require__; +/** + * This collects all known config schemas across all dependencies of the app. + */ export async function collectConfigSchemas( name: string, opts: unknown = {}, diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 9ab8627fce..76642690d5 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -25,9 +25,19 @@ import { ConfigVisibility, } from './types'; +/** + * This takes a collection of Backstage configuration schemas from various + * sources and compiles them down into a single schema validation function. + * + * It also handles the implementation of the custom "visibility" keyword used + * to specify the scope of different config paths. + */ export function compileConfigSchemas( schemas: ConfigSchemaPackageEntry[], ): ValidationFunc { + // The ajv instance below is stateful and doesn't really allow for additional + // output during validation. We work around this by having this extra piece + // of state that we reset before each validation. const visibilityByPath = new Map(); const ajv = new Ajv({ @@ -63,8 +73,13 @@ export function compileConfigSchemas( const merged = mergeAllOf( { allOf: schemas.map(_ => _.value) }, { + // JSONSchema is typically subtractive, as in it always reduces the set of allowed + // inputs through constraints. This changes the object property merging to be additive + // rather than subtractive. ignoreAdditionalProperties: true, resolvers: { + // This ensures that the visibilities across different schemas are sound, and + // selects the most specific visibility for each path. visibility(values: string[], path: string[]) { const hasApp = values.some(_ => _ === 'frontend'); const hasSecret = values.some(_ => _ === 'secret'); @@ -95,6 +110,7 @@ export function compileConfigSchemas( const valid = validate(config); if (!valid) { + // TODO(Rugvip): better messages here, with more context such as which file the error occurred in const errors = ajv.errorsText(validate.errors); return { errors: [errors], diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 3ed4c958b8..d6dcaf2dd7 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -17,6 +17,10 @@ import { JsonObject, JsonValue } from '@backstage/config'; import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; +/** + * This filters data by visibility by discovering the visibility of each + * value, and then only keeping the ones that are specified in `includeVisibilities`. + */ export function filterByVisibility( data: JsonObject, includeVisibilities: ConfigVisibility[], diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 75fcb86ba7..286b363b50 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -24,6 +24,9 @@ type Options = { dependencies: string[]; }; +/** + * Loads config schema for a Backstage instance. + */ export async function loadConfigSchema( options: Options, ): Promise { diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 9e5624ca6a..7e831a7a1c 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -17,30 +17,75 @@ import { AppConfig } from '@backstage/config'; import { JSONSchema7 as JSONSchema } from 'json-schema'; +/** + * An sub-set of configuration schema. + */ export type ConfigSchemaPackageEntry = { + /** + * The configuration schema itself, as JSONSchema draft-07 + */ value: JSONSchema; + /** + * The path that the configuration schema was discovered at. + */ path: string; }; +/** + * A list of all possible configuration value visibilities. + */ export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const; +/** + * A type representing the possible configuration value visibilities + */ export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number]; +/** + * The default configuration visibility if no other values is given. + */ export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend'; +/** + * An explanation of a configuration validation error. + */ type ValidationError = string; +/** + * The result of validating configuration data using a schema. + */ type ValidationResult = { + /** + * Errors that where emitted during validation, if any. + */ errors?: ValidationError[]; + /** + * The configuration visibilities the where discovered during validation. + * + * The path in the key uses the form `////` + */ visibilityByPath: Map; }; +/** + * A function used validate configuration data. + */ export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; +/** + * Options used to process configuration data with a schema. + */ type ConfigProcessingOptions = { + /** + * The visibilities that should be included in the output data. + * If omitted, the data will not be filtered by visibility. + */ visibilities?: ConfigVisibility[]; }; +/** + * A loaded configuration schema that is ready to process configuration data. + */ export type ConfigSchema = { process( appConfigs: AppConfig[], From 4ccba1aa9620af569ec4c424806bafb510982570 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 14:59:35 +0100 Subject: [PATCH 11/51] config-loader: remove debugging output in schema lib --- packages/config-loader/src/lib/schema/load.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 286b363b50..46e1358ba9 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -30,15 +30,8 @@ type Options = { export async function loadConfigSchema( options: Options, ): Promise { - const start = process.hrtime(); - const schemas = await collectConfigSchemas(options.dependencies[0]); - const [durS, durNs] = process.hrtime(start); - const dur = (durS + durNs / 10 ** 9).toFixed(3); - console.log(`DEBUG: collected config schemas in ${dur}s`); - console.log('DEBUG: schemas =', schemas); - const validate = compileConfigSchemas(schemas); return { @@ -58,7 +51,6 @@ export async function loadConfigSchema( data: filterByVisibility(data, visibilities, result.visibilityByPath), })); } - console.log('DEBUG: result.visibilityByPath =', result.visibilityByPath); return processedConfigs; }, From f4af5d1dced13c29bf6fdb07fd024979453920d7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 15:22:28 +0100 Subject: [PATCH 12/51] config-loader: consider all deps when discovering schema and parallelize processing --- .../config-loader/src/lib/schema/collect.ts | 109 +++++++++++------- packages/config-loader/src/lib/schema/load.ts | 2 +- 2 files changed, 66 insertions(+), 45 deletions(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 25bdbe7f55..6e3755926c 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -18,6 +18,11 @@ import fs from 'fs-extra'; import { resolve as resolvePath, dirname } from 'path'; import { ConfigSchemaPackageEntry } from './types'; +type Item = { + name: string; + parentPath?: string; +}; + const req = typeof __non_webpack_require__ === 'undefined' ? require @@ -27,55 +32,71 @@ const req = * This collects all known config schemas across all dependencies of the app. */ export async function collectConfigSchemas( - name: string, - opts: unknown = {}, - visited: Set = new Set(), - schemas: ConfigSchemaPackageEntry[] = [], + packageNames: string[], ): Promise { - const pkgPath = req.resolve(`${name}/package.json`, opts); - if (visited.has(pkgPath)) { - return schemas; - } - visited.add(pkgPath); - const pkg = await fs.readJson(pkgPath); - const depNames = [ - ...Object.keys(pkg.dependencies ?? {}), - ...Object.keys(pkg.peerDependencies ?? {}), - ]; + const visitedPackages = new Set(); + const schemas = Array(); - // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, - // since that's pretty slow. We probably need a better way to determine when - // we've left the Backstage ecosystem, but this will do for now. - const hasSchema = 'configSchema' in pkg; - const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); - if (!hasSchema && !hasBackstageDep) { - return schemas; - } - if (hasSchema) { - if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { - throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, - ); - } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: pkgPath, - }); - } else { - schemas.push({ - value: pkg.configSchema, - path: pkgPath, - }); + async function process({ name, parentPath }: Item) { + // Ensures that we only process each package once. We don't bother with + // loading in schemas from duplicates of different versions, as that's not + // supported by Backstage right now anyway. We may want to change that in + // the future though, if it for example becomes possible to load in two + // different versions of e.g. @backstage/core at once. + if (visitedPackages.has(name)) { + return; } + visitedPackages.add(name); + + const pkgPath = req.resolve( + `${name}/package.json`, + parentPath && { + paths: [parentPath], + }, + ); + + const pkg = await fs.readJson(pkgPath); + const depNames = [ + ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.peerDependencies ?? {}), + ]; + + // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph, + // since that's pretty slow. We probably need a better way to determine when + // we've left the Backstage ecosystem, but this will do for now. + const hasSchema = 'configSchema' in pkg; + const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/')); + if (!hasSchema && !hasBackstageDep) { + return; + } + if (hasSchema) { + if (typeof pkg.configSchema === 'string') { + if (!pkg.configSchema.endsWith('.json')) { + throw new Error( + `Config schema files must be .json, got ${pkg.configSchema}`, + ); + } + const value = await fs.readJson( + resolvePath(dirname(pkgPath), pkg.configSchema), + ); + schemas.push({ + value, + path: pkgPath, + }); + } else { + schemas.push({ + value: pkg.configSchema, + path: pkgPath, + }); + } + } + + await Promise.all( + depNames.map(name => process({ name, parentPath: pkgPath })), + ); } - for (const depName of depNames) { - await collectConfigSchemas(depName, { paths: [pkgPath] }, visited, schemas); - } + await Promise.all(packageNames.map(name => process({ name }))); return schemas; } diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 46e1358ba9..6352da65af 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -30,7 +30,7 @@ type Options = { export async function loadConfigSchema( options: Options, ): Promise { - const schemas = await collectConfigSchemas(options.dependencies[0]); + const schemas = await collectConfigSchemas(options.dependencies); const validate = compileConfigSchemas(schemas); From 7298f8b26576042b05b37ec38fac7e6a8ac1f2b6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 15:30:03 +0100 Subject: [PATCH 13/51] config-loader: remove support for conditionally reading secrets --- packages/backend-common/src/config.ts | 1 - packages/cli/src/lib/config.ts | 6 +- packages/config-loader/src/lib/reader.test.ts | 19 ------ packages/config-loader/src/lib/reader.ts | 4 -- .../config-loader/src/lib/secrets.test.ts | 1 - packages/config-loader/src/lib/types.ts | 1 - packages/config-loader/src/loader.test.ts | 59 ++----------------- packages/config-loader/src/loader.ts | 27 +-------- 8 files changed, 7 insertions(+), 111 deletions(-) diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 69e46bd5b4..86beb65805 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -40,7 +40,6 @@ export async function loadBackendConfig(options: Options): Promise { env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development', configRoot: paths.targetRoot, configPaths: configOpts.map(opt => resolvePath(opt)), - shouldReadSecrets: true, }); options.logger.info( diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 5801632ea5..0ce19c80f5 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -18,14 +18,10 @@ import { loadConfig } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; -export async function loadCliConfig( - configArgs: string[], - shouldReadSecrets: boolean = false, -) { +export async function loadCliConfig(configArgs: string[]) { const configPaths = configArgs.map(arg => paths.resolveTarget(arg)); const appConfigs = await loadConfig({ - shouldReadSecrets, env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production', configRoot: paths.targetRoot, configPaths, diff --git a/packages/config-loader/src/lib/reader.test.ts b/packages/config-loader/src/lib/reader.test.ts index 6ccceef60d..a0a8495714 100644 --- a/packages/config-loader/src/lib/reader.test.ts +++ b/packages/config-loader/src/lib/reader.test.ts @@ -28,7 +28,6 @@ function memoryFiles(files: { [path: string]: string }) { const mockContext: ReaderContext = { env: {}, - skip: () => false, readFile: jest.fn(), readSecret: jest.fn(), }; @@ -179,22 +178,4 @@ describe('readConfigFile', () => { await expect(config).rejects.toThrow('Invalid secret at .app: NOPE'); }); - - it('should omit skipped values', async () => { - const readFile = memoryFiles({ - './app-config.yaml': 'app: { title: skip, name: include }', - }); - - const config = readConfigFile('./app-config.yaml', { - ...mockContext, - readFile, - skip: (path: string) => path === '.app.title', - readSecret: jest.fn() as ReadSecretFunc, - }); - - await expect(config).resolves.toEqual({ - context: 'app-config.yaml', - data: { app: { name: 'include' } }, - }); - }); }); diff --git a/packages/config-loader/src/lib/reader.ts b/packages/config-loader/src/lib/reader.ts index 8eadae0fe0..9eba58be97 100644 --- a/packages/config-loader/src/lib/reader.ts +++ b/packages/config-loader/src/lib/reader.ts @@ -37,10 +37,6 @@ export async function readConfigFile( obj: JsonValue, path: string, ): Promise { - if (ctx.skip(path)) { - return undefined; - } - if (typeof obj !== 'object') { return obj; } else if (obj === null) { diff --git a/packages/config-loader/src/lib/secrets.test.ts b/packages/config-loader/src/lib/secrets.test.ts index cfc4150d68..d80ada193b 100644 --- a/packages/config-loader/src/lib/secrets.test.ts +++ b/packages/config-loader/src/lib/secrets.test.ts @@ -21,7 +21,6 @@ const ctx: ReaderContext = { env: { SECRET: 'my-secret', }, - skip: () => false, readSecret: jest.fn(), async readFile(path) { const content = ({ diff --git a/packages/config-loader/src/lib/types.ts b/packages/config-loader/src/lib/types.ts index 02b8a9d053..e189aef20d 100644 --- a/packages/config-loader/src/lib/types.ts +++ b/packages/config-loader/src/lib/types.ts @@ -28,7 +28,6 @@ export type SkipFunc = (path: string) => boolean; */ export type ReaderContext = { env: { [name in string]?: string }; - skip: SkipFunc; readFile: ReadFileFunc; readSecret: ReadSecretFunc; }; diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3b0d8e6e92..a9857a784f 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -44,47 +44,6 @@ describe('loadConfig', () => { configRoot: '/root', configPaths: [], env: 'production', - shouldReadSecrets: false, - }), - ).resolves.toEqual([ - { - context: 'app-config.yaml', - data: { - app: { - title: 'Example App', - }, - }, - }, - ]); - }); - - it('loads config without secrets', async () => { - await expect( - loadConfig({ - configRoot: '/root', - configPaths: ['/root/app-config.yaml'], - env: 'production', - shouldReadSecrets: false, - }), - ).resolves.toEqual([ - { - context: 'app-config.yaml', - data: { - app: { - title: 'Example App', - }, - }, - }, - ]); - }); - - it('loads config with secrets', async () => { - await expect( - loadConfig({ - configRoot: '/root', - configPaths: ['/root/app-config.yaml'], - env: 'production', - shouldReadSecrets: true, }), ).resolves.toEqual([ { @@ -99,16 +58,12 @@ describe('loadConfig', () => { ]); }); - it('loads development config without secrets', async () => { + it('loads config with secrets', async () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', - ], - env: 'development', - shouldReadSecrets: false, + configPaths: ['/root/app-config.yaml'], + env: 'production', }), ).resolves.toEqual([ { @@ -116,15 +71,10 @@ describe('loadConfig', () => { data: { app: { title: 'Example App', + sessionKey: 'abc123', }, }, }, - { - context: 'app-config.development.yaml', - data: { - app: {}, - }, - }, ]); }); @@ -137,7 +87,6 @@ describe('loadConfig', () => { '/root/app-config.development.yaml', ], env: 'development', - shouldReadSecrets: true, }), ).resolves.toEqual([ { diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 00414f86eb..a647367469 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -28,18 +28,13 @@ export type LoadConfigOptions = { // TODO(Rugvip): This will be removed in the future, but for now we use it to warn about possible mistakes. env: string; - - // Whether to read secrets or omit them, defaults to false. - shouldReadSecrets?: boolean; }; class Context { constructor( private readonly options: { - secretPaths: Set; env: { [name in string]?: string }; rootPath: string; - shouldReadSecrets: boolean; }, ) {} @@ -47,35 +42,21 @@ class Context { return this.options.env; } - skip(path: string): boolean { - if (this.options.shouldReadSecrets) { - return false; - } - return this.options.secretPaths.has(path); - } - async readFile(path: string): Promise { return fs.readFile(resolvePath(this.options.rootPath, path), 'utf8'); } async readSecret( - path: string, + _path: string, desc: JsonObject, ): Promise { - this.options.secretPaths.add(path); - if (!this.options.shouldReadSecrets) { - return undefined; - } - return readSecret(desc, this); } } -type LoadedConfig = AppConfig[]; - export async function loadConfig( options: LoadConfigOptions, -): Promise { +): Promise { const configs = []; const { configRoot } = options; const configPaths = options.configPaths.slice(); @@ -102,8 +83,6 @@ export async function loadConfig( } try { - const secretPaths = new Set(); - for (const configPath of configPaths) { if (!isAbsolute(configPath)) { throw new Error(`Config load path is not absolute: '${configPath}'`); @@ -111,10 +90,8 @@ export async function loadConfig( const config = await readConfigFile( configPath, new Context({ - secretPaths, env: process.env, rootPath: dirname(configPath), - shouldReadSecrets: Boolean(options.shouldReadSecrets), }), ); From 564f438edd4a4c016e2ac73ff132c420ae371a69 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 8 Nov 2020 16:10:39 +0100 Subject: [PATCH 14/51] cli: load config schema and use to filter config + add --frontend flag to print --- packages/cli/package.json | 1 + packages/cli/src/commands/config/print.ts | 35 ++++++++++++++++++++--- packages/cli/src/commands/index.ts | 1 + packages/cli/src/lib/bundler/bundle.ts | 2 +- packages/cli/src/lib/bundler/config.ts | 10 +++---- packages/cli/src/lib/bundler/server.ts | 2 +- packages/cli/src/lib/bundler/types.ts | 12 ++++---- packages/cli/src/lib/config.ts | 20 +++++++++++-- 8 files changed, 64 insertions(+), 19 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 317e78ca16..66c56dd8cc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -76,6 +76,7 @@ "jest": "^26.0.1", "jest-css-modules": "^2.1.0", "jest-esm-transformer": "^1.0.0", + "lodash": "^4.17.19", "mini-css-extract-plugin": "^0.9.0", "ora": "^4.0.3", "raw-loader": "^4.0.1", diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index eb814a5bee..ce0d9976b1 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -16,16 +16,43 @@ import { Command } from 'commander'; import { stringify as stringifyYaml } from 'yaml'; +import { ConfigReader } from '@backstage/config'; import { loadCliConfig } from '../../lib/config'; +import cloneDeepWith from 'lodash/cloneDeepWith'; export default async (cmd: Command) => { - const { config } = await loadCliConfig(cmd.config, cmd.withSecrets ?? false); + const { schema, appConfigs } = await loadCliConfig(cmd.config); + let data; - const flatConfig = config.get(); + if (cmd.frontend) { + const frontendConfigs = schema.process(appConfigs, { + visibilities: ['frontend'], + }); + data = ConfigReader.fromConfigs(frontendConfigs).get(); + } else if (cmd.withSecrets) { + data = ConfigReader.fromConfigs(appConfigs).get(); + } else { + let secretConfigs = schema.process(appConfigs, { + visibilities: ['secret'], + }); + + secretConfigs = secretConfigs.map(entry => ({ + context: entry.context, + data: cloneDeepWith(entry.data, value => { + console.log('DEBUG: value =', value); + if (typeof value !== 'object') { + return ''; + } + return undefined; + }), + })); + + data = ConfigReader.fromConfigs(appConfigs.concat(secretConfigs)).get(); + } if (cmd.format === 'json') { - process.stdout.write(`${JSON.stringify(flatConfig, null, 2)}\n`); + process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); } else { - process.stdout.write(`${stringifyYaml(flatConfig)}\n`); + process.stdout.write(`${stringifyYaml(data)}\n`); } }; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 5b0dae6244..bcbde20d84 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -136,6 +136,7 @@ export function registerCommands(program: CommanderStatic) { program .command('config:print') + .option('--frontend', 'Print only the frontend configuration') .option('--with-secrets', 'Include secrets in the printed configuration') .option( '--format ', diff --git a/packages/cli/src/lib/bundler/bundle.ts b/packages/cli/src/lib/bundler/bundle.ts index 8746c807e7..a5d581c60a 100644 --- a/packages/cli/src/lib/bundler/bundle.ts +++ b/packages/cli/src/lib/bundler/bundle.ts @@ -40,7 +40,7 @@ export async function buildBundle(options: BuildOptions) { ...options, checksEnabled: false, isDev: false, - baseUrl: resolveBaseUrl(options.config), + baseUrl: resolveBaseUrl(options.frontendConfig), }); const compiler = webpack(config); diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 34b9f498ad..9e150e0ccb 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -74,11 +74,11 @@ export async function createConfig( paths: BundlingPaths, options: BundlingOptions, ): Promise { - const { checksEnabled, isDev } = options; + const { checksEnabled, isDev, frontendConfig } = options; const { plugins, loaders } = transforms(options); - const baseUrl = options.config.getString('app.baseUrl'); + const baseUrl = frontendConfig.getString('app.baseUrl'); const validBaseUrl = new URL(baseUrl); if (checksEnabled) { @@ -99,7 +99,7 @@ export async function createConfig( plugins.push( new webpack.EnvironmentPlugin({ - APP_CONFIG: options.appConfigs, + APP_CONFIG: options.frontendAppConfigs, }), ); @@ -109,9 +109,9 @@ export async function createConfig( templateParameters: { publicPath: validBaseUrl.pathname.replace(/\/$/, ''), app: { - title: options.config.getString('app.title'), + title: frontendConfig.getString('app.title'), baseUrl: validBaseUrl.href, - googleAnalyticsTrackingId: options.config.getOptionalString( + googleAnalyticsTrackingId: frontendConfig.getOptionalString( 'app.googleAnalyticsTrackingId', ), }, diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index 2710145d83..f540e01084 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -23,7 +23,7 @@ import { ServeOptions } from './types'; import { resolveBundlingPaths } from './paths'; export async function serveBundle(options: ServeOptions) { - const url = resolveBaseUrl(options.config); + const url = resolveBaseUrl(options.frontendConfig); const host = options.config.getOptionalString('app.listen.host') || url.hostname; diff --git a/packages/cli/src/lib/bundler/types.ts b/packages/cli/src/lib/bundler/types.ts index 00f1895725..d0af0a9937 100644 --- a/packages/cli/src/lib/bundler/types.ts +++ b/packages/cli/src/lib/bundler/types.ts @@ -21,23 +21,23 @@ import { ParallelOption } from '../parallel'; export type BundlingOptions = { checksEnabled: boolean; isDev: boolean; - config: Config; - appConfigs: AppConfig[]; + frontendConfig: Config; + frontendAppConfigs: AppConfig[]; baseUrl: URL; parallel?: ParallelOption; }; export type ServeOptions = BundlingPathsOptions & { checksEnabled: boolean; - config: Config; - appConfigs: AppConfig[]; + frontendConfig: Config; + frontendAppConfigs: AppConfig[]; }; export type BuildOptions = BundlingPathsOptions & { statsJsonEnabled: boolean; parallel?: ParallelOption; - config: Config; - appConfigs: AppConfig[]; + frontendConfig: Config; + frontendAppConfigs: AppConfig[]; }; export type BackendBundlingOptions = { diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 0ce19c80f5..a282bbf88c 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,13 +14,22 @@ * limitations under the License. */ -import { loadConfig } from '@backstage/config-loader'; +import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; export async function loadCliConfig(configArgs: string[]) { const configPaths = configArgs.map(arg => paths.resolveTarget(arg)); + // Consider all packages in the monorepo when loading in config + const LernaProject = require('@lerna/project'); + const project = new LernaProject(paths.targetDir); + const packages = await project.getPackages(); + const localPackageNames = packages.map((p: any) => p.name); + const schema = await loadConfigSchema({ + dependencies: localPackageNames, + }); + const appConfigs = await loadConfig({ env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production', configRoot: paths.targetRoot, @@ -31,8 +40,15 @@ export async function loadCliConfig(configArgs: string[]) { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); + const frontendAppConfigs = schema.process(appConfigs, { + visibilities: ['frontend'], + }); + const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); + return { + schema, appConfigs, - config: ConfigReader.fromConfigs(appConfigs), + frontendConfig, + frontendAppConfigs, }; } From 3849e24112fefdea9a9a28d56176472b7c80a4e6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 00:19:30 +0100 Subject: [PATCH 15/51] config-loader: add support for serializing and deserializing schema --- packages/config-loader/src/index.ts | 1 + packages/config-loader/src/lib/index.ts | 2 +- .../config-loader/src/lib/schema/collect.ts | 17 ++++++---- .../config-loader/src/lib/schema/index.ts | 1 + packages/config-loader/src/lib/schema/load.ts | 34 +++++++++++++++---- .../config-loader/src/lib/schema/types.ts | 7 ++-- 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 415b6d4022..b08380d8ce 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,5 +15,6 @@ */ export { readEnvConfig, loadConfigSchema } from './lib'; +export type { ConfigSchema } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 4da0c97788..ceb7c34222 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,4 +17,4 @@ export { readConfigFile } from './reader'; export { readEnvConfig } from './env'; export { readSecret } from './secrets'; -export { loadConfigSchema } from './schema'; +export * from './schema'; diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 6e3755926c..f7fed94748 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -15,7 +15,11 @@ */ import fs from 'fs-extra'; -import { resolve as resolvePath, dirname } from 'path'; +import { + resolve as resolvePath, + relative as relativePath, + dirname, +} from 'path'; import { ConfigSchemaPackageEntry } from './types'; type Item = { @@ -36,8 +40,9 @@ export async function collectConfigSchemas( ): Promise { const visitedPackages = new Set(); const schemas = Array(); + const currentDir = process.cwd(); - async function process({ name, parentPath }: Item) { + async function processItem({ name, parentPath }: Item) { // Ensures that we only process each package once. We don't bother with // loading in schemas from duplicates of different versions, as that's not // supported by Backstage right now anyway. We may want to change that in @@ -81,22 +86,22 @@ export async function collectConfigSchemas( ); schemas.push({ value, - path: pkgPath, + path: relativePath(currentDir, pkgPath), }); } else { schemas.push({ value: pkg.configSchema, - path: pkgPath, + path: relativePath(currentDir, pkgPath), }); } } await Promise.all( - depNames.map(name => process({ name, parentPath: pkgPath })), + depNames.map(name => processItem({ name, parentPath: pkgPath })), ); } - await Promise.all(packageNames.map(name => process({ name }))); + await Promise.all(packageNames.map(name => processItem({ name }))); return schemas; } diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index 990c0dc182..bc3c7d8ebc 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -15,3 +15,4 @@ */ export { loadConfigSchema } from './load'; +export type { ConfigSchema } from './types'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 6352da65af..cdfc8f9549 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -14,15 +14,19 @@ * limitations under the License. */ -import { AppConfig } from '@backstage/config'; +import { AppConfig, JsonObject } from '@backstage/config'; import { compileConfigSchemas } from './compile'; import { collectConfigSchemas } from './collect'; import { filterByVisibility } from './filtering'; -import { ConfigSchema } from './types'; +import { ConfigSchema, ConfigSchemaPackageEntry } from './types'; -type Options = { - dependencies: string[]; -}; +type Options = + | { + dependencies: string[]; + } + | { + serialized: JsonObject; + }; /** * Loads config schema for a Backstage instance. @@ -30,7 +34,19 @@ type Options = { export async function loadConfigSchema( options: Options, ): Promise { - const schemas = await collectConfigSchemas(options.dependencies); + let schemas: ConfigSchemaPackageEntry[]; + + if ('dependencies' in options) { + schemas = await collectConfigSchemas(options.dependencies); + } else { + const { serialized } = options; + if (serialized?.backstageConfigSchemaVersion !== 1) { + throw new Error( + 'Serialized configuration schema is invalid or has an invalid version number', + ); + } + schemas = serialized.schemas as ConfigSchemaPackageEntry[]; + } const validate = compileConfigSchemas(schemas); @@ -54,5 +70,11 @@ export async function loadConfigSchema( return processedConfigs; }, + serialize(): JsonObject { + return { + schemas, + backstageConfigSchemaVersion: 1, + }; + }, }; } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 7e831a7a1c..cec2da0160 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -14,8 +14,7 @@ * limitations under the License. */ -import { AppConfig } from '@backstage/config'; -import { JSONSchema7 as JSONSchema } from 'json-schema'; +import { AppConfig, JsonObject } from '@backstage/config'; /** * An sub-set of configuration schema. @@ -24,7 +23,7 @@ export type ConfigSchemaPackageEntry = { /** * The configuration schema itself, as JSONSchema draft-07 */ - value: JSONSchema; + value: JsonObject; /** * The path that the configuration schema was discovered at. */ @@ -91,4 +90,6 @@ export type ConfigSchema = { appConfigs: AppConfig[], options?: ConfigProcessingOptions, ): AppConfig[]; + + serialize(): JsonObject; }; From e43aa0dc60012aa11c68a8dfd3d40c27381bdfcd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 00:27:24 +0100 Subject: [PATCH 16/51] cli: write serialized config schema to app output --- packages/cli/src/lib/bundler/bundle.ts | 10 +++++++++- packages/cli/src/lib/bundler/types.ts | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/lib/bundler/bundle.ts b/packages/cli/src/lib/bundler/bundle.ts index a5d581c60a..30fef2e8b7 100644 --- a/packages/cli/src/lib/bundler/bundle.ts +++ b/packages/cli/src/lib/bundler/bundle.ts @@ -33,7 +33,7 @@ const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024; const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024; export async function buildBundle(options: BuildOptions) { - const { statsJsonEnabled } = options; + const { statsJsonEnabled, schema: configSchema } = options; const paths = resolveBundlingPaths(options); const config = await createConfig(paths, { @@ -56,6 +56,14 @@ export async function buildBundle(options: BuildOptions) { }); } + if (configSchema) { + await fs.writeJson( + resolvePath(paths.targetDist, '.config-schema.json'), + configSchema.serialize(), + { spaces: 2 }, + ); + } + const { stats } = await build(compiler, isCi).catch(error => { console.log(chalk.red('Failed to compile.\n')); throw new Error(`Failed to compile.\n${error.message || error}`); diff --git a/packages/cli/src/lib/bundler/types.ts b/packages/cli/src/lib/bundler/types.ts index d0af0a9937..71343a1761 100644 --- a/packages/cli/src/lib/bundler/types.ts +++ b/packages/cli/src/lib/bundler/types.ts @@ -17,6 +17,7 @@ import { AppConfig, Config } from '@backstage/config'; import { BundlingPathsOptions } from './paths'; import { ParallelOption } from '../parallel'; +import { ConfigSchema } from '@backstage/config-loader'; export type BundlingOptions = { checksEnabled: boolean; @@ -36,6 +37,7 @@ export type ServeOptions = BundlingPathsOptions & { export type BuildOptions = BundlingPathsOptions & { statsJsonEnabled: boolean; parallel?: ParallelOption; + schema?: ConfigSchema; frontendConfig: Config; frontendAppConfigs: AppConfig[]; }; From 2e4de993383b72be6ee93bdf6ce60aa30070ce7a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 10:11:50 +0100 Subject: [PATCH 17/51] config-loader: switch to ajv v6 --- packages/config-loader/package.json | 2 +- .../config-loader/src/lib/schema/compile.ts | 40 ++++++++----------- .../src/lib/schema/filtering.test.ts | 36 ++++++++--------- .../config-loader/src/lib/schema/filtering.ts | 4 +- yarn.lock | 14 ------- 5 files changed, 38 insertions(+), 58 deletions(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 39735ff2fd..32776d71b7 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -34,7 +34,7 @@ "@backstage/config": "^0.1.1", "@types/json-schema": "^7.0.6", "@types/json-schema-merge-allof": "^0.6.0", - "ajv": "^7.0.0-beta.2", + "ajv": "^6.12.5", "fs-extra": "^9.0.0", "json-schema": "^0.2.5", "json-schema-merge-allof": "^0.7.0", diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 76642690d5..f63dc3e636 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -41,33 +41,27 @@ export function compileConfigSchemas( const visibilityByPath = new Map(); const ajv = new Ajv({ - strict: true, allErrors: true, - defaultMeta: 'http://json-schema.org/draft-07/schema#', schemas: { 'https://backstage.io/schema/config-v1': true, }, - keywords: [ - { - keyword: 'visibility', - schemaType: 'string', - metaSchema: { - type: 'string', - enum: CONFIG_VISIBILITIES, - }, - compile(visibility: ConfigVisibility) { - return (_data, ctx) => { - if (!ctx) { - return false; - } - if (visibility) { - visibilityByPath.set(ctx.dataPath, visibility); - } - return true; - }; - }, - }, - ], + }).addKeyword('visibility', { + type: 'string', + metaSchema: { + type: 'string', + enum: CONFIG_VISIBILITIES, + }, + compile(visibility: ConfigVisibility) { + return (_data, dataPath) => { + if (!dataPath) { + return false; + } + if (visibility) { + visibilityByPath.set(dataPath, visibility); + } + return true; + }; + }, }); const merged = mergeAllOf( diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 8cc907417b..191edf4824 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -40,24 +40,24 @@ const data = { const visibilities = new Map( Object.entries({ - '/arr/0': 'frontend', - '/arr/1': 'backend', - '/arr/2': 'secret', - '/obj/f': 'frontend', - '/obj/b': 'backend', - '/obj/b/s': 'secret', - '/objArr/0/f': 'frontend', - '/objArr/0/b': 'backend', - '/objArr/0/s': 'secret', - '/objArr/1/f': 'frontend', - '/objArr/1/b': 'backend', - '/objArr/1/s': 'secret', - '/arrF': 'frontend', - '/arrB': 'backend', - '/arrS': 'secret', - '/objF': 'frontend', - '/objB': 'backend', - '/objS': 'secret', + '.arr[0]': 'frontend', + '.arr[1]': 'backend', + '.arr[2]': 'secret', + '.obj.f': 'frontend', + '.obj.b': 'backend', + '.obj.b.s': 'secret', + '.objArr[0].f': 'frontend', + '.objArr[0].b': 'backend', + '.objArr[0].s': 'secret', + '.objArr[1].f': 'frontend', + '.objArr[1].b': 'backend', + '.objArr[1].s': 'secret', + '.arrF': 'frontend', + '.arrB': 'backend', + '.arrS': 'secret', + '.objF': 'frontend', + '.objB': 'backend', + '.objS': 'secret', }), ); diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index d6dcaf2dd7..908e9e4739 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -39,7 +39,7 @@ export function filterByVisibility( const arr = new Array(); for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}/${index}`); + const out = transform(value, `${path}[${index}]`); if (out !== undefined) { arr.push(out); } @@ -58,7 +58,7 @@ export function filterByVisibility( if (value === undefined) { continue; } - const out = transform(value, `${path}/${key}`); + const out = transform(value, `${path}.${key}`); if (out !== undefined) { outObj[key] = out; hasOutput = true; diff --git a/yarn.lock b/yarn.lock index 98e1b2476f..a85e24210f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6569,15 +6569,6 @@ ajv@^6.1.0, ajv@^6.10.0, ajv@^6.10.1, ajv@^6.10.2, ajv@^6.12.4, ajv@^6.12.5, ajv json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^7.0.0-beta.2: - version "7.0.0-beta.3" - resolved "https://registry.npmjs.org/ajv/-/ajv-7.0.0-beta.3.tgz#d34861ccfbdebb55bf9f49a08b29f76bf656fc5c" - integrity sha512-gUGVvM4NmyqrFvCNAQnP4P7FC0RjxMQyRnrXpozNglBkDJnTysVbvycyOZUy5n6yLKSqVDUqWZBXj7dXINrSqw== - dependencies: - fast-deep-equal "^3.1.1" - json-schema-traverse "^0.5.0" - uri-js "^4.2.2" - alphanum-sort@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3" @@ -15052,11 +15043,6 @@ json-schema-traverse@^0.4.1: resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== -json-schema-traverse@^0.5.0: - version "0.5.0" - resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.5.0.tgz#1069a6097f9e9a567bfc6cc215e85f606ebf6480" - integrity sha512-x+TRJIQFskrNnFKE2Viz9FCSjK1vIh+H/uaBiOYszh/IcZmAFneQ35H4osWDJp1NPXccuV2I0RMXmi2ZS6Kqcg== - json-schema@0.2.3: version "0.2.3" resolved "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" From 7f95e40fbb8e43cd75a9e61f62ac6fda728037a6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 10:28:11 +0100 Subject: [PATCH 18/51] app-backend,cli: add support for injecting full config based on serialized schema --- packages/backend/src/plugins/app.ts | 6 ++- packages/core/src/api-wrappers/createApp.tsx | 6 ++- plugins/app-backend/package.json | 1 + plugins/app-backend/src/lib/config.test.ts | 26 +++++------ plugins/app-backend/src/lib/config.ts | 46 +++++++++++++------ .../app-backend/src/service/router.test.ts | 4 +- plugins/app-backend/src/service/router.ts | 25 ++++++---- .../src/service/standaloneServer.ts | 5 +- 8 files changed, 79 insertions(+), 40 deletions(-) diff --git a/packages/backend/src/plugins/app.ts b/packages/backend/src/plugins/app.ts index c9f7c0622a..637af80974 100644 --- a/packages/backend/src/plugins/app.ts +++ b/packages/backend/src/plugins/app.ts @@ -17,9 +17,13 @@ import { createRouter } from '@backstage/plugin-app-backend'; import { PluginEnvironment } from '../types'; -export default async function createPlugin({ logger }: PluginEnvironment) { +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { return await createRouter({ logger, + config, appPackageName: 'example-app', }); } diff --git a/packages/core/src/api-wrappers/createApp.tsx b/packages/core/src/api-wrappers/createApp.tsx index 592a2417ca..4a1e58db42 100644 --- a/packages/core/src/api-wrappers/createApp.tsx +++ b/packages/core/src/api-wrappers/createApp.tsx @@ -61,7 +61,11 @@ export const defaultConfigLoader: AppConfigLoader = async ( if (runtimeConfigJson !== '__app_injected_runtime_config__'.toUpperCase()) { try { const data = JSON.parse(runtimeConfigJson) as JsonObject; - configs.push({ data, context: 'env' }); + if (Array.isArray(data)) { + configs.push(...data); + } else { + configs.push({ data, context: 'env' }); + } } catch (error) { throw new Error(`Failed to load runtime configuration, ${error}`); } diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index c9b82cf6f1..08266acb68 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -22,6 +22,7 @@ "dependencies": { "@backstage/backend-common": "^0.2.0", "@backstage/config-loader": "^0.2.0", + "@backstage/config": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", diff --git a/plugins/app-backend/src/lib/config.test.ts b/plugins/app-backend/src/lib/config.test.ts index 16c5600b1d..475f72f3b1 100644 --- a/plugins/app-backend/src/lib/config.test.ts +++ b/plugins/app-backend/src/lib/config.test.ts @@ -17,7 +17,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'path'; import { getVoidLogger } from '@backstage/backend-common'; -import { injectEnvConfig } from './config'; +import { injectConfig } from './config'; jest.mock('fs-extra'); @@ -29,12 +29,12 @@ const readFileMock = (fsMock.readFile as unknown) as jest.MockedFunction< const MOCK_DIR = 'mock-dir'; const baseOptions = { - env: {}, + appConfigs: [], staticDir: MOCK_DIR, logger: getVoidLogger(), }; -describe('injectEnvConfig', () => { +describe('injectConfig', () => { beforeEach(() => { fsMock.readdir.mockResolvedValue(['main.js']); }); @@ -44,7 +44,7 @@ describe('injectEnvConfig', () => { }); it('should not inject without config', async () => { - await injectEnvConfig(baseOptions); + await injectConfig(baseOptions); expect(fsMock.readdir).toHaveBeenCalledTimes(0); expect(fsMock.readFile).toHaveBeenCalledTimes(0); expect(fsMock.writeFile).toHaveBeenCalledTimes(0); @@ -64,7 +64,10 @@ describe('injectEnvConfig', () => { return 'NO_PLACEHOLDER_HERE'; }); - await injectEnvConfig({ ...baseOptions, env: { APP_CONFIG_x: '0' } }); + await injectConfig({ + ...baseOptions, + appConfigs: [{ data: { x: 0 }, context: 'test' }], + }); expect(fsMock.readFile).toHaveBeenCalledTimes(2); expect(fsMock.readFile).toHaveBeenNthCalledWith( 1, @@ -96,11 +99,9 @@ describe('injectEnvConfig', () => { 'JSON.parse("__APP_INJECTED_RUNTIME_CONFIG__")', ); - await injectEnvConfig({ + await injectConfig({ ...baseOptions, - env: { - APP_CONFIG_x: '0', - }, + appConfigs: [{ data: { x: 0 }, context: 'test' }], }); expect(fsMock.writeFile).toHaveBeenCalledTimes(1); @@ -115,12 +116,9 @@ describe('injectEnvConfig', () => { readFileMock.mockResolvedValue(fsMock.writeFile.mock.calls[0][1]); - await injectEnvConfig({ + await injectConfig({ ...baseOptions, - env: { - APP_CONFIG_x: '1', - APP_CONFIG_y: '2', - }, + appConfigs: [{ data: { x: 1, y: 2 }, context: 'test' }], }); expect(fsMock.writeFile).toHaveBeenCalledTimes(2); diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 0ed56ef722..60b5d28806 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -16,12 +16,12 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'path'; -import { readEnvConfig } from '@backstage/config-loader'; import { Logger } from 'winston'; +import { AppConfig, Config, JsonObject } from '@backstage/config'; +import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader'; -type Options = { - // Environment to read config from - env: { [name: string]: string | undefined }; +type InjectOptions = { + appConfigs: AppConfig[]; // Directory of the static JS files to search for file to inject staticDir: string; logger: Logger; @@ -31,19 +31,13 @@ type Options = { * Injects config from APP_CONFIG_ env vars, replacing existing * injected config if it has already been injected. */ -export async function injectEnvConfig(options: Options) { - const { env, staticDir, logger } = options; - - const envConfig = readEnvConfig(env); - if (envConfig.length === 0) { - return; - } +export async function injectConfig(options: InjectOptions) { + const { staticDir, logger, appConfigs } = options; const files = await fs.readdir(staticDir); const jsFiles = files.filter(file => file.endsWith('.js')); - const [{ data }] = envConfig; - const escapedData = JSON.stringify(data).replace(/("|'|\\)/g, '\\$1'); + const escapedData = JSON.stringify(appConfigs).replace(/("|'|\\)/g, '\\$1'); const injected = `/*__APP_INJECTED_CONFIG_MARKER__*/"${escapedData}"/*__INJECTED_END__*/`; for (const jsFile of jsFiles) { @@ -72,3 +66,29 @@ export async function injectEnvConfig(options: Options) { } logger.info('Env config not injected'); } + +type ReadOptions = { + env: { [name: string]: string | undefined }; + appDistDir: string; + config: Config; +}; + +export async function readConfigs(options: ReadOptions): Promise { + const { env, appDistDir, config } = options; + + const appConfigs = readEnvConfig(env); + + const schemaPath = resolvePath(appDistDir, '.config-schema.json'); + if (await fs.pathExists(schemaPath)) { + const serializedSchema = await fs.readJson(schemaPath); + const schema = await loadConfigSchema({ serialized: serializedSchema }); + + const frontendConfigs = await schema.process( + [{ data: config.get() as JsonObject, context: 'app' }], + { visibilities: ['frontend'] }, + ); + appConfigs.push(...frontendConfigs); + } + + return appConfigs; +} diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts index aed4280ca5..b0cc3c59f7 100644 --- a/plugins/app-backend/src/service/router.test.ts +++ b/plugins/app-backend/src/service/router.test.ts @@ -16,10 +16,10 @@ import { resolve as resolvePath } from 'path'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; import request from 'supertest'; - import { createRouter } from './router'; jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() })); @@ -35,6 +35,7 @@ describe('createRouter', () => { beforeAll(async () => { const router = await createRouter({ logger: getVoidLogger(), + config: new ConfigReader({}), appPackageName: 'example-app', }); app = express().use(router); @@ -76,6 +77,7 @@ describe('createRouter with static fallback handler', () => { const router = await createRouter({ logger: getVoidLogger(), + config: new ConfigReader({}), appPackageName: 'example-app', staticFallbackHandler, }); diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index c9a06e4dda..b435ec15f1 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -15,13 +15,15 @@ */ import { resolve as resolvePath } from 'path'; -import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { injectEnvConfig } from '../lib/config'; +import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; +import { injectConfig, readConfigs } from '../lib/config'; export interface RouterOptions { + config: Config; logger: Logger; appPackageName: string; staticFallbackHandler?: express.Handler; @@ -30,22 +32,27 @@ export interface RouterOptions { export async function createRouter( options: RouterOptions, ): Promise { - const appDistDir = resolvePackagePath(options.appPackageName, 'dist'); - options.logger.info(`Serving static app content from ${appDistDir}`); + const { config, logger, appPackageName, staticFallbackHandler } = options; - await injectEnvConfig({ + const appDistDir = resolvePackagePath(appPackageName, 'dist'); + logger.info(`Serving static app content from ${appDistDir}`); + const staticDir = resolvePath(appDistDir, 'static'); + + const appConfigs = await readConfigs({ + config, + appDistDir, env: process.env, - logger: options.logger, - staticDir: resolvePath(appDistDir, 'static'), }); + await injectConfig({ appConfigs, logger, staticDir }); + const router = Router(); // Use a separate router for static content so that a fallback can be provided by backend const staticRouter = Router(); staticRouter.use(express.static(resolvePath(appDistDir, 'static'))); - if (options.staticFallbackHandler) { - staticRouter.use(options.staticFallbackHandler); + if (staticFallbackHandler) { + staticRouter.use(staticFallbackHandler); } staticRouter.use(notFoundHandler()); diff --git a/plugins/app-backend/src/service/standaloneServer.ts b/plugins/app-backend/src/service/standaloneServer.ts index 8abf3b81f2..005d80027b 100644 --- a/plugins/app-backend/src/service/standaloneServer.ts +++ b/plugins/app-backend/src/service/standaloneServer.ts @@ -14,14 +14,16 @@ * limitations under the License. */ -import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; +import { createServiceBuilder } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; import { createRouter } from './router'; export interface ServerOptions { port: number; enableCors: boolean; + config: Config; logger: Logger; } @@ -32,6 +34,7 @@ export async function startStandaloneServer( logger.debug('Starting application server...'); const router = await createRouter({ logger, + config: options.config, appPackageName: 'example-app', }); From eff7b6e3ba05901a44cc99f836535669f222b857 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 9 Nov 2020 10:32:25 +0100 Subject: [PATCH 19/51] cli: remove debug print --- packages/cli/src/commands/config/print.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index ce0d9976b1..66b709e601 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -35,11 +35,9 @@ export default async (cmd: Command) => { let secretConfigs = schema.process(appConfigs, { visibilities: ['secret'], }); - secretConfigs = secretConfigs.map(entry => ({ context: entry.context, data: cloneDeepWith(entry.data, value => { - console.log('DEBUG: value =', value); if (typeof value !== 'object') { return ''; } From c18769cc00dd85017886cd03ca22a58f3ddbb9b6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 14:21:53 +0100 Subject: [PATCH 20/51] config-loader: visibilities -> visiblity --- packages/cli/src/commands/config/print.ts | 4 ++-- packages/cli/src/lib/config.ts | 2 +- packages/config-loader/src/lib/schema/filtering.test.ts | 4 ++-- packages/config-loader/src/lib/schema/load.ts | 6 +++--- packages/config-loader/src/lib/schema/types.ts | 2 +- plugins/app-backend/src/lib/config.ts | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index 66b709e601..246847b610 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -26,14 +26,14 @@ export default async (cmd: Command) => { if (cmd.frontend) { const frontendConfigs = schema.process(appConfigs, { - visibilities: ['frontend'], + visiblity: ['frontend'], }); data = ConfigReader.fromConfigs(frontendConfigs).get(); } else if (cmd.withSecrets) { data = ConfigReader.fromConfigs(appConfigs).get(); } else { let secretConfigs = schema.process(appConfigs, { - visibilities: ['secret'], + visiblity: ['secret'], }); secretConfigs = secretConfigs.map(entry => ({ context: entry.context, diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index a282bbf88c..63e3958a6f 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -41,7 +41,7 @@ export async function loadCliConfig(configArgs: string[]) { ); const frontendAppConfigs = schema.process(appConfigs, { - visibilities: ['frontend'], + visiblity: ['frontend'], }); const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 191edf4824..8e3fb61ae5 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -38,7 +38,7 @@ const data = { objS: { never: 'here' }, }; -const visibilities = new Map( +const visiblity = new Map( Object.entries({ '.arr[0]': 'frontend', '.arr[1]': 'backend', @@ -100,6 +100,6 @@ describe('filterByVisibility', () => { ], [['frontend', 'backend', 'secret'], data], ])('should filter correctly with %p', (filter, expected) => { - expect(filterByVisibility(data, filter, visibilities)).toEqual(expected); + expect(filterByVisibility(data, filter, visiblity)).toEqual(expected); }); }); diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index cdfc8f9549..d886926647 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -51,7 +51,7 @@ export async function loadConfigSchema( const validate = compileConfigSchemas(schemas); return { - process(configs: AppConfig[], { visibilities } = {}): AppConfig[] { + process(configs: AppConfig[], { visiblity } = {}): AppConfig[] { const result = validate(configs); if (result.errors) { throw new Error( @@ -61,10 +61,10 @@ export async function loadConfigSchema( let processedConfigs = configs; - if (visibilities) { + if (visiblity) { processedConfigs = processedConfigs.map(({ data, context }) => ({ context, - data: filterByVisibility(data, visibilities, result.visibilityByPath), + data: filterByVisibility(data, visiblity, result.visibilityByPath), })); } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index cec2da0160..6ab0602fb3 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -79,7 +79,7 @@ type ConfigProcessingOptions = { * The visibilities that should be included in the output data. * If omitted, the data will not be filtered by visibility. */ - visibilities?: ConfigVisibility[]; + visiblity?: ConfigVisibility[]; }; /** diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 60b5d28806..82b431c33f 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -85,7 +85,7 @@ export async function readConfigs(options: ReadOptions): Promise { const frontendConfigs = await schema.process( [{ data: config.get() as JsonObject, context: 'app' }], - { visibilities: ['frontend'] }, + { visiblity: ['frontend'] }, ); appConfigs.push(...frontendConfigs); } From 051e0fd17e7b58cbfc8a0f548f8b891139f5859e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 14:28:48 +0100 Subject: [PATCH 21/51] config-loader: hasApp -> hasFrontend --- packages/config-loader/src/lib/schema/compile.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index f63dc3e636..7dc65be910 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -75,15 +75,15 @@ export function compileConfigSchemas( // This ensures that the visibilities across different schemas are sound, and // selects the most specific visibility for each path. visibility(values: string[], path: string[]) { - const hasApp = values.some(_ => _ === 'frontend'); + const hasFrontend = values.some(_ => _ === 'frontend'); const hasSecret = values.some(_ => _ === 'secret'); - if (hasApp && hasSecret) { + if (hasFrontend && hasSecret) { throw new Error( `Config schema visibility is both 'frontend' and 'secret' for ${path.join( '/', )}`, ); - } else if (hasApp) { + } else if (hasFrontend) { return 'frontend'; } else if (hasSecret) { return 'secret'; From 49ab961b8bb659336aead47f786740a9959b6fb6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 15:48:02 +0100 Subject: [PATCH 22/51] config-loader: add support for transforming values during schema processing --- packages/config-loader/src/index.ts | 2 +- .../config-loader/src/lib/schema/filtering.ts | 20 +++++++++---- .../config-loader/src/lib/schema/index.ts | 2 +- packages/config-loader/src/lib/schema/load.ts | 28 +++++++++++++++++-- .../config-loader/src/lib/schema/types.ts | 16 +++++++++++ 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index b08380d8ce..9ad54c5f18 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -15,6 +15,6 @@ */ export { readEnvConfig, loadConfigSchema } from './lib'; -export type { ConfigSchema } from './lib'; +export type { ConfigSchema, ConfigVisibility } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 908e9e4739..74e75f465d 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -15,7 +15,11 @@ */ import { JsonObject, JsonValue } from '@backstage/config'; -import { ConfigVisibility, DEFAULT_CONFIG_VISIBILITY } from './types'; +import { + ConfigVisibility, + DEFAULT_CONFIG_VISIBILITY, + TransformFunc, +} from './types'; /** * This filters data by visibility by discovering the visibility of each @@ -25,14 +29,20 @@ export function filterByVisibility( data: JsonObject, includeVisibilities: ConfigVisibility[], visibilityByPath: Map, + transformFunc?: TransformFunc, ): JsonObject { function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { - const isVisible = includeVisibilities.includes( - visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY, - ); + const visibility = visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + const isVisible = includeVisibilities.includes(visibility); if (typeof jsonVal !== 'object') { - return isVisible ? jsonVal : undefined; + if (isVisible) { + if (transformFunc) { + return transformFunc(jsonVal, { visibility }); + } + return jsonVal; + } + return undefined; } else if (jsonVal === null) { return undefined; } else if (Array.isArray(jsonVal)) { diff --git a/packages/config-loader/src/lib/schema/index.ts b/packages/config-loader/src/lib/schema/index.ts index bc3c7d8ebc..8cefb93b3c 100644 --- a/packages/config-loader/src/lib/schema/index.ts +++ b/packages/config-loader/src/lib/schema/index.ts @@ -15,4 +15,4 @@ */ export { loadConfigSchema } from './load'; -export type { ConfigSchema } from './types'; +export type { ConfigSchema, ConfigVisibility } from './types'; diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index d886926647..010317849c 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -18,7 +18,11 @@ import { AppConfig, JsonObject } from '@backstage/config'; import { compileConfigSchemas } from './compile'; import { collectConfigSchemas } from './collect'; import { filterByVisibility } from './filtering'; -import { ConfigSchema, ConfigSchemaPackageEntry } from './types'; +import { + ConfigSchema, + ConfigSchemaPackageEntry, + CONFIG_VISIBILITIES, +} from './types'; type Options = | { @@ -51,7 +55,10 @@ export async function loadConfigSchema( const validate = compileConfigSchemas(schemas); return { - process(configs: AppConfig[], { visiblity } = {}): AppConfig[] { + process( + configs: AppConfig[], + { visiblity, valueTransform } = {}, + ): AppConfig[] { const result = validate(configs); if (result.errors) { throw new Error( @@ -64,7 +71,22 @@ export async function loadConfigSchema( if (visiblity) { processedConfigs = processedConfigs.map(({ data, context }) => ({ context, - data: filterByVisibility(data, visiblity, result.visibilityByPath), + data: filterByVisibility( + data, + visiblity, + result.visibilityByPath, + valueTransform, + ), + })); + } else if (valueTransform) { + processedConfigs = processedConfigs.map(({ data, context }) => ({ + context, + data: filterByVisibility( + data, + Array.from(CONFIG_VISIBILITIES), + result.visibilityByPath, + valueTransform, + ), })); } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 6ab0602fb3..c82036bbf4 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -71,6 +71,14 @@ type ValidationResult = { */ export type ValidationFunc = (configs: AppConfig[]) => ValidationResult; +/** + * A function used to transform primitive configuration values. + */ +export type TransformFunc = ( + value: T, + context: { visibility: ConfigVisibility }, +) => T | undefined; + /** * Options used to process configuration data with a schema. */ @@ -80,6 +88,14 @@ type ConfigProcessingOptions = { * If omitted, the data will not be filtered by visibility. */ visiblity?: ConfigVisibility[]; + + /** + * A transform function that can be used to transform primitive configuration values + * during validation. The value returned from the transform function will be used + * instead of the original value. If the transform returns `undefined`, the value + * will be omitted. + */ + valueTransform?: TransformFunc; }; /** From b6e59e6368ccc62c713a6d253b331e3c603ad737 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 15:50:41 +0100 Subject: [PATCH 23/51] cli: refactor and fix config:print to properly filter secrets --- packages/cli/src/commands/config/print.ts | 65 +++++++++++++---------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/packages/cli/src/commands/config/print.ts b/packages/cli/src/commands/config/print.ts index 246847b610..9649c70ef3 100644 --- a/packages/cli/src/commands/config/print.ts +++ b/packages/cli/src/commands/config/print.ts @@ -16,37 +16,14 @@ import { Command } from 'commander'; import { stringify as stringifyYaml } from 'yaml'; -import { ConfigReader } from '@backstage/config'; +import { AppConfig, ConfigReader } from '@backstage/config'; import { loadCliConfig } from '../../lib/config'; -import cloneDeepWith from 'lodash/cloneDeepWith'; +import { ConfigSchema, ConfigVisibility } from '@backstage/config-loader'; export default async (cmd: Command) => { const { schema, appConfigs } = await loadCliConfig(cmd.config); - let data; - - if (cmd.frontend) { - const frontendConfigs = schema.process(appConfigs, { - visiblity: ['frontend'], - }); - data = ConfigReader.fromConfigs(frontendConfigs).get(); - } else if (cmd.withSecrets) { - data = ConfigReader.fromConfigs(appConfigs).get(); - } else { - let secretConfigs = schema.process(appConfigs, { - visiblity: ['secret'], - }); - secretConfigs = secretConfigs.map(entry => ({ - context: entry.context, - data: cloneDeepWith(entry.data, value => { - if (typeof value !== 'object') { - return ''; - } - return undefined; - }), - })); - - data = ConfigReader.fromConfigs(appConfigs.concat(secretConfigs)).get(); - } + const visibility = getVisiblityOption(cmd); + const data = serializeConfigData(appConfigs, schema, visibility); if (cmd.format === 'json') { process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); @@ -54,3 +31,37 @@ export default async (cmd: Command) => { process.stdout.write(`${stringifyYaml(data)}\n`); } }; + +function getVisiblityOption(cmd: Command): ConfigVisibility { + if (cmd.frontend && cmd.withSecrets) { + throw new Error('Not allowed to combine frontend and secret config'); + } + if (cmd.frontend) { + return 'frontend'; + } else if (cmd.withSecrets) { + return 'secret'; + } + return 'backend'; +} + +function serializeConfigData( + appConfigs: AppConfig[], + schema: ConfigSchema, + visiblity: ConfigVisibility, +) { + if (visiblity === 'frontend') { + const frontendConfigs = schema.process(appConfigs, { + visiblity: ['frontend'], + }); + return ConfigReader.fromConfigs(frontendConfigs).get(); + } else if (visiblity === 'secret') { + return ConfigReader.fromConfigs(appConfigs).get(); + } + + const sanitizedConfigs = schema.process(appConfigs, { + valueTransform: (value, { visibility }) => + visibility === 'secret' ? '' : value, + }); + + return ConfigReader.fromConfigs(sanitizedConfigs).get(); +} From eb9e293fcef38a29cb75020f0afaa92e02581c84 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 14 Nov 2020 18:03:10 +0100 Subject: [PATCH 24/51] config-loader: skip schema collection from packages that don't export package.json --- .../config-loader/src/lib/schema/collect.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index f7fed94748..6f441a2735 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -53,12 +53,19 @@ export async function collectConfigSchemas( } visitedPackages.add(name); - const pkgPath = req.resolve( - `${name}/package.json`, - parentPath && { - paths: [parentPath], - }, - ); + let pkgPath: string; + try { + pkgPath = req.resolve( + `${name}/package.json`, + parentPath && { + paths: [parentPath], + }, + ); + } catch { + // We can somewhat safely ignore packages that don't export package.json, + // as they are likely not part of the Backstage ecosystem anyway. + return; + } const pkg = await fs.readJson(pkgPath); const depNames = [ From d02d79e39276c1c61e695d3b3995685d0e643d9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 13:07:13 +0100 Subject: [PATCH 25/51] config-loader,cli: better presentation of config schema validation errors --- packages/cli/src/lib/config.ts | 29 ++++++++++++------- .../config-loader/src/lib/schema/compile.ts | 10 +++++-- packages/config-loader/src/lib/schema/load.ts | 4 ++- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 63e3958a6f..08a8193818 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -40,15 +40,24 @@ export async function loadCliConfig(configArgs: string[]) { `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`, ); - const frontendAppConfigs = schema.process(appConfigs, { - visiblity: ['frontend'], - }); - const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); + try { + const frontendAppConfigs = schema.process(appConfigs, { + visiblity: ['frontend'], + }); + const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); - return { - schema, - appConfigs, - frontendConfig, - frontendAppConfigs, - }; + return { + schema, + appConfigs, + frontendConfig, + frontendAppConfigs, + }; + } catch (error) { + const maybeSchemaError = error as Error & { messages?: string[] }; + if (maybeSchemaError.messages) { + const messages = maybeSchemaError.messages.join('\n '); + throw new Error(`Configuration does not match schema\n\n ${messages}`); + } + throw error; + } } diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 7dc65be910..6ee24f8d21 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -104,10 +104,14 @@ export function compileConfigSchemas( const valid = validate(config); if (!valid) { - // TODO(Rugvip): better messages here, with more context such as which file the error occurred in - const errors = ajv.errorsText(validate.errors); + const errors = validate.errors ?? []; return { - errors: [errors], + errors: errors.map(({ dataPath, message, params }) => { + const paramStr = Object.entries(params) + .map(([name, value]) => `${name}=${value}`) + .join(' '); + return `Config ${message || ''} { ${paramStr} } at ${dataPath}`; + }), visibilityByPath: new Map(), }; } diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index 010317849c..01a9499983 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -61,9 +61,11 @@ export async function loadConfigSchema( ): AppConfig[] { const result = validate(configs); if (result.errors) { - throw new Error( + const error = new Error( `Config validation failed, ${result.errors.join('; ')}`, ); + (error as any).messages = result.errors; + throw error; } let processedConfigs = configs; From 28a49c2ffa49a2316c97a738f7e13da2c13477e3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:09:07 +0100 Subject: [PATCH 26/51] config-loader: make visibility schema keyword work for all types --- packages/config-loader/src/lib/schema/compile.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 6ee24f8d21..5009dccce7 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -46,7 +46,6 @@ export function compileConfigSchemas( 'https://backstage.io/schema/config-v1': true, }, }).addKeyword('visibility', { - type: 'string', metaSchema: { type: 'string', enum: CONFIG_VISIBILITIES, From 06205426d8b285709c068b1fad98e62cd2f172f2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:13:42 +0100 Subject: [PATCH 27/51] config-loader: add support for defining config schema using .d.ts files --- packages/config-loader/package.json | 1 + .../config-loader/src/lib/schema/collect.ts | 72 ++++++++++++++++--- yarn.lock | 21 ++++-- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 32776d71b7..9abd3c754e 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -38,6 +38,7 @@ "fs-extra": "^9.0.0", "json-schema": "^0.2.5", "json-schema-merge-allof": "^0.7.0", + "typescript-json-schema": "^0.43.0", "yaml": "^1.9.2", "yup": "^0.29.3" }, diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 6f441a2735..dd0776213c 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -21,6 +21,8 @@ import { dirname, } from 'path'; import { ConfigSchemaPackageEntry } from './types'; +import { getProgramFromFiles, generateSchema } from 'typescript-json-schema'; +import { JsonObject } from '@backstage/config'; type Item = { name: string; @@ -40,6 +42,7 @@ export async function collectConfigSchemas( ): Promise { const visitedPackages = new Set(); const schemas = Array(); + const tsSchemaPaths = Array(); const currentDir = process.cwd(); async function processItem({ name, parentPath }: Item) { @@ -83,18 +86,23 @@ export async function collectConfigSchemas( } if (hasSchema) { if (typeof pkg.configSchema === 'string') { - if (!pkg.configSchema.endsWith('.json')) { + const isJson = pkg.configSchema.endsWith('.json'); + const isDts = pkg.configSchema.endsWith('.d.ts'); + if (!isJson && !isDts) { throw new Error( - `Config schema files must be .json, got ${pkg.configSchema}`, + `Config schema files must be .json or .d.ts, got ${pkg.configSchema}`, ); } - const value = await fs.readJson( - resolvePath(dirname(pkgPath), pkg.configSchema), - ); - schemas.push({ - value, - path: relativePath(currentDir, pkgPath), - }); + if (isDts) { + tsSchemaPaths.push(resolvePath(dirname(pkgPath), pkg.configSchema)); + } else { + const path = resolvePath(dirname(pkgPath), pkg.configSchema); + const value = await fs.readJson(path); + schemas.push({ + value, + path: relativePath(currentDir, path), + }); + } } else { schemas.push({ value: pkg.configSchema, @@ -110,5 +118,49 @@ export async function collectConfigSchemas( await Promise.all(packageNames.map(name => processItem({ name }))); - return schemas; + const tsSchemas = compileTsSchemas(currentDir, tsSchemaPaths); + + return schemas.concat(tsSchemas); +} + +// This handles the support of TypeScript .d.ts config schema declarations. +// We collect all typescript schema definition and compile them all in one go. +// This is much faster than compiling them separately. +function compileTsSchemas(currentDir: string, paths: string[]) { + if (paths.length === 0) { + return []; + } + + const program = getProgramFromFiles(paths, { + incremental: false, + isolatedModules: true, + lib: ['ES5'], // Skipping most libs speeds processing up a lot, we just need the primitive types anyway + noEmit: true, + noResolve: true, + skipLibCheck: true, // Skipping lib checks speeds things up + skipDefaultLibCheck: true, + strict: true, + typeRoots: [], // Do not include any additional types + types: [], + }); + + const tsSchemas = paths.map(schemaFile => { + const value = generateSchema( + program, + // All schemas should export a `Config` symbol + 'Config', + // This enables usage of @visibility is doc comments + { validationKeywords: ['visibility'] }, + [schemaFile], + ) as JsonObject | null; + + const path = relativePath(currentDir, schemaFile); + + if (!value) { + throw new Error(`Invalid schema in ${path}, missing Config export`); + } + return { path, value }; + }); + + return tsSchemas; } diff --git a/yarn.lock b/yarn.lock index a85e24210f..104aa4a174 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12478,7 +12478,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: +glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@~7.1.6: version "7.1.6" resolved "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== @@ -23239,10 +23239,21 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^4.0.3: - version "4.0.3" - resolved "https://registry.npmjs.org/typescript/-/typescript-4.0.3.tgz#153bbd468ef07725c1df9c77e8b453f8d36abba5" - integrity sha512-tEu6DGxGgRJPb/mVPIZ48e69xCn2yRmCgYmDugAVwmJ6o+0u1RI18eO7E7WBTLYLaEVVOhwQmcdhQHweux/WPg== +typescript-json-schema@^0.43.0: + version "0.43.0" + resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.43.0.tgz#8bd9c832f1f15f006ff933907ce192222fdfd92f" + integrity sha512-4c9IMlIlHYJiQtzL1gh2nIPJEjBgJjDUs50gsnnc+GFyDSK1oFM3uQIBSVosiuA/4t6LSAXDS9vTdqbQC6EcgA== + dependencies: + "@types/json-schema" "^7.0.5" + glob "~7.1.6" + json-stable-stringify "^1.0.1" + typescript "~4.0.2" + yargs "^15.4.1" + +typescript@^4.0.3, typescript@~4.0.2: + version "4.0.5" + resolved "https://registry.npmjs.org/typescript/-/typescript-4.0.5.tgz#ae9dddfd1069f1cb5beb3ef3b2170dd7c1332389" + integrity sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ== ua-parser-js@^0.7.18: version "0.7.21" From 567ce7804a79ce669b5767d0f954e74fdf9d7f2d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:47:04 +0100 Subject: [PATCH 28/51] backend-common: port config schema to d.ts and fill out --- packages/backend-common/config-schema.json | 80 ----------- packages/backend-common/config.d.ts | 148 +++++++++++++++++++++ packages/backend-common/package.json | 5 +- 3 files changed, 151 insertions(+), 82 deletions(-) delete mode 100644 packages/backend-common/config-schema.json create mode 100644 packages/backend-common/config.d.ts diff --git a/packages/backend-common/config-schema.json b/packages/backend-common/config-schema.json deleted file mode 100644 index 2012447a11..0000000000 --- a/packages/backend-common/config-schema.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/backend-common", - "type": "object", - "properties": { - "integrations": { - "type": "object", - "additionalProperties": false, - "properties": { - "github": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "token": { - "type": "string", - "visibility": "secret" - } - } - } - }, - "gitlab": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "token": { - "type": "string", - "visibility": "secret" - } - } - } - }, - "bitbucket": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "username": { - "type": "string" - }, - "appPassword": { - "type": "string", - "visibility": "secret" - } - } - } - }, - "azure": { - "type": "array", - "items": { - "type": "object", - "required": ["host"], - "properties": { - "host": { - "type": "string" - }, - "token": { - "type": "string", - "visibility": "secret" - } - } - } - } - } - } - } -} diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts new file mode 100644 index 0000000000..64e883aa26 --- /dev/null +++ b/packages/backend-common/config.d.ts @@ -0,0 +1,148 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + frontend: { + baseUrl: string; // defined in core, but repeated here without doc + }; + + backend: { + baseUrl: string; // defined in core, but repeated here without doc + /** Address that the backend should listen to. */ + + listen: + | string + | { + /** Address of the interface that the backend should bind to. */ + address?: string; + /** Port that the backend should listen to. */ + port?: number; + }; + + /** HTTPS configuration for the backend. If omitted the backend will serve HTTP */ + https?: { + /** Certificate configuration or parameters for generating a self-signed certificate */ + certificate?: + | { + /** Algorithm to use to generate a self-signed certificate */ + algorithm: string; + keySize?: number; + days?: number; + } + | { + /** PEM encoded certificate. Use $file to load in a file */ + cert: string; + /** + * PEM encoded certificate key. Use $file to load in a file. + * @visibility secret + */ + key: string; + }; + }; + + /** Database connection configuration, select database type using the `client` field */ + database: + | { + client: 'sqlite3'; + connection: ':memory:' | string; + } + | { + client: 'pg'; + /** + * PostgreSQL connection string or knex configuration object. + * @secret + */ + connection: string | object; + }; + + cors?: { + origin?: string | string[]; + methods?: string | string[]; + allowedHeaders?: string | string[]; + exposedHeaders?: string | string[]; + credentials?: boolean; + maxAge?: number; + preflightContinue?: boolean; + optionsSuccessStatus?: number; + }; + + /** */ + csp?: object; + }; + + /** Configuration for integrations towards various external repository provider systems */ + integrations: { + /** Integration configuration for Azure */ + azure: { + /** The hostname of the given Azure instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + }[]; + + /** Integration configuration for BitBucket */ + bitbucket: { + /** The hostname of the given Bitbucket instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + /** The base url for the BitBucket API, for example https://api.bitbucket.org/2.0 */ + apiBaseUrl?: string; + /** + * The username to use for authenticated requests. + * @visibility secret + */ + username?: string; + /** + * BitBucket app password used to authenticate requests. + * @visibility secret + */ + appPassword?: string; + }[]; + + /** Integration configuration for GitHub */ + github: { + /** The hostname of the given GitHub instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + /** The base url for the GitHub API, for example https://api.github.com */ + apiBaseUrl?: string; + /** The base url for GitHub raw resources, for example https://raw.githubusercontent.com */ + rawBaseUrl?: string; + }[]; + + /** Integration configuration for GitLab */ + gitlab: { + /** The hostname of the given GitLab instance */ + host: string; + /** + * Token used to authenticate requests. + * @visibility secret + */ + token?: string; + }[]; + }; +} diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 2f58bd0536..737e9664bf 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -89,7 +89,8 @@ "supertest": "^4.0.2" }, "files": [ - "dist" + "dist", + "config.d.ts" ], - "configSchema": "config-schema.json" + "configSchema": "config.d.ts" } From 8782781f16f87a7bbbc2093b44af70cabef49887 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 20:53:00 +0100 Subject: [PATCH 29/51] core: ported config schema to d.t.s --- packages/core/config.d.ts | 65 ++++++++++++++++++++++++++++++++++++++ packages/core/package.json | 53 ++----------------------------- 2 files changed, 68 insertions(+), 50 deletions(-) create mode 100644 packages/core/config.d.ts diff --git a/packages/core/config.d.ts b/packages/core/config.d.ts new file mode 100644 index 0000000000..014e4ac934 --- /dev/null +++ b/packages/core/config.d.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** + * Generic frontend configuration. + */ + app: { + /** + * The public absolute root URL that the frontend. + * @visibility frontend + */ + baseUrl: string; + + /** + * The title of the app. + * @visibility frontend + */ + title?: string; + }; + + /** + * Generic backend configuration. + */ + backend: { + /** + * The public absolute root URL that the backend is reachable at. + * @visibility frontend + */ + baseUrl: string; + }; + + /** + * Configuration that provides information about the organization that the app is for. + */ + organization?: { + /** + * The name of the organization that the app belongs to. + * @visibility frontend + */ + name?: string; + }; + + homepage?: { + clocks?: { + /** @visibility frontend */ + label: string; + /** @visibility frontend */ + timezone: string; + }[]; + }; +} diff --git a/packages/core/package.json b/packages/core/package.json index 3d4b53d35b..d0b623ea83 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -79,55 +79,8 @@ "@types/zen-observable": "^0.8.0" }, "files": [ - "dist" + "dist", + "config.d.ts" ], - "configSchema": { - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/core", - "type": "object", - "properties": { - "app": { - "type": "object", - "properties": { - "title": { - "type": "string", - "visibility": "frontend" - }, - "baseUrl": { - "type": "string", - "visibility": "frontend" - } - }, - "required": [ - "title", - "baseUrl" - ] - }, - "backend": { - "type": "object", - "properties": { - "baseUrl": { - "type": "string", - "visibility": "frontend" - } - }, - "required": [ - "baseUrl" - ] - }, - "organization": { - "type": "object", - "properties": { - "name": { - "type": "string", - "visibility": "frontend" - } - } - } - }, - "required": [ - "app", - "backend" - ] - } + "configSchema": "config.d.ts" } From cb1634e818847e629002f5b5cb91b2029264491a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:11:47 +0100 Subject: [PATCH 30/51] cli: added config schema --- packages/cli/package.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/cli/package.json b/packages/cli/package.json index 66c56dd8cc..23c5c5d6af 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -147,5 +147,31 @@ "watch": "./src", "exec": "bin/backstage-cli", "ext": "ts" + }, + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/cli", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "visibility": "frontend" + }, + "title": { + "type": "string", + "visibility": "frontend" + }, + "googleAnalyticsTrackingId": { + "type": "string", + "visibility": "frontend", + "description": "Tracking ID for Google Analytics", + "example": "UA-000000-0" + } + } + } + } } } From f6ed7d5d0d966c14bb48895e094741abde9be7f3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:29:30 +0100 Subject: [PATCH 31/51] config-loader: normalize data paths to ensure consistent behavior --- .../config-loader/src/lib/schema/compile.ts | 6 +++++- .../src/lib/schema/filtering.test.ts | 18 +++++++++--------- .../config-loader/src/lib/schema/filtering.ts | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 5009dccce7..93ab8e978f 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -56,7 +56,11 @@ export function compileConfigSchemas( return false; } if (visibility) { - visibilityByPath.set(dataPath, visibility); + const normalizedPath = dataPath.replace( + /\['?(.*?)'?\]/g, + (_, segment) => `.${segment}`, + ); + visibilityByPath.set(normalizedPath, visibility); } return true; }; diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 8e3fb61ae5..bfad2ca95a 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -40,18 +40,18 @@ const data = { const visiblity = new Map( Object.entries({ - '.arr[0]': 'frontend', - '.arr[1]': 'backend', - '.arr[2]': 'secret', + '.arr.0': 'frontend', + '.arr.1': 'backend', + '.arr.2': 'secret', '.obj.f': 'frontend', '.obj.b': 'backend', '.obj.b.s': 'secret', - '.objArr[0].f': 'frontend', - '.objArr[0].b': 'backend', - '.objArr[0].s': 'secret', - '.objArr[1].f': 'frontend', - '.objArr[1].b': 'backend', - '.objArr[1].s': 'secret', + '.objArr.0.f': 'frontend', + '.objArr.0.b': 'backend', + '.objArr.0.s': 'secret', + '.objArr.1.f': 'frontend', + '.objArr.1.b': 'backend', + '.objArr.1.s': 'secret', '.arrF': 'frontend', '.arrB': 'backend', '.arrS': 'secret', diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 74e75f465d..10a97f9a7f 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -49,7 +49,7 @@ export function filterByVisibility( const arr = new Array(); for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}[${index}]`); + const out = transform(value, `${path}.${index}`); if (out !== undefined) { arr.push(out); } From feb9905a377a5d441c7b7568010b618b8b5e5a37 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:30:49 +0100 Subject: [PATCH 32/51] plugins: add config schema needed by frontend plugins --- plugins/auth-backend/package.json | 35 +------------------- plugins/cost-insights/config.d.ts | 52 ++++++++++++++++++++++++++++++ plugins/cost-insights/package.json | 6 ++-- plugins/lighthouse/package.json | 18 ++++++++++- plugins/rollbar/package.json | 18 ++++++++++- plugins/sentry/package.json | 18 ++++++++++- plugins/techdocs/package.json | 18 ++++++++++- plugins/user-settings/package.json | 21 +++++++++++- 8 files changed, 145 insertions(+), 41 deletions(-) create mode 100644 plugins/cost-insights/config.d.ts diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 2bc00f56a2..af3d056b28 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -66,38 +66,5 @@ "files": [ "dist", "migrations" - ], - "configSchema": { - "$schema": "https://backstage.io/schema/config-v1", - "title": "@backstage/plugin-auth-backend", - "type": "object", - "properties": { - "auth": { - "type": "object", - "properties": { - "providers": { - "type": "object", - "properties": { - "google": { - "type": "object", - "visibility": "frontend", - "additionalProperties": { - "type": "object", - "properties": { - "clientId": { - "type": "string" - }, - "clientSecret": { - "type": "string", - "visibility": "secret" - } - } - } - } - } - } - } - } - } - } + ] } diff --git a/plugins/cost-insights/config.d.ts b/plugins/cost-insights/config.d.ts new file mode 100644 index 0000000000..0f147c2cfe --- /dev/null +++ b/plugins/cost-insights/config.d.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface Config { + costInsights: { + /** + * @visibility frontend + */ + engineerCost: number; + + products: { + [kind: string]: { + /** + * @visibility frontend + */ + name: string; + + /** + * @visibility frontend + */ + icon: 'compute' | 'data' | 'database' | 'storage' | 'search' | 'ml'; + }; + }; + + metrics?: { + [kind: string]: { + /** + * @visibility frontend + */ + name: string; + + /** + * @visibility frontend + */ + default?: boolean; + }; + }; + }; +} diff --git a/plugins/cost-insights/package.json b/plugins/cost-insights/package.json index 39159c03c4..1d7850482c 100644 --- a/plugins/cost-insights/package.json +++ b/plugins/cost-insights/package.json @@ -61,6 +61,8 @@ "msw": "^0.21.2" }, "files": [ - "dist" - ] + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/lighthouse/package.json b/plugins/lighthouse/package.json index 9389c84856..53bfc0e78b 100644 --- a/plugins/lighthouse/package.json +++ b/plugins/lighthouse/package.json @@ -52,5 +52,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/lighthouse", + "type": "object", + "properties": { + "lighthouse": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/rollbar/package.json b/plugins/rollbar/package.json index 722fe35fb6..18e4de3024 100644 --- a/plugins/rollbar/package.json +++ b/plugins/rollbar/package.json @@ -52,5 +52,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/rollbar", + "type": "object", + "properties": { + "rollbar": { + "type": "object", + "properties": { + "organization": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index 7941df5dfd..2757c02db9 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -49,5 +49,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/sentry", + "type": "object", + "properties": { + "sentry": { + "type": "object", + "properties": { + "organization": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 88a38549ad..a513b7aa8d 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -53,5 +53,21 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/techdocs", + "type": "object", + "properties": { + "techdocs": { + "type": "object", + "properties": { + "requestUrl": { + "type": "string", + "visibility": "frontend" + } + } + } + } + } } diff --git a/plugins/user-settings/package.json b/plugins/user-settings/package.json index a48c99cbc6..f82a6b2b6f 100644 --- a/plugins/user-settings/package.json +++ b/plugins/user-settings/package.json @@ -45,5 +45,24 @@ }, "files": [ "dist" - ] + ], + "configSchema": { + "$schema": "https://backstage.io/schema/config-v1", + "title": "@backstage/user-settings", + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "providers": { + "type": "object", + "additionalProperties": { + "type": "object", + "visibility": "frontend" + } + } + } + } + } + } } From b8eba8219ede8797aa0a75214be0b8f8b3182788 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:40:50 +0100 Subject: [PATCH 33/51] config-loader: enable validation of required props in d.ts definitions + fix issues --- app-config.yaml | 2 +- packages/backend-common/config.d.ts | 2 +- packages/config-loader/src/lib/schema/collect.ts | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 3d0d093536..a331d72d7c 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -270,7 +270,7 @@ costInsights: homepage: clocks: - label: UTC - timzone: UTC + timezone: UTC - label: NYC timezone: 'America/New_York' - label: STO diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 64e883aa26..b6beff12f7 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -15,7 +15,7 @@ */ export interface Config { - frontend: { + app: { baseUrl: string; // defined in core, but repeated here without doc }; diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index dd0776213c..11a7ce4ec2 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -150,7 +150,10 @@ function compileTsSchemas(currentDir: string, paths: string[]) { // All schemas should export a `Config` symbol 'Config', // This enables usage of @visibility is doc comments - { validationKeywords: ['visibility'] }, + { + required: true, + validationKeywords: ['visibility'], + }, [schemaFile], ) as JsonObject | null; From 13c999c513d1ff4a32841b36f94077473ba4d175 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:41:32 +0100 Subject: [PATCH 34/51] cli: added config:validate --- packages/cli/src/commands/config/validate.ts | 22 ++++++++++++++++++++ packages/cli/src/commands/index.ts | 8 +++++++ 2 files changed, 30 insertions(+) create mode 100644 packages/cli/src/commands/config/validate.ts diff --git a/packages/cli/src/commands/config/validate.ts b/packages/cli/src/commands/config/validate.ts new file mode 100644 index 0000000000..229f9e07e9 --- /dev/null +++ b/packages/cli/src/commands/config/validate.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Command } from 'commander'; +import { loadCliConfig } from '../../lib/config'; + +export default async (cmd: Command) => { + await loadCliConfig(cmd.config); +}; diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index bcbde20d84..304b5bae8f 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -146,6 +146,14 @@ export function registerCommands(program: CommanderStatic) { .description('Print the app configuration for the current package') .action(lazy(() => import('./config/print').then(m => m.default))); + program + .command('config:validate') + .option(...configOption) + .description( + 'Validate that the given configuration loads and matches schema', + ) + .action(lazy(() => import('./config/validate').then(m => m.default))); + program .command('prepack') .description('Prepares a package for packaging before publishing') From acb2f2c32e7adafefe0cfb66a983f6984c22936c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:42:17 +0100 Subject: [PATCH 35/51] github/workflows: validate config in CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75c6a13cd2..00ed9ce714 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,9 @@ jobs: - name: prettier run: yarn prettier:check + - name: validate config + run: yarn backstage-cli config:validate + - name: lint run: yarn lerna -- run lint --since origin/master From 1722cb53c281001e5aa96f631b14a6c0582b4c4f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:56:40 +0100 Subject: [PATCH 36/51] changesets: added config schema changesets --- .changeset/add-config-schema-declarations.md | 13 ++++++++++ .changeset/add-config-schema-support.md | 27 ++++++++++++++++++++ .changeset/app-backend-config.md | 7 +++++ 3 files changed, 47 insertions(+) create mode 100644 .changeset/add-config-schema-declarations.md create mode 100644 .changeset/add-config-schema-support.md create mode 100644 .changeset/app-backend-config.md diff --git a/.changeset/add-config-schema-declarations.md b/.changeset/add-config-schema-declarations.md new file mode 100644 index 0000000000..2f52a2ad3f --- /dev/null +++ b/.changeset/add-config-schema-declarations.md @@ -0,0 +1,13 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/core': patch +'@backstage/plugin-cost-insights': patch +'@backstage/plugin-lighthouse': patch +'@backstage/plugin-rollbar': patch +'@backstage/plugin-sentry': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-user-settings': patch +--- + +Added configuration schema diff --git a/.changeset/add-config-schema-support.md b/.changeset/add-config-schema-support.md new file mode 100644 index 0000000000..fd20426b30 --- /dev/null +++ b/.changeset/add-config-schema-support.md @@ -0,0 +1,27 @@ +--- +'@backstage/backend-common': minor +'@backstage/cli': minor +'@backstage/config-loader': minor +--- + +Added support for loading and validating configuration schema, as well as declaring config visibility through schema. + +The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. + +Configuration schema is declared using the following JSONSchema meta schema, which is based on draft07: https://backstage.io/schema/config-v1. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. + +Packages that wish to contribute configuration schema should declare it in a root "configSchema" field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be declared in either `.json` or `.d.ts`. + +TypeScript configuration schema files should export a single `Config` type, for example: + +```ts +export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; +} +``` diff --git a/.changeset/app-backend-config.md b/.changeset/app-backend-config.md new file mode 100644 index 0000000000..9df47ef1b8 --- /dev/null +++ b/.changeset/app-backend-config.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-app-backend': minor +--- + +Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection. + +This removes the confusing behavior where configuration was only injected into the app att build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. From 12f601f830da5d49396b916a0f9baf49d5a2919c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 21:58:12 +0100 Subject: [PATCH 37/51] config-loader: fix types dependencies --- packages/config-loader/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 9abd3c754e..98583cf833 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -32,8 +32,6 @@ "dependencies": { "@backstage/cli-common": "^0.1.1", "@backstage/config": "^0.1.1", - "@types/json-schema": "^7.0.6", - "@types/json-schema-merge-allof": "^0.6.0", "ajv": "^6.12.5", "fs-extra": "^9.0.0", "json-schema": "^0.2.5", @@ -44,6 +42,8 @@ }, "devDependencies": { "@types/jest": "^26.0.7", + "@types/json-schema": "^7.0.6", + "@types/json-schema-merge-allof": "^0.6.0", "@types/mock-fs": "^4.10.0", "@types/node": "^12.0.0", "@types/yup": "^0.29.8", From 9f6f39ce5f4c4688e0dd0ab7fd90a09f1dbb870d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 23:02:59 +0100 Subject: [PATCH 38/51] backend-common: integration configs aren't required --- packages/backend-common/config.d.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index b6beff12f7..9e8fb7ee35 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -84,9 +84,9 @@ export interface Config { }; /** Configuration for integrations towards various external repository provider systems */ - integrations: { + integrations?: { /** Integration configuration for Azure */ - azure: { + azure?: { /** The hostname of the given Azure instance */ host: string; /** @@ -97,7 +97,7 @@ export interface Config { }[]; /** Integration configuration for BitBucket */ - bitbucket: { + bitbucket?: { /** The hostname of the given Bitbucket instance */ host: string; /** @@ -120,7 +120,7 @@ export interface Config { }[]; /** Integration configuration for GitHub */ - github: { + github?: { /** The hostname of the given GitHub instance */ host: string; /** @@ -135,7 +135,7 @@ export interface Config { }[]; /** Integration configuration for GitLab */ - gitlab: { + gitlab?: { /** The hostname of the given GitLab instance */ host: string; /** From 82043bfe8bcaedd4717b42767eef35367078a69d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 15 Nov 2020 23:59:15 +0100 Subject: [PATCH 39/51] docs: add docs for configuration schema --- .github/styles/vocab.txt | 1 + docs/conf/defining.md | 103 +++++++++++++++++++++++++++++++++++++-- docs/conf/index.md | 18 ++++++- docs/conf/writing.md | 10 ++-- 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 3c5fa34adc..08e9883560 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -76,6 +76,7 @@ http https img incentivised +inlined inlinehilite interop javascript diff --git a/docs/conf/defining.md b/docs/conf/defining.md index bea03e4e44..83df8c3297 100644 --- a/docs/conf/defining.md +++ b/docs/conf/defining.md @@ -4,16 +4,109 @@ title: Defining Configuration for your Plugin description: Documentation on Defining Configuration for your Plugin --- -There is currently no tooling support or helpers for defining plugin -configuration. But it's on the roadmap. +Configuration in Backstage is organized via a configuration schema, which in +turn is defined using a super set of +[JSON Schema Draft-07](https://json-schema.org/specification-links.html#draft-7). +Each plugin or package within a Backstage app can contribute to the schema, +which during validation is stitched together into a single schema. -Meanwhile, document the config values that you are reading in your plugin -README. +## Schema Collection and Definition -## Format +Schemas are collected from all packages and dependencies in each repo that are a +part of the Backstage ecosystem, including transitive dependencies. The current +definition of "part of the ecosystem" is that a package has at least one +dependency in the `@backstage` namespace, but this is subject to change. + +Each package is search for schema at a single point of entry, a top-level +`"configSchema"` field in `package.json`. The field can either contain an +inlined JSON schema, or a relative path to a schema file. Supported schema file +formats are `.json` or `.d.ts`. + +> When using a schema file, be sure to include the file in your `package.json` > +> `"files"` field as well! + +TypeScript configuration schema files should export a single `Config` type, for +example: + +```ts +export interface Config { + app: { + /** + * Frontend root URL + * @visibility frontend + */ + baseUrl: string; + }; +} +``` + +Separate `.json` schema files can use a top-level +`"$schema": "https://backstage.io/schema/config-v1"` declaration in order to +receive schema validation and autocompletion. For example: + +```json +{ + "$schema": "https://backstage.io/schema/config-v1", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "baseUrl": { + "type": "string", + "description": "Frontend root URL", + "visibility": "frontend" + } + }, + "required": ["baseUrl"] + }, + "required": ["app"] + } +} +``` + +## Visibility + +The `https://backstage.io/schema/config-v1` meta schema is a super set of JSON +Schema Draft 07. The single addition is a custom `visibility` keyword, which is +used to indicate whether the given config value should be visible in the +frontend or not. The possible values are `frontend`, `backend`, `secret`, where +`backend` is the default. A visibility of `secret` has the same scope at +runtime, but it will be treated with more care in certain contexts, and defining +both `frontend` and `secret` for the same value in two different schemas will +result in an error during schema merging. + +The visibility only applies to the direct parent of where the keyword is placed +in the schema. For example, if you set the visibility to `frontend` for a subset +of the schema with `type: "object"`, but none of the descendants, only an empty +object will be available in the frontend. The full ancestry does not need to +have correctly defined visibilities however, so it is enough to only for example +declare the visibility of a leaf node of `type: "string"`. + +## Validation + +Schema can be validated using the `backstage-cli config:validate` command. If +you want to validate anything else than the default `app-config.yaml`, be sure +to pass in all of the configuration files as `--config ` options as well. + +To validate and examine the frontend configuration, use can use the +`backstage-cli config:print --frontend` command. Just like for validation you +may need to pass in all files using one or multiple `--config ` options. + +## Guidelines + +> Limit static configuration. The first question to ask is whether a particular +> option actually needs to be static configuration, or if it might just as well +> be a TypeScript API. In general options that you want to be able to change for +> different deployment environments should be static configuration, while it +> should otherwise be avoided. When defining configuration for your plugin, keep keys camelCased and stick to existing casing conventions such as `baseUrl`. It is also usually best to prefer objects over arrays, as it makes it possible to override individual values using separate files or environment variables. + +Avoid creating new top-level fields as much as possible. Either place your +configuration within an existing known top-level block, or create a single new +one using e.g. the name of the product that the plugin integrates. diff --git a/docs/conf/index.md b/docs/conf/index.md index 5e4639cc99..96fb40087d 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -33,6 +33,20 @@ values that are common between the two only need to be defined once. Such as the For more details, see [Writing Configuration](./writing.md). +## Configuration Schema + +The configuration is validated using a JSON Schema definitions. Each plugin and +package can provide pieces of the configuration schema, which is stitched +together to form a complete schema during validation. The configuration schema +is also used to select what configuration is available in the frontend using a +custom `visibility` keyword, as configuration is by default only available in +the backend. + +You can validate your configuration against the schema using +`backstage-cli config:validate`, and define schema for your own plugin either +using JSON Schema or TypeScript. For more information, see +[Defining Configuration](./defining.md). + ## Reading Configuration As a plugin developer, you likely end up wanting to define configuration that @@ -49,5 +63,5 @@ More details are provided in dedicated sections of the documentation. plugin. - [Writing Configuration](./writing.md): How to provide configuration for your Backstage deployment. -- [Defining Configuration](./defining.md): How to define configuration for users - of your plugin. +- [Defining Configuration](./defining.md): How to define configuration schema + for users of your plugin or package. diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 6b60c015ae..f28dd5b2d3 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -97,10 +97,10 @@ order: - If no config flags are provided, `app-config.local.yaml` has higher priority than `app-config.yaml`. -## Secrets +## Secrets and Dynamic Data -Secrets are supported via special secret keys that are prefixed with `$`, which -in turn provide a number of different ways to read in secrets. To load a +Secrets are supported via special data loading keys that are prefixed with `$`, +which in turn provide a number of different ways to read in secrets. To load a configuration value as a secret, supply an object with one of the special secret keys, for example `$env` or `$file`. A full list of supported secret keys can be found below. For example, the following will read the config key @@ -117,10 +117,6 @@ will return the value of the environment variable `MY_SECRET_KEY` when the backend started up. All secrets are loaded at startup, so changing the contents of secret files or environment variables will not be reflected at runtime. -Note that secrets will never be included in the frontend bundle or development -builds. When loading configuration you have to explicitly enable reading of -secrets, which is only done for the backend configuration. - As hinted at, secrets can be loaded from a bunch of different sources, and can be extended with more. Below is a list of the currently supported methods for loading secrets. From e2fc08cb15a3e449563d6777a08e033c468fa8ce Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 00:35:09 +0100 Subject: [PATCH 40/51] app-backend: fix tests --- plugins/app-backend/src/lib/config.test.ts | 45 ++++++++++++++----- plugins/app-backend/src/lib/config.ts | 7 ++- .../app-backend/src/service/router.test.ts | 5 ++- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/plugins/app-backend/src/lib/config.test.ts b/plugins/app-backend/src/lib/config.test.ts index 475f72f3b1..95b34c8422 100644 --- a/plugins/app-backend/src/lib/config.test.ts +++ b/plugins/app-backend/src/lib/config.test.ts @@ -43,11 +43,23 @@ describe('injectConfig', () => { jest.resetAllMocks(); }); - it('should not inject without config', async () => { + it('should inject without config', async () => { + fsMock.readdir.mockResolvedValue(['main.js']); + readFileMock.mockImplementation( + async () => '"__APP_INJECTED_RUNTIME_CONFIG__"', + ); await injectConfig(baseOptions); - expect(fsMock.readdir).toHaveBeenCalledTimes(0); - expect(fsMock.readFile).toHaveBeenCalledTimes(0); - expect(fsMock.writeFile).toHaveBeenCalledTimes(0); + expect(fsMock.readdir).toHaveBeenCalledTimes(1); + expect(fsMock.readFile).toHaveBeenCalledTimes(1); + expect(fsMock.writeFile).toHaveBeenCalledTimes(1); + expect(fsMock.writeFile).toHaveBeenCalledWith( + resolvePath(MOCK_DIR, 'main.js'), + '/*__APP_INJECTED_CONFIG_MARKER__*/"[]"/*__INJECTED_END__*/', + 'utf8', + ); + + // eslint-disable-next-line no-eval + expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual([]); }); it('should find the correct file to inject', async () => { @@ -83,14 +95,19 @@ describe('injectConfig', () => { expect(fsMock.writeFile).toHaveBeenCalledTimes(1); expect(fsMock.writeFile).toHaveBeenCalledWith( resolvePath(MOCK_DIR, 'main.js'), - '/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/', + '/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":0},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/', 'utf8', ); // eslint-disable-next-line no-eval - expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual({ - x: 0, - }); + expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual([ + { + data: { + x: 0, + }, + context: 'test', + }, + ]); }); it('should re-inject config', async () => { @@ -107,12 +124,14 @@ describe('injectConfig', () => { expect(fsMock.writeFile).toHaveBeenCalledTimes(1); expect(fsMock.writeFile).toHaveBeenCalledWith( resolvePath(MOCK_DIR, 'main.js'), - 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/)', + 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":0},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/)', 'utf8', ); // eslint-disable-next-line no-eval - expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual({ x: 0 }); + expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual([ + { data: { x: 0 }, context: 'test' }, + ]); readFileMock.mockResolvedValue(fsMock.writeFile.mock.calls[0][1]); @@ -124,11 +143,13 @@ describe('injectConfig', () => { expect(fsMock.writeFile).toHaveBeenCalledTimes(2); expect(fsMock.writeFile).toHaveBeenLastCalledWith( resolvePath(MOCK_DIR, 'main.js'), - 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":1,\\"y\\":2}"/*__INJECTED_END__*/)', + 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":1,\\"y\\":2},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/)', 'utf8', ); // eslint-disable-next-line no-eval - expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual({ x: 1, y: 2 }); + expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual([ + { data: { x: 1, y: 2 }, context: 'test' }, + ]); }); }); diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts index 82b431c33f..f076967280 100644 --- a/plugins/app-backend/src/lib/config.ts +++ b/plugins/app-backend/src/lib/config.ts @@ -28,8 +28,7 @@ type InjectOptions = { }; /** - * Injects config from APP_CONFIG_ env vars, replacing existing - * injected config if it has already been injected. + * Injects configs into the app bundle, replacing any existing injected config. */ export async function injectConfig(options: InjectOptions) { const { staticDir, logger, appConfigs } = options; @@ -73,6 +72,10 @@ type ReadOptions = { config: Config; }; +/** + * Read config from environment and process the backend config using the + * schema that is embedded in the frontend build. + */ export async function readConfigs(options: ReadOptions): Promise { const { env, appDistDir, config } = options; diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts index b0cc3c59f7..35dd9287a6 100644 --- a/plugins/app-backend/src/service/router.test.ts +++ b/plugins/app-backend/src/service/router.test.ts @@ -22,7 +22,10 @@ import Router from 'express-promise-router'; import request from 'supertest'; import { createRouter } from './router'; -jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() })); +jest.mock('../lib/config', () => ({ + injectConfig: jest.fn(), + readConfigs: jest.fn(), +})); global.__non_webpack_require__ = { /* eslint-disable-next-line no-restricted-syntax */ From 90fb5c81d06779e9a63bdaa1e59da96f76e6c75a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 01:05:54 +0100 Subject: [PATCH 41/51] cli: update diff to handle package with configSchema files --- packages/cli/src/lib/diff/handlers.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/lib/diff/handlers.ts b/packages/cli/src/lib/diff/handlers.ts index 40e3d4c587..a210986b2a 100644 --- a/packages/cli/src/lib/diff/handlers.ts +++ b/packages/cli/src/lib/diff/handlers.ts @@ -61,7 +61,7 @@ class PackageJsonHandler { await this.syncField('main:src'); } await this.syncField('types'); - await this.syncField('files'); + await this.syncFiles(); await this.syncScripts(); await this.syncPublishConfig(); await this.syncDependencies('dependencies'); @@ -105,6 +105,15 @@ class PackageJsonHandler { } } + private async syncFiles() { + if (typeof this.targetPkg.configSchema === 'string') { + const files = [...this.pkg.files, this.targetPkg.configSchema]; + await this.syncField('files', { files }); + } else { + await this.syncField('files'); + } + } + private async syncScripts() { const pkgScripts = this.pkg.scripts; const targetScripts = (this.targetPkg.scripts = From 9621ef3a9617277cdbfe97ebd06245d9b51975ef Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 10:28:23 +0100 Subject: [PATCH 42/51] techdocs: add storageUrl to config schema Co-authored-by: Sebastian Qvarfordt --- plugins/techdocs/package.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index a513b7aa8d..a2b1349d53 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -65,8 +65,14 @@ "requestUrl": { "type": "string", "visibility": "frontend" + }, + "storageUrl": { + "type": "string" } - } + }, + "required": [ + "requestUrl" + ] } } } From 5357723ea456332b1dfb32f6b2adbe445511bd9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 14:56:08 +0100 Subject: [PATCH 43/51] config-loader: fix .d.ts schema path handling on Windows --- .../config-loader/src/lib/schema/collect.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 11a7ce4ec2..9d8e6e73f9 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -19,6 +19,7 @@ import { resolve as resolvePath, relative as relativePath, dirname, + sep, } from 'path'; import { ConfigSchemaPackageEntry } from './types'; import { getProgramFromFiles, generateSchema } from 'typescript-json-schema'; @@ -43,7 +44,7 @@ export async function collectConfigSchemas( const visitedPackages = new Set(); const schemas = Array(); const tsSchemaPaths = Array(); - const currentDir = process.cwd(); + const currentDir = await fs.realpath(process.cwd()); async function processItem({ name, parentPath }: Item) { // Ensures that we only process each package once. We don't bother with @@ -94,7 +95,12 @@ export async function collectConfigSchemas( ); } if (isDts) { - tsSchemaPaths.push(resolvePath(dirname(pkgPath), pkg.configSchema)); + tsSchemaPaths.push( + relativePath( + currentDir, + resolvePath(dirname(pkgPath), pkg.configSchema), + ), + ); } else { const path = resolvePath(dirname(pkgPath), pkg.configSchema); const value = await fs.readJson(path); @@ -118,7 +124,7 @@ export async function collectConfigSchemas( await Promise.all(packageNames.map(name => processItem({ name }))); - const tsSchemas = compileTsSchemas(currentDir, tsSchemaPaths); + const tsSchemas = compileTsSchemas(tsSchemaPaths); return schemas.concat(tsSchemas); } @@ -126,7 +132,7 @@ export async function collectConfigSchemas( // This handles the support of TypeScript .d.ts config schema declarations. // We collect all typescript schema definition and compile them all in one go. // This is much faster than compiling them separately. -function compileTsSchemas(currentDir: string, paths: string[]) { +function compileTsSchemas(paths: string[]) { if (paths.length === 0) { return []; } @@ -144,7 +150,7 @@ function compileTsSchemas(currentDir: string, paths: string[]) { types: [], }); - const tsSchemas = paths.map(schemaFile => { + const tsSchemas = paths.map(path => { const value = generateSchema( program, // All schemas should export a `Config` symbol @@ -154,11 +160,9 @@ function compileTsSchemas(currentDir: string, paths: string[]) { required: true, validationKeywords: ['visibility'], }, - [schemaFile], + [path.split(sep).join('/')], // Unix paths are expected for all OSes here ) as JsonObject | null; - const path = relativePath(currentDir, schemaFile); - if (!value) { throw new Error(`Invalid schema in ${path}, missing Config export`); } From 72116243287db45f35f3b94cd655632a0902fe35 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 19:20:16 +0100 Subject: [PATCH 44/51] config-loader: added tests for schema collection --- .../src/lib/schema/collect.test.ts | 229 ++++++++++++++++++ .../config-loader/src/lib/schema/collect.ts | 29 ++- 2 files changed, 247 insertions(+), 11 deletions(-) create mode 100644 packages/config-loader/src/lib/schema/collect.test.ts diff --git a/packages/config-loader/src/lib/schema/collect.test.ts b/packages/config-loader/src/lib/schema/collect.test.ts new file mode 100644 index 0000000000..7aca4f7c0c --- /dev/null +++ b/packages/config-loader/src/lib/schema/collect.test.ts @@ -0,0 +1,229 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { collectConfigSchemas } from './collect'; + +const mockSchema = { + type: 'object', + properties: { + key: { + type: 'string', + visibility: 'frontend', + }, + }, +}; + +describe('collectConfigSchemas', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should not find any schemas without packages', async () => { + mockFs({ + 'lerna.json': JSON.stringify({ + packages: ['packages/*'], + }), + }); + + await expect(collectConfigSchemas([])).resolves.toEqual([]); + }); + + it('should find schema in a local package', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: mockSchema, + }), + }, + }, + }); + + await expect(collectConfigSchemas(['a'])).resolves.toEqual([ + { + path: 'node_modules/a/package.json', + value: mockSchema, + }, + ]); + }); + + it('should find schema in transitive dependencies', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + dependencies: { b: '0.0.0', '@backstage/mock': '0.0.0' }, + }), + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + dependencies: { + c1: '0.0.0', + c2: '0.0.0', + '@backstage/mock': '0.0.0', + }, + configSchema: { ...mockSchema, title: 'b' }, + }), + }, + c1: { + 'package.json': JSON.stringify({ + name: 'c1', + dependencies: { d1: '0.0.0' }, + configSchema: { ...mockSchema, title: 'c1' }, + }), + }, + c2: { + 'package.json': JSON.stringify({ + name: 'c2', + dependencies: { d2: '0.0.0' }, + }), + }, + d1: { + 'package.json': JSON.stringify({ + name: 'd1', + dependencies: {}, + configSchema: { ...mockSchema, title: 'd1' }, + }), + }, + d2: { + 'package.json': JSON.stringify({ + name: 'd2', + dependencies: {}, + configSchema: { ...mockSchema, title: 'd2' }, + }), + }, + }, + }); + + await expect(collectConfigSchemas(['a'])).resolves.toEqual([ + { + path: 'node_modules/b/package.json', + value: { ...mockSchema, title: 'b' }, + }, + { + path: 'node_modules/c1/package.json', + value: { ...mockSchema, title: 'c1' }, + }, + { + path: 'node_modules/d1/package.json', + value: { ...mockSchema, title: 'd1' }, + }, + ]); + }); + + it('should schema of different types', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: { ...mockSchema, title: 'inline' }, + }), + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + configSchema: 'schema.json', + }), + 'schema.json': JSON.stringify({ ...mockSchema, title: 'external' }), + }, + c: { + 'package.json': JSON.stringify({ + name: 'c', + configSchema: 'schema.d.ts', + }), + 'schema.d.ts': `export interface Config { + /** @visibility secret */ + tsKey: string + }`, + }, + }, + // TypeScript compilation needs to load some real files inside the typescript dir + '../../node_modules/typescript': (mockFs as any).load( + '../../node_modules/typescript', + ), + }); + + await expect(collectConfigSchemas(['a', 'b', 'c'])).resolves.toEqual([ + { + path: 'node_modules/a/package.json', + value: { ...mockSchema, title: 'inline' }, + }, + { + path: 'node_modules/b/schema.json', + value: { ...mockSchema, title: 'external' }, + }, + { + path: 'node_modules/c/schema.d.ts', + value: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { + tsKey: { + type: 'string', + visibility: 'secret', + }, + }, + required: ['tsKey'], + }, + }, + ]); + }); + + it('should not allow unknown schema file types', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: 'schema.yaml', + }), + 'schema.yaml': mockSchema, + }, + }, + }); + + await expect(collectConfigSchemas(['a'])).rejects.toThrow( + 'Config schema files must be .json or .d.ts, got schema.yaml', + ); + }); + + it('should reject typescript config declaration without a Config type', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: 'schema.d.ts', + }), + 'schema.d.ts': `export interface NotConfig {}`, + }, + }, + // TypeScript compilation needs to load some real files inside the typescript dir + '../../node_modules/typescript': (mockFs as any).load( + '../../node_modules/typescript', + ), + }); + + await expect(collectConfigSchemas(['a'])).rejects.toThrow( + 'Invalid schema in node_modules/a/schema.d.ts, missing Config export', + ); + }); +}); diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index 9d8e6e73f9..0e53562875 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -151,17 +151,24 @@ function compileTsSchemas(paths: string[]) { }); const tsSchemas = paths.map(path => { - const value = generateSchema( - program, - // All schemas should export a `Config` symbol - 'Config', - // This enables usage of @visibility is doc comments - { - required: true, - validationKeywords: ['visibility'], - }, - [path.split(sep).join('/')], // Unix paths are expected for all OSes here - ) as JsonObject | null; + let value; + try { + value = generateSchema( + program, + // All schemas should export a `Config` symbol + 'Config', + // This enables usage of @visibility is doc comments + { + required: true, + validationKeywords: ['visibility'], + }, + [path.split(sep).join('/')], // Unix paths are expected for all OSes here + ) as JsonObject | null; + } catch (error) { + if (error.message !== 'type Config not found') { + throw error; + } + } if (!value) { throw new Error(`Invalid schema in ${path}, missing Config export`); From eda9f91515b6024a79e273d24044f2f643ee8d40 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 19:32:25 +0100 Subject: [PATCH 45/51] config-loader: added test for config schema load --- .../config-loader/src/lib/schema/load.test.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 packages/config-loader/src/lib/schema/load.test.ts diff --git a/packages/config-loader/src/lib/schema/load.test.ts b/packages/config-loader/src/lib/schema/load.test.ts new file mode 100644 index 0000000000..a13f36be20 --- /dev/null +++ b/packages/config-loader/src/lib/schema/load.test.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { loadConfigSchema } from './load'; + +describe('loadConfigSchema', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should load schema from packages or data', async () => { + mockFs({ + node_modules: { + a: { + 'package.json': JSON.stringify({ + name: 'a', + configSchema: { + type: 'object', + properties: { + key1: { type: 'string', visibility: 'frontend' }, + }, + }, + }), + }, + b: { + 'package.json': JSON.stringify({ + name: 'b', + configSchema: 'schema.json', + }), + 'schema.json': JSON.stringify({ + name: 'a', + configSchema: { + type: 'object', + properties: { + key2: { type: 'number' }, + }, + }, + }), + }, + }, + }); + + const schema = await loadConfigSchema({ + dependencies: ['a'], + }); + + const configs = [{ data: { key1: 'a', key2: 2 }, context: 'test' }]; + + expect(schema.process(configs)).toEqual(configs); + expect(schema.process(configs, { visiblity: ['frontend'] })).toEqual([ + { data: { key1: 'a' }, context: 'test' }, + ]); + expect( + schema.process(configs, { + visiblity: ['frontend'], + valueTransform: () => 'X', + }), + ).toEqual([{ data: { key1: 'X' }, context: 'test' }]); + expect( + schema.process(configs, { + valueTransform: () => 'X', + }), + ).toEqual([{ data: { key1: 'X', key2: 'X' }, context: 'test' }]); + + const serialized = schema.serialize(); + + const schema2 = await loadConfigSchema({ serialized }); + expect(schema2.process(configs, { visiblity: ['frontend'] })).toEqual([ + { data: { key1: 'a' }, context: 'test' }, + ]); + expect(() => + schema2.process([...configs, { data: { key1: 3 }, context: 'test2' }]), + ).toThrow( + 'Config validation failed, Config should be string { type=string } at .key1', + ); + + await expect( + loadConfigSchema({ + serialized: { ...serialized, backstageConfigSchemaVersion: 2 }, + }), + ).rejects.toThrow( + 'Serialized configuration schema is invalid or has an invalid version number', + ); + }); +}); From 39591369bacda81c315a6e3f01ce07af004f4692 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Nov 2020 19:42:52 +0100 Subject: [PATCH 46/51] config-loader: add tests for schema compile and skip adding backend visibility paths --- .../src/lib/schema/compile.test.ts | 114 ++++++++++++++++++ .../config-loader/src/lib/schema/compile.ts | 2 +- 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 packages/config-loader/src/lib/schema/compile.test.ts diff --git a/packages/config-loader/src/lib/schema/compile.test.ts b/packages/config-loader/src/lib/schema/compile.test.ts new file mode 100644 index 0000000000..91e7aa687e --- /dev/null +++ b/packages/config-loader/src/lib/schema/compile.test.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { compileConfigSchemas } from './compile'; + +describe('compileConfigSchemas', () => { + it('should merge schemas', () => { + const validate = compileConfigSchemas([ + { + path: 'a', + value: { type: 'object', properties: { a: { type: 'string' } } }, + }, + { + path: 'b', + value: { type: 'object', properties: { b: { type: 'number' } } }, + }, + ]); + expect(validate([{ data: { a: 1 }, context: 'test' }])).toEqual({ + errors: ['Config should be string { type=string } at .a'], + visibilityByPath: new Map(), + }); + expect(validate([{ data: { b: 'b' }, context: 'test' }])).toEqual({ + errors: ['Config should be number { type=number } at .b'], + visibilityByPath: new Map(), + }); + }); + + it('should discover visibilities', () => { + const validate = compileConfigSchemas([ + { + path: 'a1', + value: { + type: 'object', + properties: { + a: { type: 'string', visibility: 'frontend' }, + b: { type: 'string', visibility: 'backend' }, + c: { type: 'string' }, + d: { + type: 'array', + visibility: 'secret', + items: { type: 'string', visibility: 'frontend' }, + }, + }, + }, + }, + { + path: 'a2', + value: { + type: 'object', + properties: { + a: { type: 'string' }, + b: { type: 'string', visibility: 'secret' }, + c: { type: 'string', visibility: 'backend' }, + d: { + type: 'array', + visibility: 'secret', + items: { type: 'string' }, + }, + }, + }, + }, + ]); + expect( + validate([ + { data: { a: 'a', b: 'b', c: 'c', d: ['d'] }, context: 'test' }, + ]), + ).toEqual({ + visibilityByPath: new Map( + Object.entries({ + '.a': 'frontend', + '.b': 'secret', + '.d': 'secret', + '.d.0': 'frontend', + }), + ), + }); + }); + + it('should reject visiblity conflicts', () => { + expect(() => + compileConfigSchemas([ + { + path: 'a1', + value: { + type: 'object', + properties: { a: { type: 'string', visibility: 'frontend' } }, + }, + }, + { + path: 'a2', + value: { + type: 'object', + properties: { a: { type: 'string', visibility: 'secret' } }, + }, + }, + ]), + ).toThrow( + "Config schema visibility is both 'frontend' and 'secret' for properties/a/visibility", + ); + }); +}); diff --git a/packages/config-loader/src/lib/schema/compile.ts b/packages/config-loader/src/lib/schema/compile.ts index 93ab8e978f..f01a4f640a 100644 --- a/packages/config-loader/src/lib/schema/compile.ts +++ b/packages/config-loader/src/lib/schema/compile.ts @@ -55,7 +55,7 @@ export function compileConfigSchemas( if (!dataPath) { return false; } - if (visibility) { + if (visibility && visibility !== 'backend') { const normalizedPath = dataPath.replace( /\['?(.*?)'?\]/g, (_, segment) => `.${segment}`, From 0b97a89f55b9bc9174ab1475456593400a21f300 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Nov 2020 18:25:55 +0100 Subject: [PATCH 47/51] address config schema review comments --- .changeset/add-config-schema-support.md | 6 ++-- .changeset/app-backend-config.md | 2 +- .github/styles/vocab.txt | 1 + .github/workflows/master.yml | 3 ++ docs/conf/defining.md | 30 +++++++++---------- docs/conf/index.md | 8 ++--- packages/backend-common/config.d.ts | 18 +++++------ .../config-loader/src/lib/schema/types.ts | 6 ++-- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/.changeset/add-config-schema-support.md b/.changeset/add-config-schema-support.md index fd20426b30..300c75f651 100644 --- a/.changeset/add-config-schema-support.md +++ b/.changeset/add-config-schema-support.md @@ -4,13 +4,13 @@ '@backstage/config-loader': minor --- -Added support for loading and validating configuration schema, as well as declaring config visibility through schema. +Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas. The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project. -Configuration schema is declared using the following JSONSchema meta schema, which is based on draft07: https://backstage.io/schema/config-v1. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. +A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. -Packages that wish to contribute configuration schema should declare it in a root "configSchema" field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be declared in either `.json` or `.d.ts`. +Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format. TypeScript configuration schema files should export a single `Config` type, for example: diff --git a/.changeset/app-backend-config.md b/.changeset/app-backend-config.md index 9df47ef1b8..39b0b956ad 100644 --- a/.changeset/app-backend-config.md +++ b/.changeset/app-backend-config.md @@ -4,4 +4,4 @@ Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection. -This removes the confusing behavior where configuration was only injected into the app att build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. +This removes the confusing behavior where configuration was only injected into the app at build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 08e9883560..740cf5b07f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -177,6 +177,7 @@ src subkey superfences Superfences +superset talkdesk Talkdesk tasklist diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f5029bf836..4c20194f32 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -47,6 +47,9 @@ jobs: run: yarn install --frozen-lockfile # End of yarn setup + - name: validate config + run: yarn backstage-cli config:validate + - name: lint run: yarn lerna -- run lint diff --git a/docs/conf/defining.md b/docs/conf/defining.md index 83df8c3297..d231320e93 100644 --- a/docs/conf/defining.md +++ b/docs/conf/defining.md @@ -5,7 +5,7 @@ description: Documentation on Defining Configuration for your Plugin --- Configuration in Backstage is organized via a configuration schema, which in -turn is defined using a super set of +turn is defined using a superset of [JSON Schema Draft-07](https://json-schema.org/specification-links.html#draft-7). Each plugin or package within a Backstage app can contribute to the schema, which during validation is stitched together into a single schema. @@ -17,13 +17,13 @@ part of the Backstage ecosystem, including transitive dependencies. The current definition of "part of the ecosystem" is that a package has at least one dependency in the `@backstage` namespace, but this is subject to change. -Each package is search for schema at a single point of entry, a top-level +Each package is searched for a schema at a single point of entry, a top-level `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Supported schema file formats are `.json` or `.d.ts`. -> When using a schema file, be sure to include the file in your `package.json` > -> `"files"` field as well! +> When defining a schema file, be sure to include the file in your +> `package.json` > `"files"` field as well! TypeScript configuration schema files should export a single `Config` type, for example: @@ -67,11 +67,11 @@ receive schema validation and autocompletion. For example: ## Visibility -The `https://backstage.io/schema/config-v1` meta schema is a super set of JSON +The `https://backstage.io/schema/config-v1` meta schema is a superset of JSON Schema Draft 07. The single addition is a custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the -frontend or not. The possible values are `frontend`, `backend`, `secret`, where -`backend` is the default. A visibility of `secret` has the same scope at +frontend or not. The possible values are `frontend`, `backend`, and `secret`, +where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging. @@ -85,24 +85,24 @@ declare the visibility of a leaf node of `type: "string"`. ## Validation -Schema can be validated using the `backstage-cli config:validate` command. If +Schemas can be validated using the `backstage-cli config:validate` command. If you want to validate anything else than the default `app-config.yaml`, be sure to pass in all of the configuration files as `--config ` options as well. -To validate and examine the frontend configuration, use can use the +To validate and examine the frontend configuration, use the `backstage-cli config:print --frontend` command. Just like for validation you may need to pass in all files using one or multiple `--config ` options. ## Guidelines -> Limit static configuration. The first question to ask is whether a particular -> option actually needs to be static configuration, or if it might just as well -> be a TypeScript API. In general options that you want to be able to change for -> different deployment environments should be static configuration, while it -> should otherwise be avoided. +> Make limited use of static configuration. The first question to ask is whether +> a particular option actually needs to be static configuration, or if it might +> just as well be a TypeScript API. In general, options that you want to be able +> to change for different deployment environments should be static +> configuration, while it should otherwise be avoided. When defining configuration for your plugin, keep keys camelCased and stick to -existing casing conventions such as `baseUrl`. +existing casing conventions such as `baseUrl` rather than `baseURL`. It is also usually best to prefer objects over arrays, as it makes it possible to override individual values using separate files or environment variables. diff --git a/docs/conf/index.md b/docs/conf/index.md index 96fb40087d..48da4eb74e 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -35,15 +35,15 @@ For more details, see [Writing Configuration](./writing.md). ## Configuration Schema -The configuration is validated using a JSON Schema definitions. Each plugin and -package can provide pieces of the configuration schema, which is stitched +The configuration is validated using JSON Schema definitions. Each plugin and +package can provide pieces of the configuration schema, which are stitched together to form a complete schema during validation. The configuration schema is also used to select what configuration is available in the frontend using a custom `visibility` keyword, as configuration is by default only available in the backend. You can validate your configuration against the schema using -`backstage-cli config:validate`, and define schema for your own plugin either +`backstage-cli config:validate`, and define a schema for your own plugin either using JSON Schema or TypeScript. For more information, see [Defining Configuration](./defining.md). @@ -63,5 +63,5 @@ More details are provided in dedicated sections of the documentation. plugin. - [Writing Configuration](./writing.md): How to provide configuration for your Backstage deployment. -- [Defining Configuration](./defining.md): How to define configuration schema +- [Defining Configuration](./defining.md): How to define a configuration schema for users of your plugin or package. diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 9e8fb7ee35..7ebcd01e8c 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -21,8 +21,8 @@ export interface Config { backend: { baseUrl: string; // defined in core, but repeated here without doc - /** Address that the backend should listen to. */ + /** Address that the backend should listen to. */ listen: | string | { @@ -86,7 +86,7 @@ export interface Config { /** Configuration for integrations towards various external repository provider systems */ integrations?: { /** Integration configuration for Azure */ - azure?: { + azure?: Array<{ /** The hostname of the given Azure instance */ host: string; /** @@ -94,10 +94,10 @@ export interface Config { * @visibility secret */ token?: string; - }[]; + }>; /** Integration configuration for BitBucket */ - bitbucket?: { + bitbucket?: Array<{ /** The hostname of the given Bitbucket instance */ host: string; /** @@ -117,10 +117,10 @@ export interface Config { * @visibility secret */ appPassword?: string; - }[]; + }>; /** Integration configuration for GitHub */ - github?: { + github?: Array<{ /** The hostname of the given GitHub instance */ host: string; /** @@ -132,10 +132,10 @@ export interface Config { apiBaseUrl?: string; /** The base url for GitHub raw resources, for example https://raw.githubusercontent.com */ rawBaseUrl?: string; - }[]; + }>; /** Integration configuration for GitLab */ - gitlab?: { + gitlab?: Array<{ /** The hostname of the given GitLab instance */ host: string; /** @@ -143,6 +143,6 @@ export interface Config { * @visibility secret */ token?: string; - }[]; + }>; }; } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index c82036bbf4..7705242b31 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -21,11 +21,11 @@ import { AppConfig, JsonObject } from '@backstage/config'; */ export type ConfigSchemaPackageEntry = { /** - * The configuration schema itself, as JSONSchema draft-07 + * The configuration schema itself. */ value: JsonObject; /** - * The path that the configuration schema was discovered at. + * The relative path that the configuration schema was discovered at. */ path: string; }; @@ -59,7 +59,7 @@ type ValidationResult = { */ errors?: ValidationError[]; /** - * The configuration visibilities the where discovered during validation. + * The configuration visibilities that were discovered during validation. * * The path in the key uses the form `////` */ From 80bf8c7f4fb97f2af0dbefce0fabb42980de50c8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Nov 2020 18:26:40 +0100 Subject: [PATCH 48/51] cli: rename config:validate to config:check --- .github/workflows/ci.yml | 2 +- .github/workflows/master.yml | 2 +- docs/conf/defining.md | 6 +++--- docs/conf/index.md | 2 +- packages/cli/src/commands/index.ts | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00ed9ce714..c35e2a88ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: run: yarn prettier:check - name: validate config - run: yarn backstage-cli config:validate + run: yarn backstage-cli config:check - name: lint run: yarn lerna -- run lint --since origin/master diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4c20194f32..ca63e627e9 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -48,7 +48,7 @@ jobs: # End of yarn setup - name: validate config - run: yarn backstage-cli config:validate + run: yarn backstage-cli config:check - name: lint run: yarn lerna -- run lint diff --git a/docs/conf/defining.md b/docs/conf/defining.md index d231320e93..ead5c4ff67 100644 --- a/docs/conf/defining.md +++ b/docs/conf/defining.md @@ -85,9 +85,9 @@ declare the visibility of a leaf node of `type: "string"`. ## Validation -Schemas can be validated using the `backstage-cli config:validate` command. If -you want to validate anything else than the default `app-config.yaml`, be sure -to pass in all of the configuration files as `--config ` options as well. +Schemas can be validated using the `backstage-cli config:check` command. If you +want to validate anything else than the default `app-config.yaml`, be sure to +pass in all of the configuration files as `--config ` options as well. To validate and examine the frontend configuration, use the `backstage-cli config:print --frontend` command. Just like for validation you diff --git a/docs/conf/index.md b/docs/conf/index.md index 48da4eb74e..a6f1d1f6f7 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -43,7 +43,7 @@ custom `visibility` keyword, as configuration is by default only available in the backend. You can validate your configuration against the schema using -`backstage-cli config:validate`, and define a schema for your own plugin either +`backstage-cli config:check`, and define a schema for your own plugin either using JSON Schema or TypeScript. For more information, see [Defining Configuration](./defining.md). diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 304b5bae8f..1ebe518cac 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -147,7 +147,7 @@ export function registerCommands(program: CommanderStatic) { .action(lazy(() => import('./config/print').then(m => m.default))); program - .command('config:validate') + .command('config:check') .option(...configOption) .description( 'Validate that the given configuration loads and matches schema', From c7d6fefe9c9751d667208ae3bc6c9c879f22773c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Nov 2020 10:18:10 +0100 Subject: [PATCH 49/51] cli: add config schema for frontend listening config + fix usage --- packages/cli/package.json | 16 ++++++++++++++++ packages/cli/src/lib/bundler/server.ts | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 23c5c5d6af..516e354012 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -169,6 +169,22 @@ "visibility": "frontend", "description": "Tracking ID for Google Analytics", "example": "UA-000000-0" + }, + "listen": { + "type": "object", + "description": "Listening configuration for local development", + "properties": { + "host": { + "type": "number", + "visibility": "frontend", + "description": "The host that the frontend should be bound to. Only used for local development." + }, + "post": { + "type": "number", + "visibility": "frontend", + "description": "The port that the frontend should be bound to. Only used for local development." + } + } } } } diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index f540e01084..d198f5507c 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -26,9 +26,9 @@ export async function serveBundle(options: ServeOptions) { const url = resolveBaseUrl(options.frontendConfig); const host = - options.config.getOptionalString('app.listen.host') || url.hostname; + options.frontendConfig.getOptionalString('app.listen.host') || url.hostname; const port = - options.config.getOptionalNumber('app.listen.port') || + options.frontendConfig.getOptionalNumber('app.listen.port') || Number(url.port) || (url.protocol === 'https:' ? 443 : 80); From 717e43de146b7c18798bcef4ebd0d5bd740578f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 18 Nov 2020 19:58:36 +0100 Subject: [PATCH 50/51] catalog-client: change entities interface, add fields support (#3296) --- .changeset/shaggy-turkeys-warn.md | 5 + .../catalog-client/src/CatalogClient.test.ts | 45 ++++++--- packages/catalog-client/src/CatalogClient.ts | 96 +++++++++++-------- packages/catalog-client/src/types.ts | 13 ++- .../ApiExplorerPage/ApiExplorerPage.test.tsx | 34 +++---- .../ApiExplorerPage/ApiExplorerPage.tsx | 6 +- .../CatalogFilter/CatalogFilter.test.tsx | 46 ++++----- .../CatalogPage/CatalogPage.test.tsx | 50 +++++----- .../ResultsFilter/ResultsFilter.test.tsx | 74 +++++++------- .../UnregisterEntityDialog.tsx | 10 +- .../src/filter/EntityFilterGroupsProvider.tsx | 9 +- .../src/filter/useEntityFilterGroup.test.tsx | 18 ++-- .../RollbarHome/RollbarHome.test.tsx | 2 +- .../rollbar/src/hooks/useRollbarEntities.ts | 4 +- .../ScaffolderPage/ScaffolderPage.tsx | 10 +- .../TemplatePage/TemplatePage.test.tsx | 60 ++++++------ .../components/TemplatePage/TemplatePage.tsx | 30 +++--- .../reader/components/TechDocsHome.test.tsx | 3 +- .../src/reader/components/TechDocsHome.tsx | 18 ++-- 19 files changed, 298 insertions(+), 235 deletions(-) create mode 100644 .changeset/shaggy-turkeys-warn.md diff --git a/.changeset/shaggy-turkeys-warn.md b/.changeset/shaggy-turkeys-warn.md new file mode 100644 index 0000000000..442cc788f2 --- /dev/null +++ b/.changeset/shaggy-turkeys-warn.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +Changed the getEntities interface to (1) nest parameters in an object, (2) support field selection, and (3) return an object with an items field for future extension diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 65625c11b8..6369f95b76 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ +import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { Entity } from '@backstage/catalog-model'; -import { DiscoveryApi } from './types'; +import { CatalogListResponse, DiscoveryApi } from './types'; const server = setupServer(); const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base'; @@ -40,7 +40,7 @@ describe('CatalogClient', () => { }); describe('getEntities', () => { - const defaultResponse: Entity[] = [ + const defaultServiceResponse: Entity[] = [ { apiVersion: '1', kind: 'Component', @@ -58,22 +58,26 @@ describe('CatalogClient', () => { }, }, ]; + const defaultResponse: CatalogListResponse = { + items: defaultServiceResponse, + }; beforeEach(() => { server.use( rest.get(`${mockBaseUrl}/entities`, (_, res, ctx) => { - return res(ctx.json(defaultResponse)); + return res(ctx.json(defaultServiceResponse)); }), ); }); it('should entities from correct endpoint', async () => { - const entities = await client.getEntities(); - expect(entities).toEqual(defaultResponse); + const response = await client.getEntities(); + expect(response).toEqual(defaultResponse); }); it('builds entity search filters properly', async () => { expect.assertions(2); + server.use( rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D'); @@ -81,13 +85,32 @@ describe('CatalogClient', () => { }), ); - const entities = await client.getEntities({ - a: '1', - b: ['2', '3'], - ö: '=', + const response = await client.getEntities({ + filter: { + a: '1', + b: ['2', '3'], + ö: '=', + }, }); - expect(entities).toEqual([]); + expect(response.items).toEqual([]); + }); + + it('builds entity field selectors properly', async () => { + expect.assertions(2); + + server.use( + rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { + expect(req.url.search).toBe('?fields=a.b,%C3%B6'); + return res(ctx.json([])); + }), + ); + + const response = await client.getEntities({ + fields: ['a.b', 'ö'], + }); + + expect(response.items).toEqual([]); }); }); }); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index e39759a88d..362b1e71da 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -25,6 +25,8 @@ import { AddLocationRequest, AddLocationResponse, CatalogApi, + CatalogEntitiesRequest, + CatalogListResponse, DiscoveryApi, } from './types'; @@ -35,55 +37,33 @@ export class CatalogClient implements CatalogApi { this.discoveryApi = options.discoveryApi; } - private async getRequired(path: string): Promise { - const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; - const response = await fetch(url); - - if (!response.ok) { - const payload = await response.text(); - const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; - throw new Error(message); - } - - return await response.json(); - } - - private async getOptional(path: string): Promise { - const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; - const response = await fetch(url); - - if (!response.ok) { - if (response.status === 404) { - return undefined; - } - - const payload = await response.text(); - const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; - throw new Error(message); - } - - return await response.json(); - } - async getLocationById(id: String): Promise { return await this.getOptional(`/locations/${id}`); } async getEntities( - filter?: Record, - ): Promise { - let path = `/entities`; - if (filter) { - const parts: string[] = []; - for (const [key, value] of Object.entries(filter)) { - for (const v of [value].flat()) { - parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(v)}`); - } + request?: CatalogEntitiesRequest, + ): Promise> { + const { filter = {}, fields = [] } = request ?? {}; + const params: string[] = []; + + const filterParts: string[] = []; + for (const [key, value] of Object.entries(filter)) { + for (const v of [value].flat()) { + filterParts.push(`${encodeURIComponent(key)}=${encodeURIComponent(v)}`); } - path += `?filter=${parts.join(',')}`; + } + if (filterParts.length) { + params.push(`filter=${filterParts.join(',')}`); } - return await this.getRequired(path); + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + + const query = params.length ? `?${params.join('&')}` : ''; + const entities: Entity[] = await this.getRequired(`/entities${query}`); + return { items: entities }; } async getEntityByName(compoundName: EntityName): Promise { @@ -153,4 +133,38 @@ export class CatalogClient implements CatalogApi { } return undefined; } + + // + // Private methods + // + + private async getRequired(path: string): Promise { + const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; + const response = await fetch(url); + + if (!response.ok) { + const payload = await response.text(); + const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; + throw new Error(message); + } + + return await response.json(); + } + + private async getOptional(path: string): Promise { + const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; + const response = await fetch(url); + + if (!response.ok) { + if (response.status === 404) { + return undefined; + } + + const payload = await response.text(); + const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`; + throw new Error(message); + } + + return await response.json(); + } } diff --git a/packages/catalog-client/src/types.ts b/packages/catalog-client/src/types.ts index 575b8d6ca2..e72317d444 100644 --- a/packages/catalog-client/src/types.ts +++ b/packages/catalog-client/src/types.ts @@ -16,10 +16,21 @@ import { Entity, EntityName, Location } from '@backstage/catalog-model'; +export type CatalogEntitiesRequest = { + filter?: Record | undefined; + fields?: string[] | undefined; +}; + +export type CatalogListResponse = { + items: T[]; +}; + export interface CatalogApi { getLocationById(id: String): Promise; getEntityByName(name: EntityName): Promise; - getEntities(filter?: Record): Promise; + getEntities( + request?: CatalogEntitiesRequest, + ): Promise>; addLocation(location: AddLocationRequest): Promise; getLocationByEntity(entity: Entity): Promise; removeEntityByUid(uid: string): Promise; diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx index 46cf014134..10495bb957 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx @@ -26,24 +26,26 @@ import { ApiExplorerPage } from './ApiExplorerPage'; describe('ApiCatalogPage', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { - name: 'Entity1', + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { + name: 'Entity1', + }, + spec: { type: 'openapi' }, }, - spec: { type: 'openapi' }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'API', - metadata: { - name: 'Entity2', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { + name: 'Entity2', + }, + spec: { type: 'openapi' }, }, - spec: { type: 'openapi' }, - }, - ] as Entity[]), + ] as Entity[], + }), getLocationByEntity: () => Promise.resolve({ id: 'id', type: 'github', target: 'url' }), }; diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx index c6a8d62b58..398268caf2 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx @@ -25,8 +25,8 @@ import { ApiExplorerLayout } from './ApiExplorerLayout'; export const ApiExplorerPage = () => { const catalogApi = useApi(catalogApiRef); - const { loading, error, value: matchingEntities } = useAsync(() => { - return catalogApi.getEntities({ kind: 'API' }); + const { loading, error, value: catalogResponse } = useAsync(() => { + return catalogApi.getEntities({ filter: { kind: 'API' } }); }, [catalogApi]); return ( @@ -44,7 +44,7 @@ export const ApiExplorerPage = () => { All your APIs diff --git a/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx b/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx index a4ede7c2f2..b92fc3d4bc 100644 --- a/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx +++ b/plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx @@ -33,30 +33,32 @@ import { ButtonGroup, CatalogFilter } from './CatalogFilter'; describe('Catalog Filter', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity1', + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity1', + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'tools@example.com', - type: 'service', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity2', + }, + spec: { + owner: 'not-tools@example.com', + type: 'service', + }, }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity2', - }, - spec: { - owner: 'not-tools@example.com', - type: 'service', - }, - }, - ] as Entity[]), + ] as Entity[], + }), }; const identityApi: Partial = { diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx index 0d645f0f35..83bf4a6f6e 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx @@ -34,37 +34,39 @@ import { CatalogPage } from './CatalogPage'; describe('CatalogPage', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity1', + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity1', + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'tools@example.com', - type: 'service', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity2', + }, + spec: { + owner: 'not-tools@example.com', + type: 'service', + }, }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity2', - }, - spec: { - owner: 'not-tools@example.com', - type: 'service', - }, - }, - ] as Entity[]), + ] as Entity[], + }), getLocationByEntity: () => Promise.resolve({ id: 'id', type: 'github', target: 'url' }), }; const testProfile: Partial = { displayName: 'Display Name', }; - const indentityApi: Partial = { + const identityApi: Partial = { getUserId: () => 'tools@example.com', getProfile: () => testProfile, }; @@ -75,7 +77,7 @@ describe('CatalogPage', () => { diff --git a/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx b/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx index 39837d2ca1..d105979600 100644 --- a/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx +++ b/plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx @@ -33,46 +33,48 @@ import { ResultsFilter } from './ResultsFilter'; describe('Results Filter', () => { const catalogApi: Partial = { getEntities: () => - Promise.resolve([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity1', - tags: ['java'], + Promise.resolve({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity1', + tags: ['java'], + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'tools@example.com', - type: 'service', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity2', + }, + spec: { + owner: 'not-tools@example.com', + type: 'service', + }, }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity2', + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'Entity3', + tags: ['java', 'test'], + }, + spec: { + owner: 'tools@example.com', + type: 'service', + }, }, - spec: { - owner: 'not-tools@example.com', - type: 'service', - }, - }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'Entity3', - tags: ['java', 'test'], - }, - spec: { - owner: 'tools@example.com', - type: 'service', - }, - }, - ] as Entity[]), + ] as Entity[], + }), }; - const indentityApi: Partial = { + const identityApi: Partial = { getUserId: () => 'tools@example.com', }; @@ -82,7 +84,7 @@ describe('Results Filter', () => { diff --git a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx b/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx index 218cc45e78..96b6880bb9 100644 --- a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx +++ b/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx @@ -44,9 +44,13 @@ function useColocatedEntities(entity: Entity): AsyncState { const catalogApi = useApi(catalogApiRef); return useAsync(async () => { const myLocation = entity.metadata.annotations?.[LOCATION_ANNOTATION]; - return myLocation - ? await catalogApi.getEntities({ [LOCATION_ANNOTATION]: myLocation }) - : []; + if (!myLocation) { + return []; + } + const response = await catalogApi.getEntities({ + filter: { [LOCATION_ANNOTATION]: myLocation }, + }); + return response.items; }, [catalogApi, entity]); } diff --git a/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx b/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx index 36cc0c56df..edda8eecb5 100644 --- a/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx +++ b/plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx @@ -46,9 +46,12 @@ export const EntityFilterGroupsProvider = ({ // The hook that implements the actual context building function useProvideEntityFilters(): FilterGroupsContext { const catalogApi = useApi(catalogApiRef); - const [{ value: entities, error }, doReload] = useAsyncFn(() => - catalogApi.getEntities({ kind: 'Component' }), - ); + const [{ value: entities, error }, doReload] = useAsyncFn(async () => { + const response = await catalogApi.getEntities({ + filter: { kind: 'Component' }, + }); + return response.items; + }); const filterGroups = useRef<{ [filterGroupId: string]: FilterGroup; diff --git a/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx b/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx index 6b573e3721..e9af063160 100644 --- a/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx +++ b/plugins/catalog/src/filter/useEntityFilterGroup.test.tsx @@ -49,7 +49,7 @@ describe('useEntityFilterGroup', () => { }); it('works for an empty set of filters', async () => { - catalogApi.getEntities.mockResolvedValue([]); + catalogApi.getEntities.mockResolvedValue({ items: [] }); const group: FilterGroup = { filters: {} }; const { result, waitFor } = renderHook( () => useEntityFilterGroup('g1', group), @@ -60,13 +60,15 @@ describe('useEntityFilterGroup', () => { }); it('works for a single group', async () => { - catalogApi.getEntities.mockResolvedValue([ - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { name: 'n' }, - }, - ]); + catalogApi.getEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'n' }, + }, + ], + }); const group: FilterGroup = { filters: { f1: e => e.metadata.name === 'n', diff --git a/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx b/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx index c6ee052c0e..ff3e909bf7 100644 --- a/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx +++ b/plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx @@ -52,7 +52,7 @@ describe('RollbarHome component', () => { catalogApiRef, ({ async getEntities() { - return []; + return { items: [] }; }, } as Partial) as CatalogApi, ], diff --git a/plugins/rollbar/src/hooks/useRollbarEntities.ts b/plugins/rollbar/src/hooks/useRollbarEntities.ts index 3edf34d1f3..5979c5832a 100644 --- a/plugins/rollbar/src/hooks/useRollbarEntities.ts +++ b/plugins/rollbar/src/hooks/useRollbarEntities.ts @@ -28,8 +28,8 @@ export function useRollbarEntities() { configApi.getString('organization.name'); const { value, loading, error } = useAsync(async () => { - const entities = await catalogApi.getEntities(); - return entities.filter(entity => { + const response = await catalogApi.getEntities(); + return response.items.filter(entity => { return !!entity.metadata.annotations?.[ROLLBAR_ANNOTATION]; }); }, [catalogApi]); diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index d0cb25f254..abdb411ad9 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -53,10 +53,12 @@ export const ScaffolderPage = () => { const { data: templates, isValidating, error } = useStaleWhileRevalidate( 'templates/all', - async () => - catalogApi.getEntities({ kind: 'Template' }) as Promise< - TemplateEntityV1alpha1[] - >, + async () => { + const response = await catalogApi.getEntities({ + filter: { kind: 'Template' }, + }); + return response.items as TemplateEntityV1alpha1[]; + }, ); useEffect(() => { diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx index 6229a88fc3..62963b58b8 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx @@ -13,18 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React from 'react'; -import { TemplatePage } from './TemplatePage'; -import { wrapInTestApp, renderWithEffects } from '@backstage/test-utils'; -import { ApiRegistry, errorApiRef, ApiProvider } from '@backstage/core'; -import { scaffolderApiRef, ScaffolderApi } from '../../api'; -import { catalogApiRef, CatalogApi } from '@backstage/plugin-catalog'; -import { mutate } from 'swr'; -import { act } from 'react-dom/test-utils'; -import { Route, MemoryRouter } from 'react-router'; -import { rootRoute } from '../../routes'; -import { ThemeProvider } from '@material-ui/core'; +import { ApiProvider, ApiRegistry, errorApiRef } from '@backstage/core'; +import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog'; +import { renderInTestApp, renderWithEffects } from '@backstage/test-utils'; import { lightTheme } from '@backstage/theme'; +import { ThemeProvider } from '@material-ui/core'; +import React from 'react'; +import { act } from 'react-dom/test-utils'; +import { MemoryRouter, Route } from 'react-router'; +import { ScaffolderApi, scaffolderApiRef } from '../../api'; +import { rootRoute } from '../../routes'; +import { TemplatePage } from './TemplatePage'; const templateMock = { apiVersion: 'backstage.io/v1alpha1', @@ -90,48 +89,43 @@ const apis = ApiRegistry.from([ ]); describe('TemplatePage', () => { - afterEach(async () => { - // Cleaning up swr's cache - await act(async () => { - await mutate('templates/test'); - }); - }); + beforeEach(() => jest.resetAllMocks()); + it('renders correctly', async () => { - catalogApiMock.getEntities.mockResolvedValueOnce([templateMock]); - const rendered = await renderWithEffects( - wrapInTestApp( - - - , - ), + catalogApiMock.getEntities.mockResolvedValueOnce({ items: [templateMock] }); + const rendered = await renderInTestApp( + + + , ); expect(rendered.queryByText('Create a new component')).toBeInTheDocument(); expect(rendered.queryByText('React SSR Template')).toBeInTheDocument(); // await act(async () => await mutate('templates/test')); }); + it('renders spinner while loading', async () => { let resolve: Function; const promise = new Promise(res => { resolve = res; }); - catalogApiMock.getEntities.mockResolvedValueOnce(promise); - const rendered = await renderWithEffects( - wrapInTestApp( - - - , - ), + catalogApiMock.getEntities.mockReturnValueOnce(promise); + const rendered = await renderInTestApp( + + + , ); expect(rendered.queryByText('Create a new component')).toBeInTheDocument(); expect(rendered.queryByTestId('loading-progress')).toBeInTheDocument(); // Need to cleanup the promise or will timeout - resolve!(); + act(() => { + resolve!({ items: [] }); + }); }); it('navigates away if no template was loaded', async () => { - catalogApiMock.getEntities.mockResolvedValueOnce([]); + catalogApiMock.getEntities.mockResolvedValueOnce({ items: [] }); const rendered = await renderWithEffects( diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index cb6b38903e..4836cdab56 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -27,28 +27,26 @@ import { catalogApiRef } from '@backstage/plugin-catalog'; import { LinearProgress } from '@material-ui/core'; import { IChangeEvent } from '@rjsf/core'; import React, { useState } from 'react'; -import { useParams } from 'react-router-dom'; -import useStaleWhileRevalidate from 'swr'; -import { scaffolderApiRef } from '../../api'; -import { JobStatusModal } from '../JobStatusModal'; -import { Job } from '../../types'; -import { MultistepJsonForm } from '../MultistepJsonForm'; import { Navigate } from 'react-router'; +import { useParams } from 'react-router-dom'; +import { useAsync } from 'react-use'; +import { scaffolderApiRef } from '../../api'; import { rootRoute } from '../../routes'; +import { Job } from '../../types'; +import { JobStatusModal } from '../JobStatusModal'; +import { MultistepJsonForm } from '../MultistepJsonForm'; const useTemplate = ( templateName: string, catalogApi: typeof catalogApiRef.T, ) => { - const { data, error } = useStaleWhileRevalidate( - `templates/${templateName}`, - async () => - catalogApi.getEntities({ - kind: 'Template', - 'metadata.name': templateName, - }) as Promise, - ); - return { template: data?.[0], loading: !error && !data, error }; + const { value, loading, error } = useAsync(async () => { + const response = await catalogApi.getEntities({ + filter: { kind: 'Template', 'metadata.name': templateName }, + }); + return response.items as TemplateEntityV1alpha1[]; + }); + return { template: value?.[0], loading, error }; }; const OWNER_REPO_SCHEMA = { @@ -110,7 +108,7 @@ export const TemplatePage = () => { const handleCreateComplete = async (job: Job) => { const target = job.metadata.remoteUrl?.replace( /\.git$/, - // TODO(Rugvip): This is not the location we want. As part of scaffodler v2 we + // TODO(Rugvip): This is not the location we want. As part of scaffolder v2 we // want this to be more flexible, but before that we might want // to update all templates to use catalog-info.yaml instead. '/blob/master/component-info.yaml', diff --git a/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx b/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx index d5d0e19e08..0924b90fc1 100644 --- a/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsHome.test.tsx @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; import { ApiProvider, ApiRegistry } from '@backstage/core-api'; import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog'; import { wrapInTestApp } from '@backstage/test-utils'; @@ -24,7 +23,7 @@ import { TechDocsHome } from './TechDocsHome'; describe('TechDocs Home', () => { const catalogApi: Partial = { - getEntities: () => Promise.resolve([] as Entity[]), + getEntities: () => Promise.resolve({ items: [] }), }; const apiRegistry = ApiRegistry.from([[catalogApiRef, catalogApi]]); diff --git a/plugins/techdocs/src/reader/components/TechDocsHome.tsx b/plugins/techdocs/src/reader/components/TechDocsHome.tsx index 0408a540ca..9f569dd167 100644 --- a/plugins/techdocs/src/reader/components/TechDocsHome.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsHome.tsx @@ -14,19 +14,19 @@ * limitations under the License. */ -import React from 'react'; -import { useAsync } from 'react-use'; -import { useNavigate, generatePath } from 'react-router-dom'; -import { Grid } from '@material-ui/core'; import { + Content, + Header, ItemCard, + Page, Progress, useApi, - Content, - Page, - Header, } from '@backstage/core'; import { catalogApiRef } from '@backstage/plugin-catalog'; +import { Grid } from '@material-ui/core'; +import React from 'react'; +import { generatePath, useNavigate } from 'react-router-dom'; +import { useAsync } from 'react-use'; import { rootDocsRouteRef } from '../../plugin'; export const TechDocsHome = () => { @@ -34,8 +34,8 @@ export const TechDocsHome = () => { const navigate = useNavigate(); const { value, loading, error } = useAsync(async () => { - const entities = await catalogApi.getEntities(); - return entities.filter(entity => { + const response = await catalogApi.getEntities(); + return response.items.filter(entity => { return !!entity.metadata.annotations?.['backstage.io/techdocs-ref']; }); }); From f8cb1a976d82e8e54cd70dd31dc0957506f092b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 18 Nov 2020 20:09:38 +0100 Subject: [PATCH 51/51] fix: unbreak search after merging catalog client change --- plugins/search/src/apis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index 1783b91d2a..f3a050b2a2 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -35,7 +35,7 @@ class SearchApi { private async entities() { const entities = await this.catalogApi.getEntities(); - return entities.map((result: any) => ({ + return entities.items.map((result: any) => ({ name: result.metadata.name, description: result.metadata.description, owner: result.spec.owner,