diff --git a/.changeset/beige-glasses-dream.md b/.changeset/beige-glasses-dream.md new file mode 100644 index 0000000000..c1a9ad1857 --- /dev/null +++ b/.changeset/beige-glasses-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Allow to pass custom TemplateCard to ScaffolderPage diff --git a/.changeset/big-eyes-dance.md b/.changeset/big-eyes-dance.md new file mode 100644 index 0000000000..ab14979619 --- /dev/null +++ b/.changeset/big-eyes-dance.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Added support for importing font files. Imports in CSS via `url()` are supported for the final frontend bundle, but not for packages that are built for publishing. Module imports of fonts files from TypeScript are supported everywhere. diff --git a/.changeset/cuddly-cooks-fry.md b/.changeset/cuddly-cooks-fry.md new file mode 100644 index 0000000000..028b0291bd --- /dev/null +++ b/.changeset/cuddly-cooks-fry.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Added a `useEntityKinds` hook to load a unique list of entity kinds from the catalog. +Fixed a bug in `EntityTypePicker` where the component did not hide when no types were available in returned entities. diff --git a/.changeset/cuddly-impalas-wave.md b/.changeset/cuddly-impalas-wave.md new file mode 100644 index 0000000000..1199ef1482 --- /dev/null +++ b/.changeset/cuddly-impalas-wave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +GitHub discovery processor passes over repositories that do not have a default branch diff --git a/.changeset/fair-files-rest.md b/.changeset/fair-files-rest.md new file mode 100644 index 0000000000..68d3246aa4 --- /dev/null +++ b/.changeset/fair-files-rest.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +GitHub create repository webhook action: `github:webhook` for Backstage plugin Scaffolder has been added. diff --git a/.changeset/famous-pillows-love.md b/.changeset/famous-pillows-love.md deleted file mode 100644 index 972a2b10b4..0000000000 --- a/.changeset/famous-pillows-love.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/core-components': patch -'@backstage/plugin-catalog': patch -'@backstage/plugin-catalog-react': patch ---- - -Improve accessibility of core & catalog components by adjusting them with non-breaking changes. diff --git a/.changeset/forty-terms-join.md b/.changeset/forty-terms-join.md new file mode 100644 index 0000000000..1476ebff9c --- /dev/null +++ b/.changeset/forty-terms-join.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/techdocs-common': patch +--- + +Adding in-context search to TechDocs Reader component. Using existing search-backend to query for indexed search results scoped into a specific entity's techdocs. Needs TechDocsCollator enabled on the backend to work. + +Adding extra information to indexed tech docs documents for search. diff --git a/.changeset/four-bobcats-confess.md b/.changeset/four-bobcats-confess.md new file mode 100644 index 0000000000..2126329e3e --- /dev/null +++ b/.changeset/four-bobcats-confess.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +GitHubWebhook Action can be created with a default webhook secret. This allows getting secret from environment variable as an alternative to get it from context. diff --git a/.changeset/friendly-tables-heal.md b/.changeset/friendly-tables-heal.md new file mode 100644 index 0000000000..8aa987d409 --- /dev/null +++ b/.changeset/friendly-tables-heal.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Add a warning log message that `passport-saml` will require a `cert` config parameter imminently. + +We intend to upgrade this package soon, past the point where we will start to strictly require the `auth.saml.cert` configuration parameter to be present. To avoid issues starting your auth backend, please diff --git a/.changeset/fuzzy-pigs-wash.md b/.changeset/fuzzy-pigs-wash.md new file mode 100644 index 0000000000..d4a0ce4602 --- /dev/null +++ b/.changeset/fuzzy-pigs-wash.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Update to properly handle Azure DevOps Server download URL diff --git a/.changeset/good-papayas-fetch.md b/.changeset/good-papayas-fetch.md new file mode 100644 index 0000000000..6cd74ce064 --- /dev/null +++ b/.changeset/good-papayas-fetch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Adds optional namespacing for users in the GitHub Multi Org Plugin diff --git a/.changeset/hip-beers-attend.md b/.changeset/hip-beers-attend.md new file mode 100644 index 0000000000..802bfe7397 --- /dev/null +++ b/.changeset/hip-beers-attend.md @@ -0,0 +1,7 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-shortcuts': patch +--- + +Bumped `react-hook-form` to `^7.12.2` diff --git a/.changeset/honest-suits-approve.md b/.changeset/honest-suits-approve.md new file mode 100644 index 0000000000..59cdb09762 --- /dev/null +++ b/.changeset/honest-suits-approve.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-git-release-manager': patch +--- + +Wrap each feature in custom feature arrays with an element containing a 'key' to avoid missing-key-warnings diff --git a/.changeset/kind-needles-appear.md b/.changeset/kind-needles-appear.md deleted file mode 100644 index 9a8f56bf48..0000000000 --- a/.changeset/kind-needles-appear.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend-module-ldap': patch ---- - -chore(deps): bump `@types/ldapjs` from 1.0.10 to 2.2.0 diff --git a/.changeset/lazy-apples-scream.md b/.changeset/lazy-apples-scream.md deleted file mode 100644 index b666c3498a..0000000000 --- a/.changeset/lazy-apples-scream.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-auth-backend': patch ---- - -Allow to configure SAML auth `acceptedClockSkewMs` diff --git a/.changeset/eighty-snakes-melt.md b/.changeset/lovely-guests-smell.md similarity index 50% rename from .changeset/eighty-snakes-melt.md rename to .changeset/lovely-guests-smell.md index f980d1c02e..ef6bb8c8f4 100644 --- a/.changeset/eighty-snakes-melt.md +++ b/.changeset/lovely-guests-smell.md @@ -2,4 +2,4 @@ '@backstage/plugin-scaffolder': patch --- -chore: set the lifecycle as beta logo +Use themeId in TemplateCard from theme object diff --git a/.changeset/nasty-ads-nail.md b/.changeset/nasty-ads-nail.md new file mode 100644 index 0000000000..0518b1e569 --- /dev/null +++ b/.changeset/nasty-ads-nail.md @@ -0,0 +1,10 @@ +--- +'@backstage/core-components': minor +--- + +Changed the `titleComponent` prop on `ContentHeader` to accept `ReactNode` instead of a React `ComponentType`. Usages of this prop should be converted from passing a component to passing in the rendered element: + +```diff +- ++}> +``` diff --git a/.changeset/nasty-jokes-camp.md b/.changeset/nasty-jokes-camp.md new file mode 100644 index 0000000000..e30ff75911 --- /dev/null +++ b/.changeset/nasty-jokes-camp.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-org': patch +--- + +Don't open a new tab when clicking on the ownership card. diff --git a/.changeset/pink-glasses-type.md b/.changeset/pink-glasses-type.md deleted file mode 100644 index 45eb01987d..0000000000 --- a/.changeset/pink-glasses-type.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-api-docs': patch ---- - -Add explicit import for `isomorphic-form-data` needed for `swagger-ui-react` diff --git a/.changeset/pink-plants-reflect.md b/.changeset/pink-plants-reflect.md deleted file mode 100644 index 7b0069644b..0000000000 --- a/.changeset/pink-plants-reflect.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/create-app': patch ---- - -Update the `software-templates` to point to `main` branch instead of `master` diff --git a/.changeset/pretty-cameras-give.md b/.changeset/pretty-cameras-give.md new file mode 100644 index 0000000000..8f9f4a83fa --- /dev/null +++ b/.changeset/pretty-cameras-give.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Update the `AboutCard` to properly support non-standard entity types and rework the defaults for the build-in kinds. + +This change also uses `useElementFilter(...)` instead of `React.children.count(...)` in `AboutField` to properly recognize whether children are available. diff --git a/.changeset/real-pugs-rhyme.md b/.changeset/real-pugs-rhyme.md new file mode 100644 index 0000000000..94cec27487 --- /dev/null +++ b/.changeset/real-pugs-rhyme.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Updated dependencies diff --git a/.changeset/search-happy-owls-sneeze.md b/.changeset/search-happy-owls-sneeze.md new file mode 100644 index 0000000000..1f3cfd5dd1 --- /dev/null +++ b/.changeset/search-happy-owls-sneeze.md @@ -0,0 +1,13 @@ +--- +'@backstage/search-common': minor +'@backstage/plugin-search': patch +'@backstage/plugin-search-backend': patch +'@backstage/plugin-search-backend-module-elasticsearch': patch +'@backstage/plugin-search-backend-module-pg': minor +'@backstage/plugin-search-backend-node': patch +--- + +Implement optional `pageCursor` based paging in search. + +To use paging in your app, add a `` to your +`SearchPage.tsx`. diff --git a/.changeset/serious-maps-try.md b/.changeset/serious-maps-try.md new file mode 100644 index 0000000000..78e953dfd6 --- /dev/null +++ b/.changeset/serious-maps-try.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +typo fix `tenentId` in Azure auth provider docs diff --git a/.changeset/serious-pumpkins-hug.md b/.changeset/serious-pumpkins-hug.md new file mode 100644 index 0000000000..9441f3aeb7 --- /dev/null +++ b/.changeset/serious-pumpkins-hug.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Updated dependencies diff --git a/.changeset/shiny-guests-explain.md b/.changeset/shiny-guests-explain.md new file mode 100644 index 0000000000..6bd99e4aa6 --- /dev/null +++ b/.changeset/shiny-guests-explain.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-org': patch +--- + +This change hides pagination counter of search tables and group members list when results fit in one page diff --git a/.changeset/short-years-smile.md b/.changeset/short-years-smile.md new file mode 100644 index 0000000000..6e7b882c1e --- /dev/null +++ b/.changeset/short-years-smile.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Added the ability to switch entity kind on the catalog index page. This is a non-breaking change, but if you created a custom `CatalogPage` and wish to use this feature, make the modifications shown on [#6895](https://github.com/backstage/backstage/pull/6895). diff --git a/.changeset/sixty-cougars-study.md b/.changeset/sixty-cougars-study.md new file mode 100644 index 0000000000..80db2913d0 --- /dev/null +++ b/.changeset/sixty-cougars-study.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-shortcuts': patch +--- + +Fix material-ui warning about deprecated `onExit` prop diff --git a/.changeset/slimy-elephants-sell.md b/.changeset/slimy-elephants-sell.md deleted file mode 100644 index c002e8e950..0000000000 --- a/.changeset/slimy-elephants-sell.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/plugin-kubernetes-backend': patch -'@backstage/plugin-kubernetes-common': patch -'@backstage/plugin-kubernetes': patch ---- - -chore(deps): bump `@kubernetes/client-node` from 0.14.3 to 0.15.0 diff --git a/.changeset/small-emus-sin.md b/.changeset/small-emus-sin.md deleted file mode 100644 index e5683d18f9..0000000000 --- a/.changeset/small-emus-sin.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-react': patch ---- - -Fix `EntityListProvider` to not update url if unmounted diff --git a/.changeset/strange-zoos-grab.md b/.changeset/strange-zoos-grab.md new file mode 100644 index 0000000000..78a6c6eeee --- /dev/null +++ b/.changeset/strange-zoos-grab.md @@ -0,0 +1,11 @@ +--- +'@backstage/catalog-model': patch +--- + +Add an optional `metadata.title` field to all entity kinds. + +This used to be available on only the `Template` kind, and we have decided that the metadata block should be the same for all kinds. A title can be useful especially in large and complex catalogs where users have a tough time navigating or discerning among the entities. + +It also carries some risk. You do not want to end up giving a title that collides with an actual name, which at best leads to confusion and at worst could be a liability. We do not perform any collision detection in the catalog. If you want to disallow this facility you may want to add a small processor that makes sure it's not set. + +At the time of writing this message, only the scaffolder actually makes use of this field for display purposes. diff --git a/.changeset/strong-beers-ring.md b/.changeset/strong-beers-ring.md new file mode 100644 index 0000000000..17a6d89fcc --- /dev/null +++ b/.changeset/strong-beers-ring.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Minor tweaks to the create-app template to match earlier documented changes diff --git a/.changeset/strong-ravens-smoke.md b/.changeset/strong-ravens-smoke.md new file mode 100644 index 0000000000..478eb1c4c2 --- /dev/null +++ b/.changeset/strong-ravens-smoke.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Added signIn and authHandler resolver for oAuth2 provider diff --git a/.changeset/stupid-ants-call.md b/.changeset/stupid-ants-call.md new file mode 100644 index 0000000000..b2dc16d0e2 --- /dev/null +++ b/.changeset/stupid-ants-call.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Export type that are needed to implement a new `UrlReader` diff --git a/.changeset/sweet-jars-travel.md b/.changeset/sweet-jars-travel.md new file mode 100644 index 0000000000..83fcd13623 --- /dev/null +++ b/.changeset/sweet-jars-travel.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-sonarqube': patch +--- + +Update README to show a simpler proxy configuration. diff --git a/.changeset/techdocs-search-log-fatigue.md b/.changeset/techdocs-search-log-fatigue.md new file mode 100644 index 0000000000..dc81563d8c --- /dev/null +++ b/.changeset/techdocs-search-log-fatigue.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +Errors encountered while attempting to load TechDocs search indices at +collation-time are now logged at `DEBUG` instead of `WARN` level. diff --git a/.changeset/techdocs-she-was-gone.md b/.changeset/techdocs-she-was-gone.md deleted file mode 100644 index a6821abcf6..0000000000 --- a/.changeset/techdocs-she-was-gone.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-techdocs': patch ---- - -TechDocs now uses a "safe by default" sanitization library, rather than relying on its own, hard-coded list of allowable tags and attributes. diff --git a/.changeset/techdocs-the-beta-local.md b/.changeset/techdocs-the-beta-local.md new file mode 100644 index 0000000000..9f94627a60 --- /dev/null +++ b/.changeset/techdocs-the-beta-local.md @@ -0,0 +1,6 @@ +--- +'@backstage/techdocs-common': patch +--- + +"Local" (out-of-the-box) publisher explicitly follows lower-case entity triplet +logic. diff --git a/.changeset/techdocs-three-icons-collect.md b/.changeset/techdocs-three-icons-collect.md new file mode 100644 index 0000000000..a3c005f82d --- /dev/null +++ b/.changeset/techdocs-three-icons-collect.md @@ -0,0 +1,7 @@ +--- +'@backstage/techdocs-common': patch +--- + +Only write the updated `mkdocs.yml` file if the content was updated. + +This keeps local files unchanged if the `dir` annotation is used in combination with the `file` location. diff --git a/.changeset/techdocs-trains-beg.md b/.changeset/techdocs-trains-beg.md new file mode 100644 index 0000000000..05161b7093 --- /dev/null +++ b/.changeset/techdocs-trains-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Force using `posix` path for cloud storage diff --git a/.changeset/techdocs-warm-walls-smile.md b/.changeset/techdocs-warm-walls-smile.md new file mode 100644 index 0000000000..ab7469ec65 --- /dev/null +++ b/.changeset/techdocs-warm-walls-smile.md @@ -0,0 +1,17 @@ +--- +'@backstage/techdocs-common': minor +--- + +Set the correct `edit_uri` or `repo_url` for documentation pages that are hosted on GitHub and GitLab. + +The constructor of the `TechDocsGenerator` changed. +Prefer the use of `TechdocsGenerator.fromConfig(…)` instead: + +```diff +- const techdocsGenerator = new TechdocsGenerator({ ++ const techdocsGenerator = TechdocsGenerator.fromConfig(config, { + logger, + containerRunner, +- config, + }); +``` diff --git a/.changeset/thick-trees-pay.md b/.changeset/thick-trees-pay.md new file mode 100644 index 0000000000..6ee7b514ca --- /dev/null +++ b/.changeset/thick-trees-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Use a `Link` for the edit button on the `AboutCard` instead of doing `window.open(...)` diff --git a/.changeset/tiny-crews-notice.md b/.changeset/tiny-crews-notice.md new file mode 100644 index 0000000000..3d61ca44eb --- /dev/null +++ b/.changeset/tiny-crews-notice.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Export GitHub to allow use with Identity resolver diff --git a/.changeset/two-hats-judge.md b/.changeset/two-hats-judge.md new file mode 100644 index 0000000000..27768aba8b --- /dev/null +++ b/.changeset/two-hats-judge.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Display [metadata.title](https://backstage.io/docs/features/software-catalog/descriptor-format#title-optional) for components on the TechDocs homepage, if defined; otherwise fall back to `metadata.name` as displayed before. diff --git a/.changeset/weak-toes-agree.md b/.changeset/weak-toes-agree.md deleted file mode 100644 index 8bdcf46fa2..0000000000 --- a/.changeset/weak-toes-agree.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-xcmetrics': minor ---- - -New data in form of trend lines, status timeline and other is added to the dashboard of XCMetrics to give a better understanding of how the build system is behaving. diff --git a/.changeset/wicked-singers-push.md b/.changeset/wicked-singers-push.md new file mode 100644 index 0000000000..3c34a959ed --- /dev/null +++ b/.changeset/wicked-singers-push.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-org': patch +--- + +Fix OwnershipCard links to support namespaced owners diff --git a/.changeset/yellow-windows-fix.md b/.changeset/yellow-windows-fix.md new file mode 100644 index 0000000000..7572fdce88 --- /dev/null +++ b/.changeset/yellow-windows-fix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +refactor: extract common Octokit related code and use it in actions: `publish:github`, `github:actions:dispatch`, `github:webhook`. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fda500ed02..781feb1f04 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /plugins/ilert @yacut +/plugins/home @backstage/techdocs-core /packages/search-common @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core /.changeset/cost-insights-* @backstage/silver-lining diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 48740ef367..dd9224fc84 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -15,6 +15,7 @@ Avro backrub Bigtable Billett +Bitbucket Bitrise Blackbox bool @@ -75,6 +76,7 @@ dockerode Docusaurus env Env +elasticsearch esbuild eslint etag @@ -83,6 +85,8 @@ facto failover Fargate Figma +firehydrant +FireHydrant Firekube Fiverr gitbeaker @@ -138,6 +142,7 @@ maintainership makefile md memcache +memoize memoized microservice microservices @@ -211,6 +216,7 @@ repo Repo repos rerender +rerenders Reusability reusability roadmaps @@ -222,6 +228,7 @@ Routable rst rsync ruleset +runbook sam sanitization scaffolded diff --git a/.github/workflows/chromatic-storybook-test.yml b/.github/workflows/chromatic-storybook-test.yml index d189236987..4c64db309d 100644 --- a/.github/workflows/chromatic-storybook-test.yml +++ b/.github/workflows/chromatic-storybook-test.yml @@ -5,6 +5,7 @@ on: - '.github/workflows/chromatic-storybook-test.yml' - 'packages/storybook/**' - 'packages/core-components/src/**' + - '**/*.stories.tsx' jobs: chromatic: @@ -50,4 +51,5 @@ jobs: # projetToken intentionally shared to allow collaborators to run Chromatic on forks # https://www.chromatic.com/docs/custom-ci-provider#run-chromatic-on-external-forks-of-open-source-projects projectToken: 9tzak77m9nj - storybookBuildDir: 'packages/storybook/dist' + workingDir: 'packages/storybook' + storybookBuildDir: 'dist' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f013d10f8..8361654038 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: run: node scripts/verify-links.js - name: prettier - run: yarn prettier:check + run: yarn prettier:check '!ADOPTERS.md' - name: lock run: yarn lock:check diff --git a/.github/workflows/e2e-win.yml b/.github/workflows/e2e-win.yml index 1844700cea..3e2fbe0863 100644 --- a/.github/workflows/e2e-win.yml +++ b/.github/workflows/e2e-win.yml @@ -25,6 +25,13 @@ jobs: name: Node ${{ matrix.node-version }} on ${{ matrix.os }} steps: + # In order to have the create-app template function as if it was downloaded from NPM + # we need to make sure we checkout files with LF line endings only + - name: Set git to use LF + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - uses: actions/checkout@v2 - name: use node.js ${{ matrix.node-version }} diff --git a/.github/workflows/prettify.yml b/.github/workflows/prettify.yml new file mode 100644 index 0000000000..4a34ff0c88 --- /dev/null +++ b/.github/workflows/prettify.yml @@ -0,0 +1,54 @@ +name: Prettier + +on: + push: + branches: + - master + +jobs: + autofix-markdown: + name: Autofix Markdown files using Prettier + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Fetch changes to previous commit - required for 'only_changed' in Prettier action + fetch-depth: 0 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-v${{ matrix.node-version }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - name: Run Prettier on ADOPTERS.md + uses: creyD/prettier_action@v3.1 + with: + # Modifies commit only if prettier autofixed the ADOPTERS.md + prettier_options: --config docs/prettier.config.js --write ADOPTERS.md + only_changed: true + commit_message: 'Autofix ADOPTERS.md using Prettier' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.imgbotconfig b/.imgbotconfig index f337e087dc..2f4af2ed5e 100644 --- a/.imgbotconfig +++ b/.imgbotconfig @@ -1,5 +1,3 @@ { - "ignoredFiles": [ - "docs/assets/**/*.svg" - ] + "ignoredFiles": ["docs/assets/**/*.svg"] } diff --git a/ADOPTERS.md b/ADOPTERS.md index 041242c083..18d1388936 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,41 +1,47 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Erdogan Oksuz](https://github.com/erdoganoksuz) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀 | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2caa8c9c9c..466c77c575 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,7 +116,7 @@ In general, changesets are not needed for the documentation, build utilities, co 1. Run `yarn changeset` 2. Select which packages you want to include a changeset for -3. Select impact of change that you're introducing (patch, minor, or major) +3. Select impact of change that you're introducing, using `minor` for breaking changes and `patch` otherwise. We do not use `major` changes while packages are at version `0.x`. 4. Add generated changeset to Git 5. Push the commit with your changeset to the branch associated with your PR 6. Accept our gratitude for making the release process easier on the maintainers diff --git a/README.md b/README.md index 98da194c1b..1078631926 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ # [Backstage](https://backstage.io) -_During the month of July the majority of the maintainers will be on summer vacation 🏖️ Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_ - [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects) [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22) @@ -63,3 +61,9 @@ Check out [the documentation](https://backstage.io/docs/getting-started) on how Copyright 2020-2021 © The Backstage Authors. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page: https://www.linuxfoundation.org/trademark-usage Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + +## Security + +Please report sensitive security issues via Spotify's [bug-bounty program](https://hackerone.com/spotify) rather than GitHub. + +For further details please see our complete [security release process](SECURITY.md). diff --git a/SECURITY.md b/SECURITY.md index 3ec51b4210..b410404bcf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,3 +9,20 @@ ## Reporting a Vulnerability Please report sensitive security issues via Spotify's [bug-bounty program](https://hackerone.com/spotify) rather than GitHub. + +If you have questions about a potential vulnerability, please reach out on Discord by asking for a maintainer in the `#support` channel, or via direct message to a maintainer. + +## Remediation and Notification Process + +Vulnerabilities are handled and published through [GitHub Security Advisories](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories). + +In the event of a vulnerability the runbook for the maintainers is as follows: + +1. Create a [new draft security advisory](https://github.com/backstage/backstage/security/advisories/new). The values and descriptions don't need to be perfect to begin with as they can be edited later. For severity, use the "Assess severity using CVSS" and refer to [the guide](https://www.first.org/cvss/v3.1/user-guide) for help. +2. Request a CVE identification number. It can take up to 72h for one to be assigned so be sure to do this early on. +3. If there is not already a patch for the vulnerability, collaborate on one in a private fork: + 1. Head to the security advisory on GitHub and [create a private fork](https://docs.github.com/en/code-security/security-advisories/collaborating-in-a-temporary-private-fork-to-resolve-a-security-vulnerability) + 1. [Invite any collaborators](https://docs.github.com/en/code-security/security-advisories/adding-a-collaborator-to-a-security-advisory) from outside the maintainer team that should be part of creating a fix. + 1. Create one or multiple Pull Requests with fixes towards the fork. Note that these PRs will not have CI checks run on them, so more care should be taken to run local validation. The PRs are also not merged like normal PRs, but are instead merged straight into the main repo all at once when the merge button is hit on the security advisory. +4. Once the fix is ready in a PR or private fork and it is time to release it, there are a couple of options. Either merge into the main branch with a changeset and wait for a regular release, or do a quick release by manually bumping the version in `package.json` of the affected package, along with a manual `CHANGELOG.md` entry. Note that a quick release will only work if the package does not have any other pending changes that depend on pending changes in other packages, so be sure to manually check that first, and fall back to an early regular release if needed. In general it's best to stick with the regular release flow, with the quick release being used only for time sensitive fixes. +5. Finalize and publish the security advisory. Note that once you hit the publish button it's no longer possible to edit the advisory. Just like the CVE number this can take up to 72h, and expect it to be slower than the CVE number request. diff --git a/app-config.yaml b/app-config.yaml index 074e28c086..c5c4a23bdd 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -133,6 +133,9 @@ kafka: brokers: - localhost:9092 +allure: + baseUrl: http://localhost:5050/allure-docker-service + integrations: github: - host: github.com @@ -152,12 +155,21 @@ integrations: - host: bitbucket.org username: ${BITBUCKET_USERNAME} appPassword: ${BITBUCKET_APP_PASSWORD} + ### Example for how to add your bitbucket server instance using the API: + # - host: server.bitbucket.com + # apiBaseUrl: server.bitbucket.com + # username: ${BITBUCKET_SERVER_USERNAME} + # appPassword: ${BITBUCKET_SERVER_APP_PASSWORD} azure: - host: dev.azure.com token: ${AZURE_TOKEN} -# googleGcs: -# clientEmail: 'example@example.com' -# privateKey: ${GCS_PRIVATE_KEY} + # googleGcs: + # clientEmail: 'example@example.com' + # privateKey: ${GCS_PRIVATE_KEY} + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} catalog: rules: @@ -256,6 +268,8 @@ scaffolder: # defaultAuthor: # name: Scaffolder # email: scaffolder@backstage.io + # Use to customize the default commit message when new components are created + # defaultCommitMessage: 'Initial commit' github: token: ${GITHUB_TOKEN} visibility: public # or 'internal' or 'private' diff --git a/catalog-info.yaml b/catalog-info.yaml index 0cab0c558a..6de6fd3426 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -15,7 +15,7 @@ metadata: url: https://discord.com/invite/EBHEGzX annotations: github.com/project-slug: backstage/backstage - backstage.io/techdocs-ref: url:https://github.com/backstage/backstage + backstage.io/techdocs-ref: dir:. lighthouse.com/website-url: https://backstage.io spec: type: library diff --git a/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild b/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild index 1e0134017f..f0fa4a034b 100644 --- a/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild +++ b/contrib/docker/frontend-with-nginx/Dockerfile.hostbuild @@ -8,7 +8,6 @@ # simply copies in the build output into the image. If you want to also perform # the build itself inside docker, use Dockerfile.build in this folder instead. - # USAGE: # # - Copy this file and the "docker" folder from this directory to your project @@ -23,7 +22,7 @@ # # yarn install # yarn tsc -# yarn build --config --config ... +# yarn workspace app build --config --config ... # docker build -t backstage-frontend -f Dockerfile.hostbuild . @@ -38,4 +37,3 @@ COPY docker/default.conf.template /etc/nginx/templates/default.conf.template COPY docker/inject-config.sh /docker-entrypoint.d/40-inject-config.sh ENV PORT 80 - diff --git a/docs/assets/features/catalog/life-of-an-entity_ingestion.svg b/docs/assets/features/catalog/life-of-an-entity_ingestion.svg new file mode 100644 index 0000000000..b250744fcd --- /dev/null +++ b/docs/assets/features/catalog/life-of-an-entity_ingestion.svg @@ -0,0 +1,3 @@ + + +
Unprocessed Entities
Unprocessed Entities
change events
change events
Entity Provider
Entity Provider
change events
change events
Entity Provider
Entity Provider
External
Source
External...
External
Source
External...
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Entity
Viewer does not support full SVG 1.1
\ No newline at end of file diff --git a/docs/assets/features/catalog/life-of-an-entity_overview.svg b/docs/assets/features/catalog/life-of-an-entity_overview.svg new file mode 100644 index 0000000000..3fc35b4775 --- /dev/null +++ b/docs/assets/features/catalog/life-of-an-entity_overview.svg @@ -0,0 +1,3 @@ + + +
Unprocessed Entities
Unprocessed Entities
Processed Entities
Processed Entities
Relations
Relations
Edges
Edges
Final Entities
Final Entities
Search
Search
Errors
Errors
Processors
Processors
Stitcher
Stitcher
Entity Providers
Entity Providers
External
Sources
External...
Catalog
API
Catalog...
Processing
Processing
Stitching
Stitching
Ingestion
Ingestion
Viewer does not support full SVG 1.1
\ No newline at end of file diff --git a/docs/auth/add-auth-provider.md b/docs/auth/add-auth-provider.md index 56586cbfc6..472a6a0abc 100644 --- a/docs/auth/add-auth-provider.md +++ b/docs/auth/add-auth-provider.md @@ -244,9 +244,7 @@ export interface OAuthProviderHandlers { req: express.Request, options: Record, ): Promise; - handler( - req: express.Request, - ): Promise<{ + handler(req: express.Request): Promise<{ response: AuthResponse; refreshToken?: string; }>; diff --git a/docs/auth/github/provider.md b/docs/auth/github/provider.md index 4b65c8c3c2..81b2e98f2c 100644 --- a/docs/auth/github/provider.md +++ b/docs/auth/github/provider.md @@ -10,11 +10,16 @@ that can authenticate users using GitHub or GitHub Enterprise OAuth. ## Create an OAuth App on GitHub -To add GitHub authentication, you must create an OAuth App from the GitHub +To add GitHub authentication, you must create either a GitHub App, or an OAuth +App from the GitHub [developer settings](https://github.com/settings/developers). The `Homepage URL` should point to Backstage's frontend, while the `Authorization callback URL` will point to the auth backend. +Note that if you're using a GitHub App, the allowed scopes are configured as +part of that app. This means you need to verify what scopes the plugins you use +require, so be sure to check the plugin READMEs for that information. + Settings for local development: - Application name: Backstage (or your custom app name) @@ -51,3 +56,11 @@ The GitHub provider is a structure with three configuration keys: To add the provider to the frontend, add the `githubAuthApi` reference and `SignInPage` component as shown in [Adding the provider to the sign-in page](../index.md#adding-the-provider-to-the-sign-in-page). + +## Difference between GitHub Apps and GitHub OAuth Apps + +GitHub Apps handle OAuth scope at the app installation level, meaning that the +`scope` parameter for the call to `getAccessToken` in the frontend has no +effect. When calling `getAccessToken` in open source plugins, one should still +include the appropriate scope, but also document in the plugin README what +scopes are required for GitHub Apps. diff --git a/docs/auth/identity-resolver.md b/docs/auth/identity-resolver.md index 814ff63729..2c36ddcd7f 100644 --- a/docs/auth/identity-resolver.md +++ b/docs/auth/identity-resolver.md @@ -111,6 +111,48 @@ export default async function createPlugin({ ... ``` +## Resolving membership through the catalog + +If you want to provide additional claims through Sign-In resolvers but still +have the software catalog handle group (and transitive group) membership, you +can do this using the `CatalogIdentityClient` provided as context to Sign-In +resolvers: + +```ts +export default async function createPlugin({ + ... +}: PluginEnvironment): Promise { + return await createRouter({ + ... + providerFactories: { + google: createGoogleProvider({ + signIn: { + resolver: async ({ profile: { email } }, ctx) => { + const [id] = email?.split('@') ?? ''; + // Fetch from an external system that returns entity claims like: + // ['user:default/breanna.davison', ...] + const ent = await externalSystemClient.getUsernames(email); + + // Resolve group membership from the Backstage catalog + const fullEnt = await ctx.catalogIdentityClient.resolveCatalogMembership({ + entityRefs: [id].concat(ent), + logger: ctx.logger, + }); + const token = await ctx.tokenIssuer.issueToken({ + claims: { sub: id, ent: fullEnt }, + }); + return { id, token }; + }, + }, + }), + ... +``` + +The `resolveCatalogMembership` method will retrieve the referenced entities from +the catalog, if possible, and check for +[memberOf](../features/software-catalog/well-known-relations.md#memberof-and-hasmember) +relations to add additional entity claims. + ## AuthHandler Similar to a custom sign-in resolver, you can also write a custom auth handler diff --git a/docs/auth/microsoft/provider.md b/docs/auth/microsoft/provider.md index b64f9b1902..52f463bab4 100644 --- a/docs/auth/microsoft/provider.md +++ b/docs/auth/microsoft/provider.md @@ -44,7 +44,7 @@ The Microsoft provider is a structure with three configuration keys: - `clientId`: Application (client) ID, found on App Registration > Overview - `clientSecret`: Secret, found on App Registration > Certificates & secrets -- `tenentId`: Directory (tenant) ID, found on App Registration > Overview +- `tenantId`: Directory (tenant) ID, found on App Registration > Overview ## Adding the provider to the Backstage frontend diff --git a/docs/features/search/architecture.md b/docs/features/search/architecture.md index 0a1ebea4b0..4fac0e7cff 100644 --- a/docs/features/search/architecture.md +++ b/docs/features/search/architecture.md @@ -4,8 +4,6 @@ title: Search Architecture description: Documentation on Search Architecture --- -# Search Architecture - > _This architecture has not been fully implemented yet. Find our milestones to > follow our progress and help contribute on the > [Search Roadmap](./README.md#project-roadmap)._ diff --git a/docs/features/search/concepts.md b/docs/features/search/concepts.md index 5da7c9c58f..8cb17b12e6 100644 --- a/docs/features/search/concepts.md +++ b/docs/features/search/concepts.md @@ -4,8 +4,6 @@ title: Search Concepts description: Documentation on Backstage Search Concepts --- -# Search Concepts - Backstage Search lets you find the right information you are looking for in the Backstage ecosystem. diff --git a/docs/features/search/getting-started.md b/docs/features/search/getting-started.md index 2bd26faf8b..aa40c1e539 100644 --- a/docs/features/search/getting-started.md +++ b/docs/features/search/getting-started.md @@ -4,8 +4,6 @@ title: Getting Started with Search description: How to set up and install Backstage Search --- -# Getting Started - Search functions as a plugin to Backstage, so you will need to use Backstage to use Search. @@ -252,13 +250,9 @@ an example: Backstage Search isn't a search engine itself, rather, it provides an interface between your Backstage instance and a [Search Engine](./concepts.md#search-engines) of your choice. Currently, we only -support one, an in-memory search Engine called Lunr. It can be instantiated like -this: - -```typescript -const searchEngine = new LunrSearchEngine({ logger }); -const indexBuilder = new IndexBuilder({ logger, searchEngine }); -``` +support two engines, an in-memory search Engine called Lunr and ElasticSearch. +See [Search Engines](./search-engines.md) documentation for more information how +to configure these in your Backstage instance. Backstage Search can be used to power search of anything! Plugins like the Catalog offer default [collators](./concepts.md#collators) (e.g. diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md new file mode 100644 index 0000000000..a0302ce279 --- /dev/null +++ b/docs/features/search/search-engines.md @@ -0,0 +1,158 @@ +--- +id: search-engines +title: Search Engines +description: Choosing and configuring your search engine for Backstage +--- + +Backstage supports 2 search engines by default, an in-memory engine called Lunr +and ElasticSearch. You can configure your own search engines by implementing the +provided interface as mentioned in the +[search backend documentation.](./getting-started.md#Backend) + +Provided search engine implementations have their own way of constructing +queries, which may be something you want to modify. Alterations to the querying +logic of a search engine can be made by providing your own implementation of a +QueryTranslator interface. This modification can be done without touching +provided search engines by using the exposed setter to set the modified query +translator into the instance. + +```typescript +const searchEngine = new LunrSearchEngine({ logger }); +searchEngine.setTranslator(new MyNewAndBetterQueryTranslator()); +``` + +## Lunr + +Lunr search engine is enabled by default for your backstage instance if you have +not done additional changes to the scaffolded app. + +Lunr can be instantiated like this: + +```typescript +// app/backend/src/plugins/search.ts +const searchEngine = new LunrSearchEngine({ logger }); +const indexBuilder = new IndexBuilder({ logger, searchEngine }); +``` + +## Postgres + +The Postgres based search engine only requires that postgres being configured as +the database engine for Backstage. Therefore it targets setups that want to +avoid maintaining another external service like elastic search. The search +provides decent results and performs well with ten thousands of indexed +documents. The connection to postgres is established via the database manager +also used by other plugins. + +> **Important**: The search plugin requires at least Postgres 12! + +To use the `PgSearchEngine`, make sure that you have a Postgres database +configured and make the following changes to your backend: + +1. Add a dependency on `@backstage/plugin-search-backend-module-pg` to your + backend's `package.json`. +2. Initialize the search engine. It is recommended to initialize it with a + fallback to the lunr search engine if you are running Backstage for + development locally with SQLite: + +```typescript +// In packages/backend/src/plugins/search.ts + +// Initialize a connection to a search engine. +const searchEngine = (await PgSearchEngine.supported(database)) + ? await PgSearchEngine.from({ database }) + : new LunrSearchEngine({ logger }); +``` + +## ElasticSearch + +Backstage supports ElasticSearch search engine connections, indexing and +querying out of the box. Available configuration options enable usage of either +AWS or Elastic.co hosted solutions, or a custom self-hosted solution. + +Similarly to Lunr above, ElasticSearch can be set up like this: + +```typescript +// app/backend/src/plugins/search.ts +const searchEngine = await ElasticSearchSearchEngine.initialize({ + logger, + config, +}); +const indexBuilder = new IndexBuilder({ logger, searchEngine }); +``` + +For the engine to be available, your backend package needs a dependency into +package `@backstage/plugin-search-backend-module-elasticsearch`. + +ElasticSearch needs some additional configuration before it is ready to use +within your instance. The configuration options are documented in the +[configuration schema definition file.](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-elasticsearch/config.d.ts) + +The underlying functionality is using official ElasticSearch client version 7.x, +meaning that ElasticSearch version 7 is the only one confirmed to be supported. + +## Example configurations + +### AWS + +Using AWS hosted ElasticSearch the only configuration option needed is the URL +to the ElasticSearch service. The implementation assumes that environment +variables for AWS access key id and secret access key are defined in accordance +to the +[default AWS credential chain.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html). + +```yaml +search: + elasticsearch: + provider: aws + node: https://my-backstage-search-asdfqwerty.eu-west-1.es.amazonaws.com +``` + +### Elastic.co + +Elastic Cloud hosted ElasticSearch uses a Cloud ID to determine the instance of +hosted ElasticSearch to connect to. Additionally, username and password needs to +be provided either directly or using environment variables like defined in +[Backstage documentation.](https://backstage.io/docs/conf/writing#includes-and-dynamic-data) + +```yaml +search: + elasticsearch: + provider: elastic + cloudId: backstage-elastic:asdfqwertyasdfqwertyasdfqwertyasdfqwerty== + auth: + username: elastic + password: changeme +``` + +### Others + +Other ElasticSearch instances can be connected to by using standard +ElasticSearch authentication methods and exposed URL, provided that the cluster +supports that. The configuration options needed are the URL to the node and +authentication information. Authentication can be handled by either providing +username/password or an API key. For more information how to create an API key, +see +[Elastic documentation on API keys](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html). + +#### Configuration examples + +##### With username and password + +```yaml +search: + elasticsearch: + node: http://localhost:9200 + auth: + username: elastic + password: changeme +``` + +##### With API key + +```yaml +search: + elasticsearch: + node: http://localhost:9200 + auth: + apiKey: base64EncodedKey +``` diff --git a/docs/features/software-catalog/catalog-customization.md b/docs/features/software-catalog/catalog-customization.md index 0c11d8f476..8f855776a0 100644 --- a/docs/features/software-catalog/catalog-customization.md +++ b/docs/features/software-catalog/catalog-customization.md @@ -34,15 +34,14 @@ export const CustomCatalogPage = ({ }: CatalogPageProps) => { return ( - - - - All your software catalog entities - - + + + }> + + All your software catalog entities + - - - + + ); }; @@ -187,7 +186,7 @@ new `CustomCatalogIndexPage`. # packages/app/src/App.tsx const routes = ( - + - } /> + } /> ``` diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index fb17f4a434..7a9aa099e9 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -245,6 +245,22 @@ the entity belongs to the `"default"` namespace. Namespaces may also be part of the catalog, and are `v1` / `Namespace` entities, i.e. not Backstage specific but the same as in Kubernetes. +### `title` [optional] + +A display name of the entity, to be presented in user interfaces instead of the +`name` property above, when available. + +This field is sometimes useful when the `name` is cumbersome or ends up being +perceived as overly technical. The title generally does not have as stringent +format requirements on it, so it may contain special characters and be more +explanatory. Do keep it very short though, and avoid situations where a title +can be confused with the name of another entity, or where two entities share a +title. + +Note that this is only for display purposes, and may be ignored by some parts of +the code. [Entity references](references.md) still always make use of the `name` +property for example, not the title. + ### `description` [optional] A human readable description of the entity, to be shown in Backstage. Should be @@ -531,8 +547,8 @@ The current set of well-known and common values for this field is: ### `spec.owner` [required] -An [entity reference](#string-references) to the owner of the component, e.g. -`artist-relations-team`. This field is required. +An [entity reference](references.md#string-references) to the owner of the +component, e.g. `artist-relations-team`. This field is required. In Backstage, the owner of a component is the singular entity (commonly a team) that bears ultimate responsibility for the component, and has the authority and @@ -550,8 +566,8 @@ component, but there will always be one ultimate owner. ### `spec.system` [optional] -An [entity reference](#string-references) to the system that the component -belongs to, e.g. `artist-engagement-portal`. This field is optional. +An [entity reference](references.md#string-references) to the system that the +component belongs to, e.g. `artist-engagement-portal`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- | @@ -559,8 +575,8 @@ belongs to, e.g. `artist-engagement-portal`. This field is optional. ### `spec.subcomponentOf` [optional] -An [entity reference](#string-references) to another component of which the -component is a part, e.g. `spotify-ios-app`. This field is optional. +An [entity reference](references.md#string-references) to another component of +which the component is a part, e.g. `spotify-ios-app`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | ---------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- | @@ -568,8 +584,8 @@ component is a part, e.g. `spotify-ios-app`. This field is optional. ### `spec.providesApis` [optional] -An array of [entity references](#string-references) to the APIs that are -provided by the component, e.g. `artist-api`. This field is optional. +An array of [entity references](references.md#string-references) to the APIs +that are provided by the component, e.g. `artist-api`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------- | @@ -577,8 +593,8 @@ provided by the component, e.g. `artist-api`. This field is optional. ### `spec.consumesApis` [optional] -An array of [entity references](#string-references) to the APIs that are -consumed by the component, e.g. `artist-api`. This field is optional. +An array of [entity references](references.md#string-references) to the APIs +that are consumed by the component, e.g. `artist-api`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------- | @@ -586,9 +602,9 @@ consumed by the component, e.g. `artist-api`. This field is optional. ### `spec.dependsOn` [optional] -An array of [entity references](#string-references) to the components and -resources that the component depends on, e.g. `artists-db`. This field is -optional. +An array of [entity references](references.md#string-references) to the +components and resources that the component depends on, e.g. `artists-db`. This +field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------- | @@ -690,12 +706,6 @@ shape, this kind has the following structure. Exactly equal to `backstage.io/v1beta2` and `Template`, respectively. -### `metadata.title` [required] - -The nice display name for the template as a string, e.g. `React SSR Template`. -This field is required as is used to reference the template to the user instead -of the `metadata.name` field. - ### `metadata.tags` [optional] A list of strings that can be associated with the template, e.g. @@ -722,8 +732,8 @@ You can find out more about the `steps` key ### `spec.owner` [optional] -An [entity reference](#string-references) to the owner of the template, e.g. -`artist-relations-team`. This field is required. +An [entity reference](references.md#string-references) to the owner of the +template, e.g. `artist-relations-team`. This field is required. In Backstage, the owner of a Template is the singular entity (commonly a team) that bears ultimate responsibility for the Template, and has the authority and @@ -832,8 +842,8 @@ The current set of well-known and common values for this field is: ### `spec.owner` [required] -An [entity reference](#string-references) to the owner of the component, e.g. -`artist-relations-team`. This field is required. +An [entity reference](references.md#string-references) to the owner of the +component, e.g. `artist-relations-team`. This field is required. In Backstage, the owner of an API is the singular entity (commonly a team) that bears ultimate responsibility for the API, and has the authority and capability @@ -851,8 +861,8 @@ one ultimate owner. ### `spec.system` [optional] -An [entity reference](#string-references) to the system that the API belongs to, -e.g. `artist-engagement-portal`. This field is optional. +An [entity reference](references.md#string-references) to the system that the +API belongs to, e.g. `artist-engagement-portal`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- | @@ -1059,8 +1069,8 @@ Exactly equal to `backstage.io/v1alpha1` and `Resource`, respectively. ### `spec.owner` [required] -An [entity reference](#string-references) to the owner of the resource, e.g. -`artist-relations-team`. This field is required. +An [entity reference](references.md#string-references) to the owner of the +resource, e.g. `artist-relations-team`. This field is required. In Backstage, the owner of a resource is the singular entity (commonly a team) that bears ultimate responsibility for the resource, and has the authority and @@ -1091,8 +1101,8 @@ Some common values for this field could be: ### `spec.system` [optional] -An [entity reference](#string-references) to the system that the resource -belongs to, e.g. `artist-engagement-portal`. This field is optional. +An [entity reference](references.md#string-references) to the system that the +resource belongs to, e.g. `artist-engagement-portal`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- | @@ -1100,9 +1110,9 @@ belongs to, e.g. `artist-engagement-portal`. This field is optional. ### `spec.dependsOn` [optional] -An array of [entity references](#string-references) to the components and -resources that the resource depends on, e.g. `artist-lookup`. This field is -optional. +An array of [entity references](references.md#string-references) to the +components and resources that the resource depends on, e.g. `artist-lookup`. +This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------- | @@ -1146,8 +1156,8 @@ Exactly equal to `backstage.io/v1alpha1` and `System`, respectively. ### `spec.owner` [required] -An [entity reference](#string-references) to the owner of the system, e.g. -`artist-relations-team`. This field is required. +An [entity reference](references.md#string-references) to the owner of the +system, e.g. `artist-relations-team`. This field is required. In Backstage, the owner of a system is the singular entity (commonly a team) that bears ultimate responsibility for the system, and has the authority and @@ -1165,8 +1175,8 @@ but there will always be one ultimate owner. ### `spec.domain` [optional] -An [entity reference](#string-references) to the domain that the system belongs -to, e.g. `artists`. This field is optional. +An [entity reference](references.md#string-references) to the domain that the +system belongs to, e.g. `artists`. This field is optional. | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type | | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- | @@ -1205,8 +1215,8 @@ Exactly equal to `backstage.io/v1alpha1` and `Domain`, respectively. ### `spec.owner` [required] -An [entity reference](#string-references) to the owner of the domain, e.g. -`artist-relations-team`. This field is required. +An [entity reference](references.md#string-references) to the owner of the +domain, e.g. `artist-relations-team`. This field is required. In Backstage, the owner of a domain is the singular entity (commonly a team) that bears ultimate responsibility for the domain, and has the authority and diff --git a/docs/features/software-catalog/life-of-an-entity.md b/docs/features/software-catalog/life-of-an-entity.md new file mode 100644 index 0000000000..817529c0f2 --- /dev/null +++ b/docs/features/software-catalog/life-of-an-entity.md @@ -0,0 +1,130 @@ +--- +id: life-of-an-entity +title: The Life of an Entity +sidebar_label: The Life of an Entity +# prettier-ignore +description: The life cycle of entities, from being introduced into the catalog, through processing, to being removed again +--- + +This document gives a high level overview of the catalog backend, and the +technical processes involved in making entities flow through it. It is mainly +aimed at developers who want to understand the internals while installing or +extending the catalog. However, it can be informative for other personas too. + +## Key Concepts + +The catalog forms a hub of sorts, where entities are ingested from various +authoritative sources and held in a database, subject to automated processing, +and then presented through an API for quick and easy access by Backstage and +others. The most common source is [YAML files](descriptor-format.md) on a +standard format, living in version control systems near the source code of +systems that they describe. Those files are registered with the catalog and +maintained by the respective owners. The catalog makes sure to keep itself up to +date with changes to those files. + +The main extension points where developers can customize the catalog are: + +- _Entity providers_, that feed initial raw entity data into the catalog, +- _Policies_, that establish baseline rules about the shape of entities, +- _Processors_, that validate, analyze, and mutate the raw entity data into its + final form. + +The high level processes involved are: + +- _Ingestion_, where entity providers fetch raw entity data from external + sources and seed it into the database, +- _Processing_, where the policies and processors continually treat the ingested + data and may emit both other raw entities (that are also subject to + processing), errors, relations to other entities, etc., +- _Stitching_, where all of the data emitted by various processors are assembled + together into the final output entity. + +An entity is not visible to the outside world (through the catalog API), until +it has passed through the last process and landed among the final entities. + +![General overview](../../assets/features/catalog/life-of-an-entity_overview.svg) + +The details of these processes are described below. + +## Ingestion + +Each catalog deployment has a number of entity providers installed. They are +responsible for fetching data from external authoritative sources in any way +that they see fit, to translate those into entity objects, and to notify the +database when those entities are added or removed. These are the _unprocessed +entities_ that will be subject to later processing (see below), and they form +the very basis of existence for entities. If there were no entity providers, no +entities would ever enter the system. + +The database always keeps track of the set of entities that belong to each +provider; no two providers can try to output the same entity. And when a +provider signals the removal of an entity, then that leads to an _eager +deletion_: the entity and all auxiliary data that it has led to in the database +is immediately purged. + +![Ingestion overview](../../assets/features/catalog/life-of-an-entity_ingestion.svg) + +There are two providers installed by default: the one that deals with user +registered locations (e.g. URLs to YAML files), and the one that deals with +static locations in the app-config. You can add more third party providers by +passing them to the catalog builder in your backend initialization code, and you +can easily write your own. + +An entity provider is a class that implements the `EntityProvider` interface. It +has three main parts: + +- The identity: Each provider instance has a unique, stable identifier that the + database can use to keep track of the originator of each unprocessed entity. +- The connection: During backend startup, each provider is attached to the + catalog runtime. +- The stream of events: During its lifetime, the provider can issue change + events to the runtime at any point in time, to modify its set of unprocessed + entities. + +It is entirely up to the provider to choose how and when it produces these +change events. For example, the app-config provider only fires off an update at +startup and then lies dormant. The location database provider does an initial +update at startup, and then small delta updates every time a location database +change is detected. The LDAP provider is driven externally by a timer loop that +occasionally triggers a full update. Some future provider may be entirely event +driven, feeding off an event bus or web hook. There is no magic coordination +among providers; if they need to arrange synchronization or locking among +themselves for example to avoid duplicate work across multiple catalog service +machines, they need to handle that out-of-band. + +The entities that are emitted get some coarse validation applied to them, to +ensure that they at least adhere to the most basic schema rules about how an +entity should be shaped. For example, they need to have a `kind`, a +`metadata.name`, and optionally a `metadata.namespace`, among others. Apart from +that, the ingestion stage considers its work done, and stores the unprocessed +entities to be picked up at a later time by the processing system. This means +that the more precise validation rules that you put in place on entities are +_not_ yet applied at this stage. + +## Processing + +Every unprocessed entity comes with a timestamp, which tells at what time that +the processing loop should next try to process it. When the entity first +appears, this timestamp is set to "now" - asking for it to be picked up as soon +as possible. + +Each catalog deployment has a number of processors installed. They are +responsible for receiving unprocessed entities that the catalog decided are due +for processing, and then running that data through a number of processing +stages. mutating the entity and emitting auxiliary data about it. When all of +that is done, the catalog takes all of that information and stores it as the +processed entity, and errors and relations to other entities separately. Then, +the catalog checks to see what entities are touched by that output, and triggers +the final assembly of those (see Stitching below). + +There are several stages involved in the processing. + +> TODO: More info here + +## Stitching + +The stitching is currently a fixed process, that cannot be modified or extended. +This means that any modifications you want to make on the final result, has to +happen during ingestion or processing. + +> TODO: More info here diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 5f474624f9..c3af6caf2d 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -57,18 +57,48 @@ if the original location delegates to another location. A common case is, that a location is registered as `bootstrap:bootstrap` which means that it is part of the `app-config.yaml` of a Backstage installation. +### backstage.io/orphan + +This annotation is either absent, or present with the exact _string_ value +`"true"`. It should never be added manually. Instead, the catalog itself injects +the annotation as part of its processing loops, on entities that are found to +have no registered locations or config locations that keep them "active" / +"alive". + +For example, suppose that the user first registers a location URL pointing to a +`Location` kind entity, which in turn refers to two `Component` kind entities in +two other files nearby. The end result is that the catalog contains those three +entities. Now suppose that the user edits the original `Location` entity to only +refer to the first of the `Component` kind entities. This will intentionally +_not_ lead to the other `Component` entity to be removed from the catalog (for +safety reasons). Instead, it gains this orphan marker annotation, to make it +clear that user action is required to completely remove it, if desired. + +```yaml +# Example: +metadata: + annotations: + backstage.io/orphan: 'true' +``` + ### backstage.io/techdocs-ref ```yaml # Example: metadata: annotations: - backstage.io/techdocs-ref: url:https://github.com/backstage/backstage/tree/master + backstage.io/techdocs-ref: dir:. ``` -The value of this annotation is a location reference string (see above). If this -annotation is specified, it is expected to point to a repository that the -TechDocs system can read and generate docs from. +The value of this annotation informs _where_ TechDocs source content is stored +so that it can be read and docs can be generated from it. Most commonly, it's +written as a path, relative to the location of the `catalog-info.yaml` itself, +where the associated `mkdocs.yml` file can be found. + +In unusual situations where the documentation for a catalog entity does not live +alongside the entity's source code, the value of this annotation can point to an +absolute URL, matching the location reference string format outlined above, for +example: `url:https://github.com/backstage/backstage/tree/master` ### backstage.io/view-url, backstage.io/edit-url diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md index 25ff2a2f3a..28001cce0b 100644 --- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md +++ b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md @@ -325,7 +325,7 @@ spec: output: links: - url: '{{steps.publish.output.remoteUrl}}' - text: 'Go to Repo' + title: 'Go to Repo' ``` ## Questions? diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 53a90e2954..a5773a6aff 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -131,6 +131,9 @@ want to have those as well as your new one, you'll need to do the following: ```ts import { createBuiltinActions } from '@backstage/plugin-scaffolder-backend'; +import { ScmIntegrations } from '@backstage/integration'; + +const integrations = ScmIntegrations.fromConfig(config); const builtInActions = createBuiltinActions({ containerRunner, diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 6732b3ab18..b798742463 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -29,7 +29,7 @@ about TechDocs and the philosophy in its - Explore and take advantage of the large ecosystem of [MkDocs plugins](https://www.mkdocs.org/user-guide/plugins/) to create a rich reading experience. -- Search for and find docs (coming soon). +- Search for and find docs. - Highlight text and raise an Issue to create feedback loop to drive quality documentation (future). - Contribute to and deploy from a marketplace of TechDocs widgets (future). @@ -54,23 +54,12 @@ providers are used. | Google Cloud Storage (GCS) | Yes ✅ | | Amazon Web Services (AWS) S3 | Yes ✅ | | Azure Blob Storage | Yes ✅ | -| OpenStack Swift | Yes ✅ | +| OpenStack Swift | Community ✅ | [Reach out to us](#feedback) if you want to request more platforms. ## Project roadmap -### **Ongoing work 🚧** - -**Beta release** - -[Milestone](https://github.com/backstage/backstage/milestone/29) - -- It should be possible and easy to use TechDocs in most environments across - organizations. -- Minimal bugs, better error handling and scalable backend and frontend. -- Documentation Search -- TechDocs Homepage with basic features - ### **Future work 🔮** **General Availability (GA) release** - diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index ef6a80d44c..46cbc2a8a2 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -24,13 +24,13 @@ techdocs: runIn: 'docker' - # techdocs.generator.dockerImage can be used to control the docker image used during documentation generation. This can be useful + # (Optional) techdocs.generator.dockerImage can be used to control the docker image used during documentation generation. This can be useful # if you want to use MkDocs plugins or other packages that are not included in the default techdocs-container (spotify/techdocs). # NOTE: This setting is only used when techdocs.generator.runIn is set to 'docker'. dockerImage: 'spotify/techdocs' - # techdocs.generator.pullImage can be used to disable pulling the latest docker image by default. This can be useful when you are + # (Optional) techdocs.generator.pullImage can be used to disable pulling the latest docker image by default. This can be useful when you are # using a custom techdocs.generator.dockerImage and you have a custom docker login requirement. For example, you need to login to # AWS ECR to pull the docker image. # NOTE: Disabling this requires the docker image was pulled by other means before running the techdocs generator. @@ -113,6 +113,14 @@ techdocs: # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} + # (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs + # sites could only be accessed over paths with case-sensitive entity triplets + # e.g. (namespace/Kind/name). If you are upgrading from an older version of + # TechDocs and are unable to perform the necessary migration of files in your + # external storage, you can set this value to `true` to temporarily revert to + # the old, case-sensitive entity triplet behavior. + legacyUseCaseSensitiveTripletPaths: false + # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 3fdc942ac4..93ce3ad4d9 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -66,9 +66,7 @@ Update your component's entity description by adding the following lines to its ```yaml metadata: annotations: - backstage.io/techdocs-ref: url:https://github.com/org/repo - # Or - # backstage.io/techdocs-ref: url:https://github.com/org/repo/tree/branchName/subFolder + backstage.io/techdocs-ref: dir:. ``` The diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index cf0bbb0de1..0b1809dc22 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -33,14 +33,24 @@ In `packages/app/src/App.tsx`, import `TechDocsPage` and add the following to `FlatRoutes`: ```tsx -import { TechDocsPage } from '@backstage/plugin-techdocs'; +import { + DefaultTechDocsHome, + TechDocsIndexPage, + TechDocsReaderPage, +} from '@backstage/plugin-techdocs'; // ... const AppRoutes = () => { // ... other plugin routes - } /> + }> + + + } + /> ; }; ``` @@ -204,7 +214,7 @@ techdocs: builder: 'local' publisher: type: 'local' - generators: + generator: techdocs: local ``` diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index 3f3c92d0de..326ac02969 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -35,38 +35,64 @@ In your Backstage instance's `app-config.yaml`, set `techdocs.builder` from `'local'` to `'external'`. By doing this, TechDocs will not try to generate docs. Look at [TechDocs configuration](configuration.md) for reference. -## How to use URL Reader in TechDocs Prepare step? +## How to understand techdocs-ref annotation values -If TechDocs is configured to generate docs, it will first download the -repository associated with the `backstage.io/techdocs-ref` annotation defined in -the Entity's `catalog-info.yaml` file. This is also called the +If TechDocs is configured to generate docs, it will first download source files +based on the value of the `backstage.io/techdocs-ref` annotation defined in the +Entity's `catalog-info.yaml` file. This is also called the [Prepare](./concepts.md#techdocs-preparer) step. -There are two kinds of preparers or two ways of downloading these source files +We strongly recommend that the `backstage.io/techdocs-ref` annotation in each +documented catalog entity's `catalog-info.yaml` be set to `dir:.` in almost all +situations. This is because TechDocs is aligned with the "docs like code" +philosophy, whereby documentation should be authored and managed alongside the +source code of the underlying software itself. -- Preparer 1: Doing a `git clone` of the repository (also known as Common Git - Preparer) -- Preparer 2: Downloading an archive.zip or equivalent of the repository (also - known as URL Reader) +When you see `dir:.`, you can translate it to mean: -If `backstage.io/techdocs-ref` is equal to any of these - +- That the documentation source code lives in the same location as the + `catalog-info.yaml` file. +- That, in particular, the `mkdocs.yml` file is a sibling of `catalog-info.yaml` + (meaning, it is in the same directory) +- And that all of the source content of the documentation would be available if + one were to download the directory containing those two files (as well as all + sub-directories). -1. `github:https://githubhost.com/org/repo` -2. `gitlab:https://gitlabhost.com/org/repo` -3. `bitbucket:https://bitbuckethost.com/project/repo` -4. `azure/api:https://azurehost.com/org/project` +The directory tree of the entity would look something like this: -Then Common Git Preparer will be used i.e. a `git clone`. But the URL Reader is -a much faster way to do this step. Convert the `backstage.io/techdocs-ref` -values to the following - +``` +├── catalog-info.yaml +├── mkdocs.yml +└── docs + └── index.md +``` -1. `url:https://githubhost.com/org/repo/tree/` -2. `url:https://gitlabhost.com/org/repo/tree/` -3. `url:https://bitbuckethost.com/project/repo/src/` -4. `url:https://azurehost.com/organization/project/_git/repository` +If, for example, you wanted to keep a lean root directory, you could place your +`mkdocs.yml` file in a subdirectory and update the `backstage.io/techdocs-ref` +annotation value accordingly, e.g. to `dir:./sub-folder`: -Note that you can also provide a path to a non-root directory inside the -repository which contains the `docs/` directory. +``` +├── catalog-info.yaml +└── sub-folder + ├── mkdocs.yml + └── docs + └── index.md +``` + +In rare situations where your TechDocs source content is managed and stored in a +location completely separate from your `catalog-info.yaml`, you can instead +specify a URL location reference, the exact value of which will vary based on +the source code hosting provider. Notice that instead of the `dir:` prefix, the +`url:` prefix is used instead. For example: + +- **GitHub**: `url:https://githubhost.com/org/repo/tree/` +- **GitLab**: `url:https://gitlabhost.com/org/repo/tree/` +- **Bitbucket**: `url:https://bitbuckethost.com/project/repo/src/` +- **Azure**: `url:https://azurehost.com/organization/project/_git/repository` + +Note, just as it's possible to specify a subdirectory with the `dir:` prefix, +you can also provide a path to a non-root directory inside the repository which +contains the `mkdocs.yml` file and `docs/` directory. e.g. `url:https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend/examples/documented-component` @@ -78,83 +104,116 @@ the repository. The archive does not have any git history attached to it. Also it is a compressed file. Hence the file size is significantly smaller than how much data git clone has to transfer. -## How to use a custom TechDocs home page? +## How to customize the TechDocs home page? -### 1st way: TechDocsCustomHome with a custom configuration +TechDocs uses a composability pattern similar to the Search and Catalog plugins +in Backstage. While a default table experience, similar to the one provided by +the Catalog plugin, is made available for ease-of-use, it's possible for you to +provide a completely custom experience, tailored to the needs of your +organization. -As an example, in your main App.tsx: +This is done in your `app` package. By default, you might see something like +this in your `App.tsx`: ```tsx -import { - TechDocsCustomHome, - PanelType, - TechDocsReaderPage, -} from '@backstage/plugin-techdocs'; -import { Entity } from '@backstage/catalog-model'; - -const tabsConfig = [ - { - label: 'Custom Tab', - panels: [ - { - title: 'Custom Documents Cards 1', - description: - 'Explore your internal technical ecosystem through documentation.', - panelType: 'DocsCardGrid' as PanelType, - // optional, is applied to a container of the panel (excludes header of panel) - panelCSS: { maxHeight: '400px', overflow:'auto' }, - filterPredicate: (entity: Entity) => !!entity.metadata.annotations?.['customCardAnnotationOne']; - }, - { - title: 'Custom Documents Cards 2', - description: - 'Explore your internal technical ecosystem through documentation.', - panelType: 'DocsCardGrid' as PanelType, - panelCSS: { maxHeight: '400px', overflow:'auto' }, - filterPredicate: (entity: Entity) => !!entity.metadata.annotations?.['customCardAnnotationTwo']; - }, - ], - }, - { - label: 'Overview', - panels: [ - { - title: 'Overview', - description: - 'Explore your internal technical ecosystem through documentation.', - panelType: 'DocsTable' as PanelType, - filterPredicate: () => true, - }, - ], - }, -]; - -const routes = ( +const AppRoutes = () => { - } - /> - } - /> - + }> + + + ; +}; ``` -An example of tabsConfig that corresponds to the default documentation home page -can be found at `plugins/techdocs/src/home/components/TechDocsHome.tsx`. +But you can replace `` with any React component, which +will be rendered in its place. Most likely, you would want to create and +maintain such a component in a new directory at +`packages/app/src/components/techdocs`, and import and use it in `App.tsx`: -Currently `panelType` has DocsCardGrid and DocsTable available. We currently -recommend that DocsCardGrid can be optionally vertically stacked by setting a -maxHeight using `panelCSS`, and DocsTable to be in a tab by itself. +```tsx +import { CustomTechDocsHome } from './components/techdocs/CustomTechDocsHome'; +// ... +const AppRoutes = () => { + + }> + + + ; +}; +``` -### 2nd way: Custom home page plugin +## How to migrate from TechDocs Alpha to Beta -A custom home page plugin can be built that uses the components extensions -DocsCardGrid and DocsTable, exported from @backstage/techdocs. They both take a -array of documentation entities ( i.e.have a 'backstage.io/techdocs-ref' -annotation ) as an 'entities' attribute. +> This guide only applies to the "recommended" TechDocs deployment method (where +> an external storage provider and external CI/CD is used). If you use the +> "basic" or "out-of-the-box" setup, you can stop here! No action needed. -For a reference to the React structure of the default home page, please refer to -`plugins/techdocs/src/home/components/TechDocsCustomHome.tsx`. +For the purposes of this guide, TechDocs Beta version is defined as: + +- **TechDocs Plugin**: At least `v0.11.0` +- **TechDocs Backend Plugin**: At least `v0.10.0` +- **TechDocs CLI**: At least `v0.7.0` + +The beta version of TechDocs made a breaking change to the way TechDocs content +was accessed and stored, allowing pages to be accessed with case-insensitive +entity triplet paths (e.g. `/docs/namespace/kind/name` whereas in prior +versions, they could only be accessed at `/docs/namespace/Kind/name`). In order +to enable this change, documentation has to be stored in an external storage +provider using an object key whose entity triplet is lower-cased. + +New installations of TechDocs since the beta version will work fine with no +action, but for those who were running TechDocs prior to this version, a +migration will need to be performed so that all existing content in your storage +bucket matches this lower-case entity triplet expectation. + +1. **Ensure you have the right permissions on your storage provider**: In order + to migrate files in your storage provider, the `techdocs-cli` needs to be + able to read/copy/rename/move/delete files. The exact instructions vary by + storage provider, but check the [using cloud storage][using-cloud-storage] + page for details. + +2. **Run a non-destructive migration of files**: Ensure you have the latest + version of `techdocs-cli` installed. Then run the following command, using + the details relevant for your provider / configuration. This will copy all + files from, e.g. `namespace/Kind/name/index.html` to + `namespace/kind/name/index.html`, without removing the original files. + +```sh +techdocs-cli migrate --publisher-type --storage-name --verbose +``` + +3. **Deploy the updated versions of the TechDocs plugins**: Once the migration + above has been run, you can deploy the beta versions of the TechDocs backend + and frontend plugins to your Backstage instance. + +4. **Verify that your TechDocs sites are still loading/accessible**: Try + accessing a TechDocs site using different entity-triplet case variants, e.g. + `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`. Your TechDocs + site should load regardless of the URL path casing you use. + +5. **Clean up the old objects from storage**: Once you've verified that your + TechDocs site is accessible, you can clean up your storage bucket by + re-running the `migrate` command on the TechDocs CLI, but with an additional + `removeOriginal` flag passed: + +```sh +techdocs-cli migrate --publisher-type --storage-name --removeOriginal --verbose +``` + +6. **Update your CI/CD pipelines to use the beta version of the TechDocs CLI**: + Finally, you can update all of your CI/CD pipelines to use at least v0.x.y of + the TechDocs CLI, ensuring that all sites are published to the new, + lower-cased entity triplet paths going forward. + +If you encounter problems running this migration, please [report the +issue][beta-migrate-bug]. You can temporarily revert to pre-beta storage +expectations with a configuration change: + +```yaml +techdocs: + legacyUseCaseSensitiveTripletPaths: true +``` + +[beta-migrate-bug]: +https://github.com/backstage/backstage/issues/new?assignees=&labels=bug&template=bug_template.md&title=[TechDocs]%20Unable%20to%20run%20beta%20migration +[using-cloud-storage]: ./using-cloud-storage.md diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 159137c879..66eaf00663 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -61,7 +61,7 @@ If you do not prefer (3a) and optionally like to use a service account, you can follow these steps. Create a new Service Account and a key associated with it. In roles of the -service account, use "Storage Admin". +service account, use "Storage Object Admin". If you want to create a custom role, make sure to include both `get` and `create` permissions for both "Objects" and "Buckets". See @@ -143,6 +143,8 @@ permissions to: - `s3:ListBucket` to retrieve bucket metadata - `s3:PutObject` to upload files to the bucket +- `s3:DeleteObject` and `s3:DeleteObjectVersion` to delete stale content during + re-publishing To _read_ TechDocs from the S3 bucket the IAM policy needs to have at a minimum permissions to: @@ -345,6 +347,10 @@ techdocs: accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} ``` +In either case, the account or credentials used to access your container and all +TechDocs objects underneath it should have the `Storage Blog Data Owner` role +applied, in order to read, write, and delete objects as needed. + **4. That's it!** Your Backstage app is now ready to use Azure Blob Storage for TechDocs, to store @@ -393,9 +399,34 @@ techdocs: Set the configs in your `app-config.yaml` to point to your container name. -https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail#password-authentication-with-unscoped-authorization +https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential for more details. +```yaml +techdocs: + publisher: + type: 'openStackSwift' + openStackSwift: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID} + secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET} + authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL} + swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL} +``` + +**4. That's it!** + +Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to +store and read the static generated documentation files. When you start the +backend of the app, you should be able to see +`techdocs info Successfully connected to the OpenStack Swift Storage container` +in the logs. + +## Bonus: Migration from old OpenStack Swift Configuration + +Let's assume we have the old OpenStack Swift configuration here. + ```yaml techdocs: publisher: @@ -412,10 +443,40 @@ techdocs: region: ${OPENSTACK_SWIFT_STORAGE_REGION} ``` -**4. That's it!** +##### Step 1: Change the credential keys -Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to -store and read the static generated documentation files. When you start the -backend of the app, you should be able to see -`techdocs info Successfully connected to the OpenStack Swift Storage container` -in the logs. +Since the new SDK uses _Application Credentials_ to authenticate OpenStack, we +need to change the keys `credentials.username` to `credentials.id`, +`credentials.password` to `credentials.secret` and use Application Credential ID +and secret here. For more detail about credentials look +[here](https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential). + +##### Step 2: Remove the unused keys + +Since the new SDK doesn't use the old way authentication, we don't need the keys +`openStackSwift.keystoneAuthVersion`, `openStackSwift.domainId`, +`openStackSwift.domainName` and `openStackSwift.region`. So you can remove them. + +##### Step 3: Add Swift URL + +The new SDK needs the OpenStack Swift connection URL for connecting the Swift. +So you need to add a new key called `openStackSwift.swiftUrl` and give the +OpenStack Swift url here. Example url should look like that: +`https://example.com:6780/swift/v1` + +##### That's it! + +Your new configuration should look like that! + +```yaml +techdocs: + publisher: + type: 'openStackSwift' + openStackSwift: + containerName: 'name-of-techdocs-storage-bucket' + credentials: + id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID} + secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET} + authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL} + swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL} +``` diff --git a/docs/getting-started/app-custom-theme.md b/docs/getting-started/app-custom-theme.md index aa3851463f..862c3c0cb6 100644 --- a/docs/getting-started/app-custom-theme.md +++ b/docs/getting-started/app-custom-theme.md @@ -35,7 +35,7 @@ If you want more control over the theme, and for example customize font sizes and margins, you can use the lower-level `createThemeOverrides` function exported by [@backstage/theme](https://www.npmjs.com/package/@backstage/theme) in combination with -[createMuiTheme](https://material-ui.com/customization/theming/#createmuitheme-options-args-theme) +[createTheme](https://material-ui.com/customization/theming/#createmuitheme-options-args-theme) from [@material-ui/core](https://www.npmjs.com/package/@material-ui/core). See the [@backstage/theme source](https://github.com/backstage/backstage/tree/master/packages/theme/src) @@ -128,7 +128,7 @@ const themeOptions = createThemeOptions({ tool: genPageTheme(['#123456','#123456'], shapes.round), service: genPageTheme(['#123456','#123456'], shapes.wave), website: genPageTheme(['#123456','#123456'], shapes.wave), - library: genPageTheme(['#123456','#123456'] shapes.wave), + library: genPageTheme(['#123456','#123456'], shapes.wave), other: genPageTheme(['#123456','#123456'], shapes.wave), app: genPageTheme(['#123456','#123456'], shapes.wave), apis: genPageTheme(['#123456','#123456'], shapes.wave), diff --git a/docs/integrations/aws-s3/locations.md b/docs/integrations/aws-s3/locations.md new file mode 100644 index 0000000000..fe1c6050c2 --- /dev/null +++ b/docs/integrations/aws-s3/locations.md @@ -0,0 +1,42 @@ +--- +id: locations +sidebar_label: Locations +title: Amazon Web Services S3 Locations +# prettier-ignore +description: Setting up an integration with Amazon Web Services S3 +--- + +The AWS S3 integration supports loading catalog entities from an S3 Bucket. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +To use this integration, add configuration to your `app-config.yaml`: + +```yaml +integrations: + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} +``` + +Then make sure the environment variables `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` are set when you run Backstage. + +Users with multiple AWS accounts may want to use a role for S3 storage that is +in a different AWS account. Using the `roleArn` parameter as seen below, you can +instruct the AWS S3 reader to assume a role before accessing S3: + +```yaml +integrations: + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} + roleArn: 'arn:aws:iam::xxxxxxxxxxxx:role/example-role' +``` diff --git a/docs/integrations/bitbucket/discovery.md b/docs/integrations/bitbucket/discovery.md index 2b04490cbe..b08386f3c7 100644 --- a/docs/integrations/bitbucket/discovery.md +++ b/docs/integrations/bitbucket/discovery.md @@ -108,13 +108,11 @@ matching repository is processed. repository. ```typescript -const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({ - client, - repository, -}) { - // Custom logic for interpret the matching repository. - // See defaultRepositoryParser for an example -}; +const customRepositoryParser: BitbucketRepositoryParser = + async function* customRepositoryParser({ client, repository }) { + // Custom logic for interpret the matching repository. + // See defaultRepositoryParser for an example + }; const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, { parser: customRepositoryParser, diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md new file mode 100644 index 0000000000..ade575b74d --- /dev/null +++ b/docs/integrations/gitlab/discovery.md @@ -0,0 +1,36 @@ +--- +id: discovery +title: GitLab Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in GitLab +--- + +The GitLab integration has a special discovery processor for discovering catalog +entities from GitLab. The processor will crawl the GitLab instance and register +entities matching the configured path. This can be useful as an alternative to +static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a location target to the +catalog configuration: + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + +Note the `gitlab-discovery` type, as this is not a regular `url` processor. + +The target is composed of three parts: + +- The base URL, `https://gitlab.com` in this case +- The group path, `group/subgroup` in this case. This is optional: If you omit + this path the processor will scan the entire GitLab instance instead. +- The path within each repository to find the catalog YAML file. This will + usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or + a similar variation for catalog files stored in the root directory of each + repository. If you want to use the repository's default branch use the `*` + wildcard, e.g.: `/blob/*/catalog-info.yaml` diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index 2b730024fd..ee2628a893 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -312,14 +312,6 @@ configuration. Stability: `1` -### `register-component` [GitHub](https://github.com/backstage/backstage/tree/master/plugins/register-component/) - -A frontend plugin that allows the user to register entity locations in the -catalog. - -Stability: `0`. This plugin is likely to be replaced by a generic entity import -plugin instead. - ### `scaffolder` [GitHub](https://github.com/backstage/backstage/tree/master/plugins/scaffolder/) The frontend scaffolder plugin where one can browse templates and initiate diff --git a/docs/plugins/github-apps.md b/docs/plugins/github-apps.md index 805e321093..f9f7590e09 100644 --- a/docs/plugins/github-apps.md +++ b/docs/plugins/github-apps.md @@ -89,6 +89,26 @@ integrations: - $include: example-backstage-app-credentials.yaml ``` +### Limiting the GitHub App installations + +If you want to limit the GitHub app installations visible to backstage you may +optionally include the `allowedInstallationOwners` option. + +```yaml +appId: 1 +allowedInstallationOwners: ['GlobexCorp'] +clientId: client id +clientSecret: client secret +webhookSecret: webhook secret +privateKey: | + -----BEGIN RSA PRIVATE KEY----- + ...Key content... + -----END RSA PRIVATE KEY----- +``` + +This will result in backstage preventing the use of any installation that is not +within the allow list. + ### Permissions for pull requests These are the minimum permissions required for creating a pull request with diff --git a/lerna.json b/lerna.json index dd2dd884eb..322929db1d 100644 --- a/lerna.json +++ b/lerna.json @@ -2,5 +2,5 @@ "packages": ["packages/*", "plugins/*"], "npmClient": "yarn", "useWorkspaces": true, - "version": "0.1.1" + "version": "0.1.0" } diff --git a/microsite/data/plugins/allure.yaml b/microsite/data/plugins/allure.yaml new file mode 100644 index 0000000000..b24a4157f9 --- /dev/null +++ b/microsite/data/plugins/allure.yaml @@ -0,0 +1,9 @@ +--- +title: Allure Reports +author: Deepak Bhardwaj +authorUrl: https://github.com/deepak-bhardwaj-ps +category: Reporting +description: View Allure reports for your components in Backstage. +documentation: https://github.com/backstage/backstage/blob/plugin-allure/plugins/allure/README.md +iconUrl: https://avatars.githubusercontent.com/u/5879127 +npmPackageName: '@backstage/plugin-allure' diff --git a/microsite/data/plugins/aws-cloudformation.yaml b/microsite/data/plugins/aws-cloudformation.yaml new file mode 100644 index 0000000000..7f0feee143 --- /dev/null +++ b/microsite/data/plugins/aws-cloudformation.yaml @@ -0,0 +1,9 @@ +--- +title: AWS CloudFormation +author: Purple Technology +authorUrl: https://blog.purple-technology.com +category: Infrastructure +description: Load Backstage entities from AWS CloudFormation stacks +documentation: https://github.com/purple-technology/backstage-aws-cloudformation-plugin#readme +iconUrl: https://raw.githubusercontent.com/purple-technology/backstage-aws-cloudformation-plugin/master/docs/cloudformation-logo.png +npmPackageName: 'backstage-aws-cloudformation-plugin' diff --git a/microsite/data/plugins/firehydrant.yaml b/microsite/data/plugins/firehydrant.yaml new file mode 100644 index 0000000000..e4dd5e7416 --- /dev/null +++ b/microsite/data/plugins/firehydrant.yaml @@ -0,0 +1,9 @@ +--- +title: FireHydrant +author: FireHydrant +authorUrl: https://firehydrant.io/ +category: Incident Management +description: View service incidents information from FireHydrant, such as active incidents and incident metrics, directly within Backstage. +documentation: https://github.com/backstage/backstage/blob/master/plugins/firehydrant/README.md +iconUrl: https://raw.githubusercontent.com/backstage/backstage/master/plugins/firehydrant/doc/firehydrant_logo.png +npmPackageName: '@backstage/plugin-firehydrant' diff --git a/microsite/data/plugins/gitlab.yaml b/microsite/data/plugins/gitlab.yaml new file mode 100644 index 0000000000..7a5493c7f4 --- /dev/null +++ b/microsite/data/plugins/gitlab.yaml @@ -0,0 +1,12 @@ +--- +title: GitLab +author: Loblaw +authorUrl: https://github.com/loblaw-sre/backstage-plugin-gitlab +category: CI/CD +description: View GitLab pipelines, merge requests, languages and contributors. +documentation: https://github.com/loblaw-sre/backstage-plugin-gitlab +iconUrl: https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png +npmPackageName: '@loblaw/backstage-plugin-gitlab' +tags: + - ci + - cd diff --git a/microsite/data/plugins/home.yaml b/microsite/data/plugins/home.yaml new file mode 100644 index 0000000000..e37b8c5e03 --- /dev/null +++ b/microsite/data/plugins/home.yaml @@ -0,0 +1,9 @@ +--- +title: Home +author: Spotify +authorUrl: https://github.com/spotify +category: Discovery +description: This plugin provides a composable home page, and ability to create home page components +documentation: https://github.com/backstage/backstage/blob/master/plugins/home/README.md +iconUrl: img/home.png +npmPackageName: '@backstage/plugin-home' diff --git a/microsite/data/plugins/xcmetrics.yaml b/microsite/data/plugins/xcmetrics.yaml new file mode 100644 index 0000000000..f2f4f7c013 --- /dev/null +++ b/microsite/data/plugins/xcmetrics.yaml @@ -0,0 +1,9 @@ +--- +title: XCMetrics +author: Spotify +authorUrl: https://github.com/spotify +category: Monitoring +description: Discover valuable insights hiding inside Xcode’s build logs. +documentation: https://xcmetrics.io/ +iconUrl: img/xcmetrics-icon.png +npmPackageName: '@backstage/plugin-xcmetrics' diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 5999524073..d200b615f1 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -36,6 +36,7 @@ "label": "Software Catalog", "ids": [ "features/software-catalog/software-catalog-overview", + "features/software-catalog/life-of-an-entity", "features/software-catalog/configuration", "features/software-catalog/system-model", "features/software-catalog/descriptor-format", @@ -80,7 +81,8 @@ "features/search/search-overview", "features/search/getting-started", "features/search/concepts", - "features/search/architecture" + "features/search/architecture", + "features/search/search-engines" ] }, { @@ -103,6 +105,11 @@ ], "Integrations": [ "integrations/index", + { + "type": "subcategory", + "label": "AWS S3", + "ids": ["integrations/aws-s3/locations"] + }, { "type": "subcategory", "label": "Azure", diff --git a/microsite/static/css/custom.css b/microsite/static/css/custom.css index 2c8b46ff00..393a5af118 100644 --- a/microsite/static/css/custom.css +++ b/microsite/static/css/custom.css @@ -288,6 +288,16 @@ td { .tocToggler { display: none; } + + .docsSliderActive .toc section .navGroups { + padding-top: 0; + padding-bottom: 0; + } + + .docsSliderActive.docsNavContainer { + background-color: #121212; + padding-bottom: 0; + } } /* content */ diff --git a/microsite/static/img/home.png b/microsite/static/img/home.png new file mode 100644 index 0000000000..e217581586 Binary files /dev/null and b/microsite/static/img/home.png differ diff --git a/microsite/static/img/xcmetrics-icon.png b/microsite/static/img/xcmetrics-icon.png new file mode 100644 index 0000000000..09f3ea1e68 Binary files /dev/null and b/microsite/static/img/xcmetrics-icon.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 4d984e63e3..bddce144d3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -33,6 +33,7 @@ nav: - Core Features: - Software Catalog: - Overview: 'features/software-catalog/index.md' + - The Life of an Entity: 'features/software-catalog/life-of-an-entity.md' - Catalog Configuration: 'features/software-catalog/configuration.md' - System Model: 'features/software-catalog/system-model.md' - YAML File Format: 'features/software-catalog/descriptor-format.md' @@ -64,6 +65,7 @@ nav: - Getting Started: 'features/search/getting-started.md' - Concepts: 'features/search/concepts.md' - Search Architecture: 'features/search/architecture.md' + - Search Engines: 'features/search/search-engines.md' - TechDocs: - Overview: 'features/techdocs/README.md' - Getting Started: 'features/techdocs/getting-started.md' @@ -78,6 +80,8 @@ nav: - FAQ: 'features/techdocs/FAQ.md' - Integrations: - Overview: 'integrations/index.md' + - AWS S3: + - Locations: 'integrations/aws-s3/locations.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/package.json b/package.json index 93517d3374..c9a33f78e6 100644 --- a/package.json +++ b/package.json @@ -38,9 +38,14 @@ "packages": [ "packages/*", "plugins/*" + ], + "nohoist": [ + "**/@storybook/**", + "**/@storybook" ] }, "resolutions": { + "**/@graphql-codegen/cli/**/ws": "^7.4.6", "**/@roadiehq/**/@backstage/core": "*", "**/@roadiehq/**/@backstage/plugin-catalog": "*", "**/@roadiehq/**/@backstage/catalog-model": "*", @@ -54,6 +59,7 @@ "@microsoft/api-extractor-model": "^7.13.3" }, "devDependencies": { + "@types/webpack": "^5.28.0", "@changesets/cli": "^2.14.0", "@octokit/openapi-types": "^2.2.0", "@spotify/prettier-config": "^10.0.0", diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index 494e7dbad2..a3af287fc8 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,147 @@ # example-app +## 0.2.43 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-explore@0.3.14 + - @backstage/core-app-api@0.1.10 + - @backstage/plugin-catalog-import@0.5.19 + - @backstage/plugin-catalog@0.6.13 + - @backstage/plugin-org@0.3.20 + - @backstage/plugin-catalog-react@0.4.4 + - @backstage/core-components@0.3.3 + - @backstage/plugin-techdocs@0.11.0 + - @backstage/plugin-search@0.4.8 + +## 0.2.42 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-scaffolder@0.11.1 + - @backstage/plugin-home@0.2.0 + - @backstage/cli@0.7.9 + - @backstage/core-app-api@0.1.9 + - @backstage/plugin-catalog-import@0.5.18 + - @backstage/plugin-catalog@0.6.12 + - @backstage/plugin-catalog-react@0.4.3 + - @backstage/plugin-sentry@0.3.19 + - @backstage/plugin-jenkins@0.5.4 + - @backstage/core-components@0.3.2 + - @backstage/theme@0.2.10 + +## 0.2.41 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-cost-insights@0.11.4 + - @backstage/plugin-github-actions@0.4.16 + - @backstage/plugin-jenkins@0.5.3 + - @backstage/plugin-kubernetes@0.4.11 + - @backstage/plugin-pagerduty@0.3.11 + - @backstage/plugin-scaffolder@0.11.0 + - @backstage/plugin-sentry@0.3.18 + - @backstage/plugin-techdocs@0.10.4 + - @backstage/core-app-api@0.1.8 + - @backstage/core-components@0.3.1 + - @backstage/core-plugin-api@0.1.6 + - @backstage/plugin-api-docs@0.6.6 + - @backstage/plugin-catalog@0.6.11 + - @backstage/plugin-catalog-import@0.5.17 + - @backstage/plugin-catalog-react@0.4.2 + - @backstage/plugin-explore@0.3.13 + - @backstage/plugin-lighthouse@0.2.23 + - @backstage/plugin-rollbar@0.3.12 + - @backstage/plugin-search@0.4.7 + - @backstage/plugin-tech-radar@0.4.5 + - @backstage/cli@0.7.8 + - @backstage/integration-react@0.1.7 + +## 0.2.40 + +### Patch Changes + +- Updated dependencies + - @backstage/cli@0.7.7 + - @backstage/plugin-catalog@0.6.10 + - @backstage/core-components@0.3.0 + - @backstage/core-plugin-api@0.1.5 + - @backstage/plugin-api-docs@0.6.5 + - @backstage/plugin-scaffolder@0.10.3 + - @backstage/plugin-search@0.4.6 + - @backstage/plugin-cost-insights@0.11.3 + - @backstage/plugin-kubernetes@0.4.10 + - @backstage/plugin-lighthouse@0.2.22 + - @backstage/plugin-pagerduty@0.3.10 + - @backstage/plugin-techdocs@0.10.3 + - @backstage/search-common@0.1.3 + - @backstage/core-app-api@0.1.7 + - @backstage/integration-react@0.1.6 + - @backstage/plugin-badges@0.2.7 + - @backstage/plugin-catalog-import@0.5.16 + - @backstage/plugin-catalog-react@0.4.1 + - @backstage/plugin-circleci@0.2.21 + - @backstage/plugin-cloudbuild@0.2.21 + - @backstage/plugin-code-coverage@0.1.9 + - @backstage/plugin-explore@0.3.12 + - @backstage/plugin-gcp-projects@0.3.2 + - @backstage/plugin-github-actions@0.4.15 + - @backstage/plugin-graphiql@0.2.14 + - @backstage/plugin-jenkins@0.5.2 + - @backstage/plugin-kafka@0.2.13 + - @backstage/plugin-newrelic@0.3.2 + - @backstage/plugin-org@0.3.19 + - @backstage/plugin-rollbar@0.3.11 + - @backstage/plugin-sentry@0.3.17 + - @backstage/plugin-shortcuts@0.1.6 + - @backstage/plugin-tech-radar@0.4.4 + - @backstage/plugin-todo@0.1.7 + - @backstage/plugin-user-settings@0.3.3 + +## 0.2.39 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-scaffolder@0.10.2 + - @backstage/core-components@0.2.0 + - @backstage/plugin-catalog@0.6.9 + - @backstage/plugin-catalog-react@0.4.0 + - @backstage/cli@0.7.6 + - @backstage/core-app-api@0.1.6 + - @backstage/core-plugin-api@0.1.4 + - @backstage/integration-react@0.1.5 + - @backstage/theme@0.2.9 + - @backstage/plugin-api-docs@0.6.4 + - @backstage/plugin-badges@0.2.6 + - @backstage/plugin-catalog-import@0.5.15 + - @backstage/plugin-circleci@0.2.20 + - @backstage/plugin-cloudbuild@0.2.20 + - @backstage/plugin-code-coverage@0.1.8 + - @backstage/plugin-cost-insights@0.11.2 + - @backstage/plugin-explore@0.3.11 + - @backstage/plugin-gcp-projects@0.3.1 + - @backstage/plugin-github-actions@0.4.14 + - @backstage/plugin-graphiql@0.2.13 + - @backstage/plugin-jenkins@0.5.1 + - @backstage/plugin-kafka@0.2.12 + - @backstage/plugin-kubernetes@0.4.9 + - @backstage/plugin-lighthouse@0.2.21 + - @backstage/plugin-newrelic@0.3.1 + - @backstage/plugin-org@0.3.18 + - @backstage/plugin-pagerduty@0.3.9 + - @backstage/plugin-rollbar@0.3.10 + - @backstage/plugin-search@0.4.5 + - @backstage/plugin-sentry@0.3.16 + - @backstage/plugin-shortcuts@0.1.5 + - @backstage/plugin-tech-radar@0.4.3 + - @backstage/plugin-techdocs@0.10.2 + - @backstage/plugin-todo@0.1.6 + - @backstage/plugin-user-settings@0.3.2 + ## 0.2.37 ### Patch Changes diff --git a/packages/app/cypress/integration/components/search/SearchPage.js b/packages/app/cypress/integration/components/search/SearchPage.js index 4db7acf4fd..6a18d16198 100644 --- a/packages/app/cypress/integration/components/search/SearchPage.js +++ b/packages/app/cypress/integration/components/search/SearchPage.js @@ -34,7 +34,7 @@ describe('SearchPage', () => { cy.visit('/search-next', { onBeforeLoad(win) { cy.stub(win, 'fetch') - .withArgs(`${API_ENDPOINT}?term=&pageCursor=`) + .withArgs(`${API_ENDPOINT}?term=`) .resolves({ ok: true, json: () => ({ results }), @@ -56,7 +56,7 @@ describe('SearchPage', () => { onBeforeLoad(win) { cy.stub(win, 'fetch') .withArgs( - `${API_ENDPOINT}?term=&filters%5Bkind%5D=Component&filters%5Blifecycle%5D%5B0%5D=experimental&pageCursor=`, + `${API_ENDPOINT}?term=&filters%5Bkind%5D=Component&filters%5Blifecycle%5D%5B0%5D=experimental`, ) .resolves({ ok: true, @@ -102,7 +102,7 @@ describe('SearchPage', () => { cy.visit('/search-next?query=backstage', { onBeforeLoad(win) { cy.stub(win, 'fetch') - .withArgs(`${API_ENDPOINT}?term=backstage&pageCursor=`) + .withArgs(`${API_ENDPOINT}?term=backstage`) .resolves({ ok: true, json: () => ({ results: [] }), diff --git a/packages/app/package.json b/packages/app/package.json index 388635e56b..162474fd0e 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,54 +1,55 @@ { "name": "example-app", - "version": "0.2.37", + "version": "0.2.43", "private": true, "bundled": true, "dependencies": { "@backstage/catalog-model": "^0.9.0", - "@backstage/cli": "^0.7.4", - "@backstage/core-app-api": "^0.1.3", - "@backstage/core-components": "^0.1.5", - "@backstage/core-plugin-api": "^0.1.3", - "@backstage/integration-react": "^0.1.4", - "@backstage/plugin-api-docs": "^0.6.2", - "@backstage/plugin-badges": "^0.2.5", - "@backstage/plugin-catalog": "^0.6.7", - "@backstage/plugin-catalog-import": "^0.5.13", - "@backstage/plugin-catalog-react": "^0.3.0", - "@backstage/plugin-circleci": "^0.2.19", - "@backstage/plugin-cloudbuild": "^0.2.19", - "@backstage/plugin-code-coverage": "^0.1.7", - "@backstage/plugin-cost-insights": "^0.11.0", - "@backstage/plugin-explore": "^0.3.10", - "@backstage/plugin-gcp-projects": "^0.3.0", - "@backstage/plugin-github-actions": "^0.4.13", - "@backstage/plugin-graphiql": "^0.2.12", - "@backstage/plugin-jenkins": "^0.5.0", - "@backstage/plugin-kafka": "^0.2.11", - "@backstage/plugin-kubernetes": "^0.4.8", - "@backstage/plugin-lighthouse": "^0.2.20", - "@backstage/plugin-newrelic": "^0.3.0", - "@backstage/plugin-org": "^0.3.17", - "@backstage/plugin-pagerduty": "0.3.8", - "@backstage/plugin-rollbar": "^0.3.9", - "@backstage/plugin-scaffolder": "^0.10.1", - "@backstage/plugin-search": "^0.4.3", - "@backstage/plugin-sentry": "^0.3.15", - "@backstage/plugin-shortcuts": "^0.1.4", - "@backstage/plugin-tech-radar": "^0.4.1", - "@backstage/plugin-techdocs": "^0.10.0", - "@backstage/plugin-todo": "^0.1.5", - "@backstage/plugin-user-settings": "^0.3.0", - "@backstage/search-common": "^0.1.2", - "@backstage/theme": "^0.2.8", - "@material-ui/core": "^4.11.0", + "@backstage/cli": "^0.7.9", + "@backstage/core-app-api": "^0.1.10", + "@backstage/core-components": "^0.3.3", + "@backstage/core-plugin-api": "^0.1.6", + "@backstage/integration-react": "^0.1.7", + "@backstage/plugin-api-docs": "^0.6.6", + "@backstage/plugin-badges": "^0.2.7", + "@backstage/plugin-catalog": "^0.6.13", + "@backstage/plugin-catalog-import": "^0.5.19", + "@backstage/plugin-catalog-react": "^0.4.4", + "@backstage/plugin-circleci": "^0.2.21", + "@backstage/plugin-cloudbuild": "^0.2.21", + "@backstage/plugin-code-coverage": "^0.1.9", + "@backstage/plugin-cost-insights": "^0.11.4", + "@backstage/plugin-explore": "^0.3.14", + "@backstage/plugin-gcp-projects": "^0.3.2", + "@backstage/plugin-github-actions": "^0.4.16", + "@backstage/plugin-graphiql": "^0.2.14", + "@backstage/plugin-home": "^0.2.0", + "@backstage/plugin-jenkins": "^0.5.4", + "@backstage/plugin-kafka": "^0.2.13", + "@backstage/plugin-kubernetes": "^0.4.11", + "@backstage/plugin-lighthouse": "^0.2.23", + "@backstage/plugin-newrelic": "^0.3.2", + "@backstage/plugin-org": "^0.3.20", + "@backstage/plugin-pagerduty": "0.3.11", + "@backstage/plugin-rollbar": "^0.3.12", + "@backstage/plugin-scaffolder": "^0.11.1", + "@backstage/plugin-search": "^0.4.8", + "@backstage/plugin-sentry": "^0.3.19", + "@backstage/plugin-shortcuts": "^0.1.6", + "@backstage/plugin-tech-radar": "^0.4.5", + "@backstage/plugin-techdocs": "^0.11.0", + "@backstage/plugin-todo": "^0.1.7", + "@backstage/plugin-user-settings": "^0.3.3", + "@backstage/search-common": "^0.1.3", + "@backstage/theme": "^0.2.10", + "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "@octokit/rest": "^18.5.3", - "@roadiehq/backstage-plugin-buildkite": "^1.0.6", - "@roadiehq/backstage-plugin-github-insights": "^1.1.20", - "@roadiehq/backstage-plugin-github-pull-requests": "^1.0.10", - "@roadiehq/backstage-plugin-travis-ci": "^1.0.8", + "@roadiehq/backstage-plugin-buildkite": "^1.0.8", + "@roadiehq/backstage-plugin-github-insights": "^1.1.23", + "@roadiehq/backstage-plugin-github-pull-requests": "^1.0.13", + "@roadiehq/backstage-plugin-travis-ci": "^1.0.11", "history": "^5.0.0", "prop-types": "^15.7.2", "react": "^16.12.0", @@ -60,7 +61,8 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/test-utils": "^0.1.14", + "@backstage/test-utils": "^0.1.17", + "@rjsf/core": "^3.0.0", "@testing-library/cypress": "^7.0.1", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^11.2.5", @@ -68,7 +70,7 @@ "@types/jest": "^26.0.7", "@types/jquery": "^3.3.34", "@types/node": "^14.14.32", - "@types/react-dom": "^16.9.8", + "@types/react-dom": "*", "@types/zen-observable": "^0.8.0", "cross-env": "^7.0.0", "cypress": "^7.3.0", diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index de62971f96..81dbdf4cac 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -20,6 +20,7 @@ import { OAuthRequestDialog, SignInPage, } from '@backstage/core-components'; +import { HomepageCompositionRoot } from '@backstage/plugin-home'; import { apiDocsPlugin, ApiExplorerPage } from '@backstage/plugin-api-docs'; import { CatalogEntityPage, @@ -41,10 +42,19 @@ import { GcpProjectsPage } from '@backstage/plugin-gcp-projects'; import { GraphiQLPage } from '@backstage/plugin-graphiql'; import { LighthousePage } from '@backstage/plugin-lighthouse'; import { NewRelicPage } from '@backstage/plugin-newrelic'; -import { ScaffolderPage, scaffolderPlugin } from '@backstage/plugin-scaffolder'; +import { + ScaffolderPage, + scaffolderPlugin, + ScaffolderFieldExtensions, +} from '@backstage/plugin-scaffolder'; import { SearchPage } from '@backstage/plugin-search'; import { TechRadarPage } from '@backstage/plugin-tech-radar'; -import { TechdocsPage } from '@backstage/plugin-techdocs'; +import { + DefaultTechDocsHome, + TechDocsIndexPage, + techdocsPlugin, + TechDocsReaderPage, +} from '@backstage/plugin-techdocs'; import { UserSettingsPage } from '@backstage/plugin-user-settings'; import AlarmIcon from '@material-ui/icons/Alarm'; import React from 'react'; @@ -54,6 +64,9 @@ import { apis } from './apis'; import { Root } from './components/Root'; import { entityPage } from './components/catalog/EntityPage'; import { searchPage } from './components/search/SearchPage'; +import { LowerCaseValuePickerFieldExtension } from './components/scaffolder/customScaffolderExtensions'; +import { HomePage } from './components/home/HomePage'; + import { providers } from './identityProviders'; import * as plugins from './plugins'; @@ -80,6 +93,7 @@ const app = createApp({ bindRoutes({ bind }) { bind(catalogPlugin.externalRoutes, { createComponent: scaffolderPlugin.routes.root, + viewTechDoc: techdocsPlugin.routes.docRoot, }); bind(apiDocsPlugin.externalRoutes, { createComponent: scaffolderPlugin.routes.root, @@ -98,7 +112,11 @@ const AppRouter = app.getRouter(); const routes = ( - + + {/* TODO(rubenl): Move this to / once its more mature and components exist */} + }> + + } /> } /> - } /> - } /> + }> + + + } + /> + }> + + + + } /> } /> } /> - } /> } /> } /> diff --git a/packages/app/src/components/Root/Root.tsx b/packages/app/src/components/Root/Root.tsx index 48141e4f53..7a666a020a 100644 --- a/packages/app/src/components/Root/Root.tsx +++ b/packages/app/src/components/Root/Root.tsx @@ -82,7 +82,7 @@ export const Root = ({ children }: PropsWithChildren<{}>) => ( {/* Global nav, not org-specific */} - + diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index 4e2e32cc86..4a392e0f05 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -40,6 +40,8 @@ import { isKind, EntityHasResourcesCard, EntityOrphanWarning, + EntityProcessingErrorsPanel, + hasCatalogProcessingErrors, isOrphan, } from '@backstage/plugin-catalog'; import { @@ -209,6 +211,26 @@ const cicdCard = ( ); +const entityWarningContent = ( + <> + + + + + + + + + + + + + + + + +); + const errorsContent = ( @@ -223,14 +245,7 @@ const errorsContent = ( const overviewContent = ( - - - - - - - - + {entityWarningContent} @@ -438,6 +453,7 @@ const apiPage = ( + {entityWarningContent} @@ -468,6 +484,7 @@ const userPage = ( + {entityWarningContent} @@ -483,6 +500,7 @@ const groupPage = ( + {entityWarningContent} @@ -501,6 +519,7 @@ const systemPage = ( + {entityWarningContent} @@ -525,6 +544,7 @@ const domainPage = ( + {entityWarningContent} diff --git a/packages/app/src/components/home/HomePage.tsx b/packages/app/src/components/home/HomePage.tsx new file mode 100644 index 0000000000..e171e22ae9 --- /dev/null +++ b/packages/app/src/components/home/HomePage.tsx @@ -0,0 +1,72 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import Grid from '@material-ui/core/Grid'; +import { + RandomJokeHomePageComponent, + ComponentAccordion, + ComponentTabs, + ComponentTab, +} from '@backstage/plugin-home'; + +export const HomePage = () => ( + + + + + + + + + + + ( + + ), + }, + { + label: 'Any', + Component: () => ( + + ), + }, + ]} + /> + + +); diff --git a/packages/app/src/components/scaffolder/customScaffolderExtensions.tsx b/packages/app/src/components/scaffolder/customScaffolderExtensions.tsx new file mode 100644 index 0000000000..5ef4f617a7 --- /dev/null +++ b/packages/app/src/components/scaffolder/customScaffolderExtensions.tsx @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { FieldValidation } from '@rjsf/core'; +import { + createScaffolderFieldExtension, + TextValuePicker, + scaffolderPlugin, +} from '@backstage/plugin-scaffolder'; + +export const LowerCaseValuePickerFieldExtension = scaffolderPlugin.provide( + createScaffolderFieldExtension({ + name: 'LowerCaseValuePicker', + component: TextValuePicker, + validation: (value: string, validation: FieldValidation) => { + if (value.toLowerCase() !== value) { + validation.addError('Only lowercase values are allowed.'); + } + }, + }), +); diff --git a/packages/app/src/components/search/SearchPage.tsx b/packages/app/src/components/search/SearchPage.tsx index 6fe02de837..54f59d1a6e 100644 --- a/packages/app/src/components/search/SearchPage.tsx +++ b/packages/app/src/components/search/SearchPage.tsx @@ -14,84 +14,34 @@ * limitations under the License. */ -import React, { useState } from 'react'; -import { makeStyles, Theme, Grid, List, Paper } from '@material-ui/core'; -import Pagination from '@material-ui/lab/Pagination'; +import { Content, Header, Lifecycle, Page } from '@backstage/core-components'; import { CatalogResultListItem } from '@backstage/plugin-catalog'; import { + DefaultResultListItem, SearchBar, SearchFilter, SearchResult, + SearchResultPager, SearchType, - DefaultResultListItem, } from '@backstage/plugin-search'; -import { Content, Header, Lifecycle, Page } from '@backstage/core-components'; import { DocsResultListItem } from '@backstage/plugin-techdocs'; -import { SearchResultSet } from '@backstage/search-common'; +import { Grid, List, makeStyles, Paper, Theme } from '@material-ui/core'; +import React from 'react'; const useStyles = makeStyles((theme: Theme) => ({ bar: { padding: theme.spacing(1, 0), }, + filter: { + '& + &': { + marginTop: theme.spacing(2.5), + }, + }, filters: { padding: theme.spacing(2), }, - filter: { - marginTop: theme.spacing(2.5), - }, })); -// TODO: Move this into the search plugin once pagination is natively supported. -// See: https://github.com/backstage/backstage/issues/6062 -const SearchResultList = ({ results }: SearchResultSet) => { - const pageSize = 10; - const [page, setPage] = useState(1); - const changePage = (_: any, pageIndex: number) => { - setPage(pageIndex); - }; - const pageAmount = Math.ceil((results.length || 0) / pageSize); - return ( - <> - - {results - .slice(pageSize * (page - 1), pageSize * page) - .map(({ type, document }) => { - switch (type) { - case 'software-catalog': - return ( - - ); - case 'techdocs': - return ( - - ); - default: - return ( - - ); - } - })} - - - - ); -}; - const SearchPage = () => { const classes = useStyles(); return ( @@ -125,8 +75,37 @@ const SearchPage = () => { - {({ results }) => } + {({ results }) => ( + + {results.map(({ type, document }) => { + switch (type) { + case 'software-catalog': + return ( + + ); + case 'techdocs': + return ( + + ); + default: + return ( + + ); + } + })} + + )} + diff --git a/packages/backend-common/CHANGELOG.md b/packages/backend-common/CHANGELOG.md index 582badb685..f3fcd5a855 100644 --- a/packages/backend-common/CHANGELOG.md +++ b/packages/backend-common/CHANGELOG.md @@ -1,5 +1,49 @@ # @backstage/backend-common +## 0.9.0 + +### Minor Changes + +- a365f1faf: The `ZipArchiveResponse` class now accepts an optional `stripFirstDirectory` parameter. Note that its default value is `false`, which leads to a breaking change in behaviour to previous versions of the class. If you use this class explicitly and want to retain the old behaviour, add a `true` parameter value to its constructor. + +### Patch Changes + +- Updated dependencies + - @backstage/integration@0.6.2 + - @backstage/config@0.1.8 + +## 0.8.10 + +### Patch Changes + +- 8543d9890: Add an optional `info` parameter to the `readTree` filter option with a `size` property. +- 4d909268c: Read responses in `UrlReader#read()` as array buffer instead of as text to allow reading non-text locations such as images. +- 9b4604b38: Add support for watching configuration by implementing the `subscribe` method in the configuration returned by `loadBackendConfig`. +- b8cb12009: Add AWS S3 URL Reader +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/config-loader@0.6.7 + - @backstage/integration@0.6.1 + +## 0.8.9 + +### Patch Changes + +- f7ce7c565: Use a more informative error message when URL reading isn't allowed due to no reader matching the target URL. +- ce1958021: Pass on credentials to the integrations package, so that it can properly pick the API route when using GitHub apps based auth +- Updated dependencies + - @backstage/integration@0.6.0 + +## 0.8.8 + +### Patch Changes + +- 6aa7c3db7: bump node-tar version to the latest +- Updated dependencies + - @backstage/config@0.1.6 + - @backstage/integration@0.5.9 + - @backstage/config-loader@0.6.6 + ## 0.8.7 ### Patch Changes diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 2652dad122..de8c1d4101 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -42,23 +42,14 @@ export class AzureUrlReader implements UrlReader { treeResponseFactory: ReadTreeResponseFactory; }, ); - // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts - // // (undocumented) static factory: ReaderFactory; // (undocumented) read(url: string): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadTreeOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts - // // (undocumented) readUrl(url: string, _options?: ReadUrlOptions): Promise; - // Warning: (ae-forgotten-export) The symbol "SearchOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) @@ -415,6 +406,28 @@ export type PluginEndpointDiscovery = { getExternalBaseUrl(pluginId: string): Promise; }; +// Warning: (ae-missing-release-tag) "ReaderFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReaderFactory = (options: { + config: Config; + logger: Logger_2; + treeResponseFactory: ReadTreeResponseFactory; +}) => UrlReaderPredicateTuple[]; + +// Warning: (ae-missing-release-tag) "ReadTreeOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReadTreeOptions = { + filter?( + path: string, + info?: { + size: number; + }, + ): boolean; + etag?: string; +}; + // Warning: (ae-missing-release-tag) "ReadTreeResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -425,6 +438,18 @@ export type ReadTreeResponse = { etag: string; }; +// Warning: (ae-missing-release-tag) "ReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export interface ReadTreeResponseFactory { + // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromTarArchive(options: FromArchiveOptions): Promise; + // (undocumented) + fromZipArchive(options: FromArchiveOptions): Promise; +} + // Warning: (ae-missing-release-tag) "ReadTreeResponseFile" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -433,6 +458,21 @@ export type ReadTreeResponseFile = { content(): Promise; }; +// Warning: (ae-missing-release-tag) "ReadUrlOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReadUrlOptions = { + etag?: string; +}; + +// Warning: (ae-missing-release-tag) "ReadUrlResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type ReadUrlResponse = { + buffer(): Promise; + etag?: string; +}; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "requestLoggingHandler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -472,6 +512,13 @@ export type RunContainerOptions = { pullImage?: boolean; }; +// Warning: (ae-missing-release-tag) "SearchOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type SearchOptions = { + etag?: string; +}; + // Warning: (ae-missing-release-tag) "SearchResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -561,6 +608,14 @@ export type UrlReader = { search(url: string, options?: SearchOptions): Promise; }; +// Warning: (ae-missing-release-tag) "UrlReaderPredicateTuple" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type UrlReaderPredicateTuple = { + predicate: (url: URL) => boolean; + reader: UrlReader; +}; + // Warning: (ae-missing-release-tag) "UrlReaders" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -592,8 +647,7 @@ export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; // // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here -// src/reading/AzureUrlReader.d.ts:9:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts -// src/reading/types.d.ts:106:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts +// src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:30:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 51ef8128e7..15a559f960 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.8.7", + "version": "0.9.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -30,16 +30,17 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.2", - "@backstage/config": "^0.1.5", - "@backstage/config-loader": "^0.6.5", + "@backstage/config": "^0.1.8", + "@backstage/config-loader": "^0.6.7", "@backstage/errors": "^0.1.1", - "@backstage/integration": "^0.5.8", + "@backstage/integration": "^0.6.2", "@google-cloud/storage": "^5.8.0", "@octokit/rest": "^18.5.3", "@types/cors": "^2.8.6", "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "archiver": "^5.0.2", + "aws-sdk": "^2.840.0", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", @@ -62,7 +63,7 @@ "raw-body": "^2.4.1", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", - "tar": "^6.0.5", + "tar": "^6.1.2", "unzipper": "^0.10.11", "winston": "^3.2.1", "yn": "^4.0.0" @@ -76,8 +77,8 @@ } }, "devDependencies": { - "@backstage/cli": "^0.7.5", - "@backstage/test-utils": "^0.1.15", + "@backstage/cli": "^0.7.9", + "@backstage/test-utils": "^0.1.17", "@types/archiver": "^5.1.0", "@types/compression": "^1.7.0", "@types/concat-stream": "^1.6.0", @@ -92,6 +93,7 @@ "@types/tar": "^4.0.3", "@types/unzipper": "^0.10.3", "@types/webpack-env": "^1.15.2", + "aws-sdk-mock": "^5.2.1", "get-port": "^5.1.1", "http-errors": "^1.7.3", "jest": "^26.0.1", diff --git a/packages/backend-common/src/cache/CacheManager.test.ts b/packages/backend-common/src/cache/CacheManager.test.ts index b55b63f469..fb73687f54 100644 --- a/packages/backend-common/src/cache/CacheManager.test.ts +++ b/packages/backend-common/src/cache/CacheManager.test.ts @@ -104,7 +104,7 @@ describe('CacheManager', () => { manager.forPlugin(plugin2Id).getClient({ defaultTtl: expectedTtl }); const client = DefaultCacheClient as jest.Mock; - const cache = (Keyv as unknown) as jest.Mock; + const cache = Keyv as unknown as jest.Mock; expect(cache).toHaveBeenCalledTimes(2); expect(client).toHaveBeenCalledTimes(2); @@ -124,7 +124,7 @@ describe('CacheManager', () => { const expectedNamespace = 'test-plugin'; manager.forPlugin(expectedNamespace).getClient(); - const cache = (Keyv as unknown) as jest.Mock; + const cache = Keyv as unknown as jest.Mock; const mockCalls = cache.mock.calls.splice(-1); const callArgs = mockCalls[0]; expect(callArgs[0].store).toBeInstanceOf(NoStore); @@ -138,7 +138,7 @@ describe('CacheManager', () => { .forPlugin(expectedNamespace) .getClient({ defaultTtl: expectedTtl }); - const cache = (Keyv as unknown) as jest.Mock; + const cache = Keyv as unknown as jest.Mock; const mockCalls = cache.mock.calls.splice(-1); const callArgs = mockCalls[0]; expect(callArgs[0]).toMatchObject({ @@ -162,7 +162,7 @@ describe('CacheManager', () => { const expectedTtl = 3600; manager.forPlugin('test').getClient({ defaultTtl: expectedTtl }); - const cache = (Keyv as unknown) as jest.Mock; + const cache = Keyv as unknown as jest.Mock; const mockCacheCalls = cache.mock.calls.splice(-1); expect(mockCacheCalls[0][0]).toMatchObject({ ttl: expectedTtl, diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 1316642ae1..7c660ee4e6 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -18,32 +18,142 @@ import { resolve as resolvePath } from 'path'; import parseArgs from 'minimist'; import { Logger } from 'winston'; import { findPaths } from '@backstage/cli-common'; -import { Config, ConfigReader } from '@backstage/config'; +import { Config, ConfigReader, JsonValue } from '@backstage/config'; import { loadConfig } from '@backstage/config-loader'; +class ObservableConfigProxy implements Config { + private config: Config = new ConfigReader({}); + + private readonly subscribers: (() => void)[] = []; + + constructor(private readonly logger: Logger) {} + + setConfig(config: Config) { + this.config = config; + for (const subscriber of this.subscribers) { + try { + subscriber(); + } catch (error) { + this.logger.error(`Config subscriber threw error, ${error}`); + } + } + } + + subscribe(onChange: () => void): { unsubscribe: () => void } { + this.subscribers.push(onChange); + return { + unsubscribe: () => { + const index = this.subscribers.indexOf(onChange); + if (index >= 0) { + this.subscribers.splice(index, 1); + } + }, + }; + } + + has(key: string): boolean { + return this.config.has(key); + } + keys(): string[] { + return this.config.keys(); + } + get(key?: string): T { + return this.config.get(key); + } + getOptional(key?: string): T | undefined { + return this.config.getOptional(key); + } + getConfig(key: string): Config { + return this.config.getConfig(key); + } + getOptionalConfig(key: string): Config | undefined { + return this.config.getOptionalConfig(key); + } + getConfigArray(key: string): Config[] { + return this.config.getConfigArray(key); + } + getOptionalConfigArray(key: string): Config[] | undefined { + return this.config.getOptionalConfigArray(key); + } + getNumber(key: string): number { + return this.config.getNumber(key); + } + getOptionalNumber(key: string): number | undefined { + return this.config.getOptionalNumber(key); + } + getBoolean(key: string): boolean { + return this.config.getBoolean(key); + } + getOptionalBoolean(key: string): boolean | undefined { + return this.config.getOptionalBoolean(key); + } + getString(key: string): string { + return this.config.getString(key); + } + getOptionalString(key: string): string | undefined { + return this.config.getOptionalString(key); + } + getStringArray(key: string): string[] { + return this.config.getStringArray(key); + } + getOptionalStringArray(key: string): string[] | undefined { + return this.config.getOptionalStringArray(key); + } +} + type Options = { logger: Logger; // process.argv or any other overrides argv: string[]; }; +// A global used to ensure that only a single file watcher is active at a time. +let currentCancelFunc: () => void; + /** - * Load configuration for a Backend + * Load configuration for a Backend. + * + * This function should only be called once, during the initialization of the backend. */ export async function loadBackendConfig(options: Options): Promise { const args = parseArgs(options.argv); - const configOpts: string[] = [args.config ?? []].flat(); + const configPaths: string[] = [args.config ?? []].flat(); + + const config = new ObservableConfigProxy(options.logger); /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); + const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configOpts.map(opt => resolvePath(opt)), + configPaths: configPaths.map(opt => resolvePath(opt)), + watch: { + onChange(newConfigs) { + options.logger.info( + `Reloaded config from ${newConfigs.map(c => c.context).join(', ')}`, + ); + + config.setConfig(ConfigReader.fromConfigs(newConfigs)); + }, + stopSignal: new Promise(resolve => { + if (currentCancelFunc) { + currentCancelFunc(); + } + currentCancelFunc = resolve; + + // For reloads of this module we need to use a dispose handler rather than the global. + if (module.hot) { + module.hot.addDisposeHandler(resolve); + } + }), + }, }); options.logger.info( `Loaded config from ${configs.map(c => c.context).join(', ')}`, ); - return ConfigReader.fromConfigs(configs); + config.setConfig(ConfigReader.fromConfigs(configs)); + + return config; } diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index f3a3f218b0..2d60104bcb 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -110,9 +110,7 @@ export class DatabaseManager { * @returns Object with client type returned as `client` and boolean representing whether * or not the client was overridden as `overridden` */ - private getClientType( - pluginId: string, - ): { + private getClientType(pluginId: string): { client: string; overridden: boolean; } { diff --git a/packages/backend-common/src/database/connectors/mysql.ts b/packages/backend-common/src/database/connectors/mysql.ts index f2f2298559..94dce508c7 100644 --- a/packages/backend-common/src/database/connectors/mysql.ts +++ b/packages/backend-common/src/database/connectors/mysql.ts @@ -149,7 +149,7 @@ export async function ensureMysqlDatabaseExists( ) { const admin = createMysqlDatabaseClient(dbConfig, { connection: { - database: (null as unknown) as string, + database: null as unknown as string, }, }); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts new file mode 100644 index 0000000000..084f7e3e13 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -0,0 +1,232 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader, JsonObject } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { DefaultReadTreeResponseFactory } from './tree'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; +import { + AwsS3Integration, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; +import { UrlReaderPredicateTuple } from './types'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +describe('AwsS3UrlReader', () => { + const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { + return AwsS3UrlReader.factory({ + config: new ConfigReader(config), + logger: getVoidLogger(), + treeResponseFactory: DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), + }), + }); + }; + + afterEach(() => { + AWSMock.restore(); + }); + + it('creates a dummy reader without the awsS3 field', () => { + const entries = createReader({ + integrations: {}, + }); + + expect(entries).toHaveLength(1); + }); + + it('creates a reader with credentials correctly configured', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + accessKeyId: 'fakekey', + secretAccessKey: 'fakekey', + }); + + const entries = createReader({ + integrations: { + awsS3: awsS3Integrations, + }, + }); + + expect(entries).toHaveLength(1); + }); + + it('creates a reader with default credentials provider', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + }); + + const entries = createReader({ + integrations: { + awsS3: awsS3Integrations, + }, + }); + + expect(entries).toHaveLength(1); + }); + + describe('predicates', () => { + const readers = createReader({ + integrations: { + awsS3: [{}], + }, + }); + const predicate = readers[0].predicate; + + it('returns true for the correct aws s3 storage host', () => { + expect( + predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')), + ).toBe(true); + }); + + it('returns true for a url with the full path and the correct host', () => { + expect( + predicate( + new URL( + 'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + + it('returns false for an incorrect host', () => { + expect(predicate(new URL('https://amazon.com'))).toBe(false); + }); + + it('returns false for a completely different host', () => { + expect(predicate(new URL('https://storage.cloud.google.com'))).toBe( + false, + ); + }); + + it("returns true for a url with a bucket with '.'", () => { + expect( + predicate( + new URL( + 'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + }); + + describe('read', () => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + ); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.read( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + expect(response.toString().trim()).toBe('site_name: Test'); + }); + + it('rejects unknown targets', async () => { + await expect( + awsS3UrlReader.read( + 'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml', + ), + ).rejects.toThrow( + Error( + `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + ), + ); + }); + }); + + describe('readUrl', () => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + ); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + const buffer = await response.buffer(); + expect(buffer.toString().trim()).toBe('site_name: Test'); + }); + + it('rejects unknown targets', async () => { + await expect( + awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml', + ), + ).rejects.toThrow( + Error( + `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + ), + ); + }); + }); +}); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts new file mode 100644 index 0000000000..d119115930 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import aws, { Credentials, S3 } from 'aws-sdk'; +import { CredentialsOptions } from 'aws-sdk/lib/credentials'; +import { + ReaderFactory, + ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, + SearchResponse, + UrlReader, +} from './types'; +import getRawBody from 'raw-body'; +import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; + +const parseURL = ( + url: string, +): { path: string; bucket: string; region: string } => { + let { host, pathname } = new URL(url); + + /** + * Removes the leading '/' from the pathname to be processed + * as a parameter by AWS S3 SDK getObject method. + */ + pathname = pathname.substr(1); + + /** + * Checks that the given URL is a valid S3 object url. + * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname + */ + const validHost = new RegExp( + /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, + ); + if (!validHost.test(host)) { + throw new Error(`not a valid AWS S3 URL: ${url}`); + } + + const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); + host = host.substring(bucket.length); + const [, , region, ,] = host.split('.'); + + return { + path: pathname, + bucket: bucket, + region: region, + }; +}; + +export class AwsS3UrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + const integrations = ScmIntegrations.fromConfig(config); + + return integrations.awsS3.list().map(integration => { + const creds = AwsS3UrlReader.buildCredentials(integration); + const s3 = new S3({ + apiVersion: '2006-03-01', + credentials: creds, + }); + const reader = new AwsS3UrlReader(integration, s3); + const predicate = (url: URL) => + url.host.endsWith(integration.config.host); + return { reader, predicate }; + }); + }; + + constructor( + private readonly integration: AwsS3Integration, + private readonly s3: S3, + ) {} + + /** + * If accesKeyId and secretAccessKey are missing, the standard credentials provider chain will be used: + * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html + */ + private static buildCredentials( + integration?: AwsS3Integration, + ): Credentials | CredentialsOptions | undefined { + if (!integration) { + return undefined; + } + + const accessKeyId = integration.config.accessKeyId; + const secretAccessKey = integration.config.secretAccessKey; + let explicitCredentials: Credentials | undefined; + + if (accessKeyId && secretAccessKey) { + explicitCredentials = new Credentials({ + accessKeyId, + secretAccessKey, + }); + } + + const roleArn = integration.config.roleArn; + if (roleArn) { + return new aws.ChainableTemporaryCredentials({ + masterCredentials: explicitCredentials, + params: { + RoleSessionName: 'backstage-aws-s3-url-reader', + RoleArn: roleArn, + }, + }); + } + + return explicitCredentials; + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + let params; + if (options?.etag) { + params = { + Bucket: bucket, + Key: path, + IfNoneMatch: options.etag, + }; + } else { + params = { + Bucket: bucket, + Key: path, + }; + } + + const response = this.s3.getObject(params); + const buffer = await getRawBody(response.createReadStream()); + const etag = (await response.promise()).ETag; + + return { + buffer: async () => buffer, + etag: etag, + }; + } catch (e) { + throw new Error(`Could not retrieve file from S3: ${e.message}`); + } + } + + async readTree(): Promise { + throw new Error('AwsS3Reader does not implement readTree'); + } + + async search(): Promise { + throw new Error('AwsS3Reader does not implement search'); + } + + toString() { + const secretAccessKey = this.integration.config.secretAccessKey; + return `awsS3{host=${this.integration.config.host},authed=${Boolean( + secretAccessKey, + )}}`; + } +} diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 2b7c21d4ab..c7c8853929 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -78,21 +78,17 @@ describe('AzureUrlReader', () => { it.each([ { - url: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + url: 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', config: createConfig(), response: expect.objectContaining({ - url: - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + url: 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', }), }, { - url: - 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + url: 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', config: createConfig(), response: expect.objectContaining({ - url: - 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + url: 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', }), }, { diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index bc22854d4e..7b4b1e345e 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -70,7 +70,7 @@ export class AzureUrlReader implements UrlReader { // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html if (response.ok && response.status !== 203) { - return Buffer.from(await response.text()); + return Buffer.from(await response.arrayBuffer()); } const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; @@ -129,7 +129,7 @@ export class AzureUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromZipArchive({ - stream: (archiveAzureResponse.body as unknown) as Readable, + stream: archiveAzureResponse.body as unknown as Readable, etag: commitSha, filter: options?.filter, }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 2849bc3cd9..7925212b20 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -60,13 +60,8 @@ export class BitbucketUrlReader implements UrlReader { private readonly integration: BitbucketIntegration, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { - const { - host, - apiBaseUrl, - token, - username, - appPassword, - } = integration.config; + const { host, apiBaseUrl, token, username, appPassword } = + integration.config; if (!apiBaseUrl) { throw new Error( @@ -91,7 +86,7 @@ export class BitbucketUrlReader implements UrlReader { } if (response.ok) { - return Buffer.from(await response.text()); + return Buffer.from(await response.arrayBuffer()); } const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; @@ -138,7 +133,7 @@ export class BitbucketUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: (archiveBitbucketResponse.body as unknown) as Readable, + stream: archiveBitbucketResponse.body as unknown as Readable, subpath: filepath, etag: lastCommitShortHash, filter: options?.filter, diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 3177ee1f8e..6b45d99cc3 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -82,7 +82,7 @@ export class FetchUrlReader implements UrlReader { if (response.ok) { return { - buffer: async () => Buffer.from(await response.text()), + buffer: async () => Buffer.from(await response.arrayBuffer()), etag: response.headers.get('ETag') ?? undefined, }; } diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index c8002cd3ce..f9717a862a 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -41,9 +41,9 @@ const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); -const mockCredentialsProvider = ({ +const mockCredentialsProvider = { getCredentials: jest.fn().mockResolvedValue({ headers: {} }), -} as unknown) as GithubCredentialsProvider; +} as unknown as GithubCredentialsProvider; const githubProcessor = new GithubUrlReader( new GitHubIntegration( diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index bccddc6838..5dc1430ea8 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -39,10 +39,14 @@ import { ReadUrlResponse, } from './types'; -export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data']; -export type GhBranchResponse = RestEndpointMethodTypes['repos']['getBranch']['response']['data']; -export type GhTreeResponse = RestEndpointMethodTypes['git']['getTree']['response']['data']; -export type GhBlobResponse = RestEndpointMethodTypes['git']['getBlob']['response']['data']; +export type GhRepoResponse = + RestEndpointMethodTypes['repos']['get']['response']['data']; +export type GhBranchResponse = + RestEndpointMethodTypes['repos']['getBranch']['response']['data']; +export type GhTreeResponse = + RestEndpointMethodTypes['git']['getTree']['response']['data']; +export type GhBlobResponse = + RestEndpointMethodTypes['git']['getBlob']['response']['data']; /** * A processor that adds the ability to read files from GitHub v3 APIs, such as @@ -87,15 +91,20 @@ export class GithubUrlReader implements UrlReader { url: string, options?: ReadUrlOptions, ): Promise { - const ghUrl = getGitHubFileFetchUrl(url, this.integration.config); - const { headers } = await this.deps.credentialsProvider.getCredentials({ + const credentials = await this.deps.credentialsProvider.getCredentials({ url, }); + const ghUrl = getGitHubFileFetchUrl( + url, + this.integration.config, + credentials, + ); + let response: Response; try { - response = await fetch(ghUrl.toString(), { + response = await fetch(ghUrl, { headers: { - ...headers, + ...credentials?.headers, ...(options?.etag && { 'If-None-Match': options.etag }), Accept: 'application/vnd.github.v3.raw', }, @@ -110,7 +119,7 @@ export class GithubUrlReader implements UrlReader { if (response.ok) { return { - buffer: async () => Buffer.from(await response.text()), + buffer: async () => Buffer.from(await response.arrayBuffer()), etag: response.headers.get('ETag') ?? undefined, }; } @@ -195,7 +204,7 @@ export class GithubUrlReader implements UrlReader { return await this.deps.treeResponseFactory.fromTarArchive({ // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want // to stick to using that in exclusively backend code. - stream: (archive.body as unknown) as Readable, + stream: archive.body as unknown as Readable, subpath, etag: sha, filter: options?.filter, @@ -258,9 +267,7 @@ export class GithubUrlReader implements UrlReader { })); } - private async getRepoDetails( - url: string, - ): Promise<{ + private async getRepoDetails(url: string): Promise<{ repo: GhRepoResponse; branch: GhBranchResponse; }> { diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 945444d8ed..51b211e698 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -108,36 +108,30 @@ describe('GitlabUrlReader', () => { it.each([ // Project URLs { - url: - 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', config: createConfig(), response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + url: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', headers: expect.objectContaining({ 'private-token': '', }), }), }, { - url: - 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + url: 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', config: createConfig('0123456789'), response: expect.objectContaining({ - url: - 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + url: 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', headers: expect.objectContaining({ 'private-token': '0123456789', }), }), }, { - url: - 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + url: 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup config: createConfig(), response: expect.objectContaining({ - url: - 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + url: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', }), }, diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 6c272471e2..baee64fab2 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -84,7 +84,7 @@ export class GitlabUrlReader implements UrlReader { if (response.ok) { return { - buffer: async () => Buffer.from(await response.text()), + buffer: async () => Buffer.from(await response.arrayBuffer()), etag: response.headers.get('ETag') ?? undefined, }; } @@ -170,7 +170,7 @@ export class GitlabUrlReader implements UrlReader { } return await this.deps.treeResponseFactory.fromTarArchive({ - stream: (archiveGitLabResponse.body as unknown) as Readable, + stream: archiveGitLabResponse.body as unknown as Readable, subpath: filepath, etag: commitSha, filter: options?.filter, diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 4a166cb517..bbe94a0fb4 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -52,7 +52,11 @@ export class UrlReaderPredicateMux implements UrlReader { } } - throw new NotAllowedError(`Reading from '${url}' is not allowed`); + throw new NotAllowedError( + `Reading from '${url}' is not allowed. ` + + `You may need to configure an integration for the target host, or add it ` + + `to the configured list of allowed hosts at 'backend.reading.allow'`, + ); } async readUrl( diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2b3a2f166c..f9a4865717 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -25,6 +25,7 @@ import { GitlabUrlReader } from './GitlabUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import { FetchUrlReader } from './FetchUrlReader'; import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; type CreateOptions = { /** Root config object */ @@ -74,6 +75,7 @@ export class UrlReaders { GithubUrlReader.factory, GitlabUrlReader.factory, GoogleGcsUrlReader.factory, + AwsS3UrlReader.factory, FetchUrlReader.factory, ]), }); diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml @@ -0,0 +1 @@ +site_name: Test diff --git a/packages/backend-common/src/reading/__fixtures__/mock-main.zip b/packages/backend-common/src/reading/__fixtures__/mock-main.zip index beee59d3a0..ae362448fc 100644 Binary files a/packages/backend-common/src/reading/__fixtures__/mock-main.zip and b/packages/backend-common/src/reading/__fixtures__/mock-main.zip differ diff --git a/packages/backend-common/src/reading/__fixtures__/mock-with-extra-root-dir.zip b/packages/backend-common/src/reading/__fixtures__/mock-with-extra-root-dir.zip new file mode 100644 index 0000000000..beee59d3a0 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/mock-with-extra-root-dir.zip differ diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 4c601556d7..207f01b37f 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -19,10 +19,17 @@ export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; export type { + ReaderFactory, + ReadTreeOptions, ReadTreeResponse, + ReadTreeResponseFactory, ReadTreeResponseFile, + ReadUrlOptions, + ReadUrlResponse, + SearchOptions, SearchResponse, SearchResponseFile, UrlReader, + UrlReaderPredicateTuple, } from './types'; export { UrlReaders } from './UrlReaders'; diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index a81add58f7..f7ab7f5349 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -28,7 +28,7 @@ import { import { stripFirstDirectoryFromPath } from './util'; // Tar types for `Parse` is not a proper constructor, but it should be -const TarParseStream = (Parse as unknown) as { new (): ParseStream }; +const TarParseStream = Parse as unknown as { new (): ParseStream }; const pipeline = promisify(pipelineCb); @@ -43,7 +43,7 @@ export class TarArchiveResponse implements ReadTreeResponse { private readonly subPath: string, private readonly workDir: string, public readonly etag: string, - private readonly filter?: (path: string) => boolean, + private readonly filter?: (path: string, info: { size: number }) => boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -92,7 +92,7 @@ export class TarArchiveResponse implements ReadTreeResponse { const path = relativePath.slice(this.subPath.length); if (this.filter) { - if (!this.filter(path)) { + if (!this.filter(path, { size: entry.remain })) { entry.resume(); return; } @@ -155,7 +155,7 @@ export class TarArchiveResponse implements ReadTreeResponse { tar.extract({ strip, cwd: dir, - filter: path => { + filter: (path, stat) => { // File path relative to the root extracted directory. Will remove the // top level dir name from the path since its name is hard to predetermine. const relativePath = stripFirstDirectoryFromPath(path); @@ -164,7 +164,7 @@ export class TarArchiveResponse implements ReadTreeResponse { } if (this.filter) { const innerPath = path.split('/').slice(strip).join('/'); - return this.filter(innerPath); + return this.filter(innerPath, { size: stat.size }); } return true; }, diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index 659875286e..f6f9a7845a 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -22,11 +22,15 @@ import { ZipArchiveResponse } from './ZipArchiveResponse'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.zip'), ); +const archiveDataWithExtraDir = fs.readFileSync( + resolvePath(__filename, '../../__fixtures__/mock-with-extra-root-dir.zip'), +); describe('ZipArchiveResponse', () => { beforeEach(() => { mockFs({ '/test-archive.zip': archiveData, + '/test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, '/tmp': mockFs.directory(), }); }); @@ -58,6 +62,36 @@ describe('ZipArchiveResponse', () => { ]); }); + it('should read files and strip root dir if requested', async () => { + const stream = fs.createReadStream('/test-archive-with-extra-root-dir.zip'); + + const res = new ZipArchiveResponse( + stream, + '', + '/tmp', + 'etag', + undefined, + true, + ); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + { + path: 'docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.zip'); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 45c6880a55..fb22f24b94 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -37,7 +37,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { private readonly subPath: string, private readonly workDir: string, public readonly etag: string, - private readonly filter?: (path: string) => boolean, + private readonly filter?: (path: string, info: { size: number }) => boolean, + private readonly stripFirstDirectory?: boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -67,7 +68,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { } private shouldBeIncluded(entry: Entry): boolean { - const strippedPath = stripFirstDirectoryFromPath(entry.path); + const strippedPath = this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path; if (this.subPath) { if (!strippedPath.startsWith(this.subPath)) { @@ -75,7 +78,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { } } if (this.filter) { - return this.filter(this.getInnerPath(entry.path)); + return this.filter(this.getInnerPath(entry.path), { + size: + (entry.vars as { uncompressedSize?: number }).uncompressedSize ?? + entry.vars.compressedSize, + }); } return true; } @@ -95,7 +102,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { if (this.shouldBeIncluded(entry)) { files.push({ - path: this.getInnerPath(stripFirstDirectoryFromPath(entry.path)), + path: this.getInnerPath( + this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path, + ), content: () => entry.buffer(), }); } else { @@ -144,7 +155,9 @@ export class ZipArchiveResponse implements ReadTreeResponse { // as a zip can have files with directories without directory entries if (entry.type === 'File' && this.shouldBeIncluded(entry)) { const entryPath = this.getInnerPath( - stripFirstDirectoryFromPath(entry.path), + this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path, ); const dirname = platformPath.dirname(entryPath); if (dirname) { diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 8efc833ead..93f287d4fe 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -104,7 +104,7 @@ export type ReadTreeOptions = { * * If no filter is provided all files are extracted. */ - filter?(path: string): boolean; + filter?(path: string, info?: { size: number }): boolean; /** * An etag can be provided to check whether readTree's response has changed from a previous execution. @@ -164,7 +164,7 @@ export type FromArchiveOptions = { // etag of the blob etag: string; // Filter passed on from the ReadTreeOptions - filter?: (path: string) => boolean; + filter?: (path: string, info?: { size: number }) => boolean; }; export interface ReadTreeResponseFactory { diff --git a/packages/backend-common/src/scm/git.test.ts b/packages/backend-common/src/scm/git.test.ts index 0b70edc45d..abbafc375d 100644 --- a/packages/backend-common/src/scm/git.test.ts +++ b/packages/backend-common/src/scm/git.test.ts @@ -123,9 +123,9 @@ describe('Git', () => { await git.clone({ url, dir }); - const { onAuth } = ((isomorphic.clone as unknown) as jest.Mock< - typeof isomorphic['clone'] - >).mock.calls[0][0]!; + const { onAuth } = ( + isomorphic.clone as unknown as jest.Mock + ).mock.calls[0][0]!; expect(onAuth()).toEqual(auth); }); @@ -190,9 +190,9 @@ describe('Git', () => { await git.fetch({ remote, dir }); - const { onAuth } = ((isomorphic.fetch as unknown) as jest.Mock< - typeof isomorphic['fetch'] - >).mock.calls[0][0]!; + const { onAuth } = ( + isomorphic.fetch as unknown as jest.Mock + ).mock.calls[0][0]!; expect(onAuth()).toEqual(auth); }); @@ -279,9 +279,9 @@ describe('Git', () => { await git.push({ remote, dir }); - const { onAuth } = ((isomorphic.push as unknown) as jest.Mock< - typeof isomorphic['push'] - >).mock.calls[0][0]!; + const { onAuth } = ( + isomorphic.push as unknown as jest.Mock + ).mock.calls[0][0]!; expect(onAuth()).toEqual(auth); }); diff --git a/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts b/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts index 7a3ebfed70..ed16c9c9a7 100644 --- a/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts +++ b/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts @@ -154,14 +154,8 @@ export class ServiceBuilderImpl implements ServiceBuilder { async start(): Promise { const app = express(); - const { - port, - host, - logger, - corsOptions, - httpsSettings, - helmetOptions, - } = this.getOptions(); + const { port, host, logger, corsOptions, httpsSettings, helmetOptions } = + this.getOptions(); app.use(helmet(helmetOptions)); if (corsOptions) { diff --git a/packages/backend-common/src/util/DockerContainerRunner.ts b/packages/backend-common/src/util/DockerContainerRunner.ts index 8dbc5335c9..33a4874151 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.ts @@ -92,18 +92,17 @@ export class DockerContainerRunner implements ContainerRunner { Env.push(`${key}=${value}`); } - const [ - { Error: error, StatusCode: statusCode }, - ] = await this.dockerClient.run(imageName, args, logStream, { - Volumes, - HostConfig: { - Binds, - }, - ...(workingDir ? { WorkingDir: workingDir } : {}), - Entrypoint: command, - Env, - ...userOptions, - } as Docker.ContainerCreateOptions); + const [{ Error: error, StatusCode: statusCode }] = + await this.dockerClient.run(imageName, args, logStream, { + Volumes, + HostConfig: { + Binds, + }, + ...(workingDir ? { WorkingDir: workingDir } : {}), + Entrypoint: command, + Env, + ...userOptions, + } as Docker.ContainerCreateOptions); if (error) { throw new Error( diff --git a/packages/backend-test-utils/CHANGELOG.md b/packages/backend-test-utils/CHANGELOG.md index 2faa5afbc3..6d33537e7c 100644 --- a/packages/backend-test-utils/CHANGELOG.md +++ b/packages/backend-test-utils/CHANGELOG.md @@ -1,5 +1,23 @@ # @backstage/backend-test-utils +## 0.1.6 + +### Patch Changes + +- Updated dependencies + - @backstage/backend-common@0.9.0 + - @backstage/config@0.1.8 + +## 0.1.5 + +### Patch Changes + +- 524045758: Make sure that the unique databases names generated by `TestDatabases.create()` + don't collide. +- Updated dependencies + - @backstage/backend-common@0.8.9 + - @backstage/cli@0.7.8 + ## 0.1.4 ### Patch Changes diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json index 9ec11a58b7..cd2855b833 100644 --- a/packages/backend-test-utils/package.json +++ b/packages/backend-test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-test-utils", "description": "Test helpers library for Backstage backends", - "version": "0.1.4", + "version": "0.1.6", "main": "src/index.ts", "types": "src/index.ts", "private": false, @@ -30,9 +30,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.8.5", - "@backstage/cli": "^0.7.1", - "@backstage/config": "^0.1.5", + "@backstage/backend-common": "^0.9.0", + "@backstage/cli": "^0.7.8", + "@backstage/config": "^0.1.8", "knex": "^0.95.1", "mysql2": "^2.2.5", "pg": "^8.3.0", @@ -41,7 +41,7 @@ "uuid": "^8.0.0" }, "devDependencies": { - "@backstage/cli": "^0.7.1", + "@backstage/cli": "^0.7.8", "jest": "^26.0.1" }, "files": [ diff --git a/packages/backend-test-utils/src/database/TestDatabases.test.ts b/packages/backend-test-utils/src/database/TestDatabases.test.ts index 62612f3fcc..000e047c4d 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.test.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.test.ts @@ -71,7 +71,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_db0'; + const database = input.client.config.connection.database; const output = knexFactory({ client: 'pg', connection: { host, port, user, password, database }, @@ -105,7 +105,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_db0'; + const database = input.client.config.connection.database; const output = knexFactory({ client: 'pg', connection: { host, port, user, password, database }, @@ -139,7 +139,7 @@ describe('TestDatabases', () => { await input.insert({ x: 'y' }).into('a'); // Look for the mark - const database = 'backstage_plugin_db0'; + const database = input.client.config.connection.database; const output = knexFactory({ client: 'mysql2', connection: { host, port, user, password, database }, diff --git a/packages/backend-test-utils/src/database/TestDatabases.ts b/packages/backend-test-utils/src/database/TestDatabases.ts index c35202354d..329aec5e8a 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.ts @@ -16,6 +16,7 @@ import { DatabaseManager } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; +import { randomBytes } from 'crypto'; import { Knex } from 'knex'; import { isDockerDisabledForTests } from '../util/isDockerDisabledForTests'; import { startMysqlContainer } from './startMysqlContainer'; @@ -34,7 +35,6 @@ import { export class TestDatabases { private readonly instanceById: Map; private readonly supportedIds: TestDatabaseId[]; - private lastDatabaseIndex: number; /** * Creates an empty `TestDatabases` instance, and sets up Jest to clean up @@ -99,7 +99,6 @@ export class TestDatabases { private constructor(supportedIds: TestDatabaseId[]) { this.instanceById = new Map(); this.supportedIds = supportedIds; - this.lastDatabaseIndex = 0; } supports(id: TestDatabaseId): boolean { @@ -142,7 +141,7 @@ export class TestDatabases { // Ensure that a unique logical database is created in the instance const connection = await instance.databaseManager - .forPlugin(String(`db${this.lastDatabaseIndex++}`)) + .forPlugin(`db${randomBytes(16).toString('hex')}`) .getClient(); instance.connections.push(connection); diff --git a/packages/backend-test-utils/src/database/types.ts b/packages/backend-test-utils/src/database/types.ts index b5516a19c0..514a3a85e8 100644 --- a/packages/backend-test-utils/src/database/types.ts +++ b/packages/backend-test-utils/src/database/types.ts @@ -38,33 +38,31 @@ export type Instance = { databaseManager: DatabaseManager; connections: Array; }; -export const allDatabases: Record< - TestDatabaseId, - TestDatabaseProperties -> = Object.freeze({ - POSTGRES_13: { - name: 'Postgres 13.x', - driver: 'pg', - dockerImageName: 'postgres:13', - connectionStringEnvironmentVariableName: - 'BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING', - }, - POSTGRES_9: { - name: 'Postgres 9.x', - driver: 'pg', - dockerImageName: 'postgres:9', - connectionStringEnvironmentVariableName: - 'BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING', - }, - MYSQL_8: { - name: 'MySQL 8.x', - driver: 'mysql2', - dockerImageName: 'mysql:8', - connectionStringEnvironmentVariableName: - 'BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING', - }, - SQLITE_3: { - name: 'SQLite 3.x', - driver: 'sqlite3', - }, -}); +export const allDatabases: Record = + Object.freeze({ + POSTGRES_13: { + name: 'Postgres 13.x', + driver: 'pg', + dockerImageName: 'postgres:13', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_POSTGRES13_CONNECTION_STRING', + }, + POSTGRES_9: { + name: 'Postgres 9.x', + driver: 'pg', + dockerImageName: 'postgres:9', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_POSTGRES9_CONNECTION_STRING', + }, + MYSQL_8: { + name: 'MySQL 8.x', + driver: 'mysql2', + dockerImageName: 'mysql:8', + connectionStringEnvironmentVariableName: + 'BACKSTAGE_TEST_DATABASE_MYSQL8_CONNECTION_STRING', + }, + SQLITE_3: { + name: 'SQLite 3.x', + driver: 'sqlite3', + }, + }); diff --git a/packages/backend/CHANGELOG.md b/packages/backend/CHANGELOG.md index a3e08db6b3..b27dbd07e7 100644 --- a/packages/backend/CHANGELOG.md +++ b/packages/backend/CHANGELOG.md @@ -1,5 +1,51 @@ # example-backend +## 0.2.43 + +### Patch Changes + +- Updated dependencies + - @backstage/backend-common@0.9.0 + - @backstage/plugin-catalog-backend@0.13.5 + - @backstage/plugin-search-backend-module-pg@0.1.3 + - @backstage/plugin-auth-backend@0.3.23 + - @backstage/plugin-scaffolder-backend@0.15.2 + - @backstage/integration@0.6.2 + - @backstage/config@0.1.8 + - @backstage/plugin-kubernetes-backend@0.3.15 + - @backstage/plugin-techdocs-backend@0.10.0 + - @backstage/plugin-jenkins-backend@0.1.4 + - @backstage/plugin-app-backend@0.3.16 + - @backstage/plugin-badges-backend@0.1.9 + - @backstage/plugin-code-coverage-backend@0.1.10 + - @backstage/plugin-graphql-backend@0.1.9 + - @backstage/plugin-kafka-backend@0.2.9 + - @backstage/plugin-proxy-backend@0.2.12 + - @backstage/plugin-rollbar-backend@0.1.14 + - @backstage/plugin-scaffolder-backend-module-rails@0.1.5 + - @backstage/plugin-search-backend@0.2.5 + - @backstage/plugin-todo-backend@0.1.11 + - example-app@0.2.43 + +## 0.2.41 + +### Patch Changes + +- Updated dependencies + - @backstage/plugin-auth-backend@0.3.20 + - @backstage/integration@0.6.0 + - @backstage/plugin-scaffolder-backend@0.15.0 + - @backstage/backend-common@0.8.9 + - @backstage/plugin-kubernetes-backend@0.3.14 + - @backstage/plugin-search-backend-module-elasticsearch@0.0.2 + - @backstage/plugin-search-backend-module-pg@0.1.1 + - @backstage/plugin-catalog-backend@0.13.2 + - @backstage/plugin-code-coverage-backend@0.1.9 + - @backstage/plugin-scaffolder-backend-module-rails@0.1.4 + - @backstage/plugin-techdocs-backend@0.9.2 + - @backstage/plugin-todo-backend@0.1.9 + - example-app@0.2.41 + ## 0.2.38 ### Patch Changes diff --git a/packages/backend/package.json b/packages/backend/package.json index 9fc455801b..3dccb32b82 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,6 +1,6 @@ { "name": "example-backend", - "version": "0.2.38", + "version": "0.2.43", "main": "dist/index.cjs.js", "types": "src/index.ts", "license": "Apache-2.0", @@ -27,43 +27,47 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { - "@backstage/backend-common": "^0.8.7", + "@backstage/backend-common": "^0.9.0", "@backstage/catalog-client": "^0.3.17", "@backstage/catalog-model": "^0.9.0", - "@backstage/config": "^0.1.5", - "@backstage/integration": "^0.5.8", - "@backstage/plugin-app-backend": "^0.3.15", - "@backstage/plugin-auth-backend": "^0.3.18", - "@backstage/plugin-badges-backend": "^0.1.8", - "@backstage/plugin-catalog-backend": "^0.13.0", - "@backstage/plugin-code-coverage-backend": "^0.1.8", - "@backstage/plugin-graphql-backend": "^0.1.8", - "@backstage/plugin-jenkins-backend": "^0.1.2", - "@backstage/plugin-kubernetes-backend": "^0.3.11", - "@backstage/plugin-kafka-backend": "^0.2.8", - "@backstage/plugin-proxy-backend": "^0.2.9", - "@backstage/plugin-rollbar-backend": "^0.1.13", - "@backstage/plugin-scaffolder-backend": "^0.14.1", - "@backstage/plugin-scaffolder-backend-module-rails": "^0.1.3", - "@backstage/plugin-search-backend": "^0.2.3", + "@backstage/config": "^0.1.8", + "@backstage/integration": "^0.6.2", + "@backstage/plugin-app-backend": "^0.3.16", + "@backstage/plugin-auth-backend": "^0.3.23", + "@backstage/plugin-badges-backend": "^0.1.9", + "@backstage/plugin-catalog-backend": "^0.13.5", + "@backstage/plugin-code-coverage-backend": "^0.1.10", + "@backstage/plugin-graphql-backend": "^0.1.9", + "@backstage/plugin-jenkins-backend": "^0.1.4", + "@backstage/plugin-kubernetes-backend": "^0.3.15", + "@backstage/plugin-kafka-backend": "^0.2.9", + "@backstage/plugin-proxy-backend": "^0.2.12", + "@backstage/plugin-rollbar-backend": "^0.1.14", + "@backstage/plugin-scaffolder-backend": "^0.15.2", + "@backstage/plugin-scaffolder-backend-module-rails": "^0.1.5", + "@backstage/plugin-search-backend": "^0.2.5", "@backstage/plugin-search-backend-node": "^0.4.0", - "@backstage/plugin-techdocs-backend": "^0.9.0", - "@backstage/plugin-todo-backend": "^0.1.8", + "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.2", + "@backstage/plugin-search-backend-module-pg": "^0.1.3", + "@backstage/plugin-techdocs-backend": "^0.10.0", + "@backstage/plugin-todo-backend": "^0.1.11", "@gitbeaker/node": "^30.2.0", "@octokit/rest": "^18.5.3", - "azure-devops-node-api": "^10.2.2", + "azure-devops-node-api": "^11.0.1", "dockerode": "^3.2.1", - "example-app": "^0.2.37", + "example-app": "^0.2.43", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "express-prom-bundle": "^6.3.6", "knex": "^0.95.1", "pg": "^8.3.0", "pg-connection-string": "^2.3.0", + "prom-client": "^13.2.0", "sqlite3": "^5.0.1", "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.7.5", + "@backstage/cli": "^0.7.8", "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5" diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index aecb233581..78c6c4c20b 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -36,6 +36,7 @@ import { } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; +import { metricsInit, metricsHandler } from './metrics'; import auth from './plugins/auth'; import catalog from './plugins/catalog'; import codeCoverage from './plugins/codecoverage'; @@ -72,6 +73,7 @@ function makeCreateEnv(config: Config) { } async function main() { + metricsInit(); const logger = getRootLogger(); logger.info( @@ -124,6 +126,7 @@ async function main() { const service = createServiceBuilder(module) .loadConfig(config) .addRouter('', await healthcheck(healthcheckEnv)) + .addRouter('', metricsHandler()) .addRouter('/api', apiRouter) .addRouter('', await app(appEnv)); diff --git a/packages/backend/src/metrics.ts b/packages/backend/src/metrics.ts new file mode 100644 index 0000000000..8834022cd9 --- /dev/null +++ b/packages/backend/src/metrics.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useHotCleanup } from '@backstage/backend-common'; +import { RequestHandler, Request } from 'express'; +import promBundle from 'express-prom-bundle'; +import prom from 'prom-client'; +import * as url from 'url'; + +/** + * Experimental Prometheus metrics used to benchmark the performance of the + * software catalog. Use this at your own risk. + */ +const rootRegEx = new RegExp('^/([^/]*)/.*'); +const apiRegEx = new RegExp('^/api/([^/]*)/.*'); + +function normalizePath(req: Request): string { + const path = url.parse(req.originalUrl || req.url).pathname || '/'; + + // Capture /api/ and the plugin name + if (apiRegEx.test(path)) { + return path.replace(apiRegEx, '/api/$1'); + } + + // Only the first path segment at root level + return path.replace(rootRegEx, '/$1'); +} + +export function metricsInit(): void { + prom.collectDefaultMetrics({ prefix: 'backstage_' }); +} + +/** + * Adds a /metrics endpoint, register default runtime metrics and instrument the router. + */ +export function metricsHandler(): RequestHandler { + // We can only initialize the metrics once and have to clean them up between hot reloads + useHotCleanup(module, () => prom.register.clear()); + + return promBundle({ + includeMethod: true, + includePath: true, + // Using includePath alone is problematic, as it will include path labels with high + // cardinality (e.g. path params). Instead we would have to template them. However, this + // is difficult, as every backend plugin might use different routes. Instead we only take + // the first directory of the path, to have at least an idea how each plugin performs: + normalizePath, + promClient: { collectDefaultMetrics: {} }, + }); +} diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 7e7f6ae400..3470110408 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -13,22 +13,55 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { useHotCleanup } from '@backstage/backend-common'; +import { + PluginDatabaseManager, + useHotCleanup, +} from '@backstage/backend-common'; +import { Config } from '@backstage/config'; +import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; import { createRouter } from '@backstage/plugin-search-backend'; +import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; import { IndexBuilder, LunrSearchEngine, + SearchEngine, } from '@backstage/plugin-search-backend-node'; -import { PluginEnvironment } from '../types'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { Logger } from 'winston'; +import { PluginEnvironment } from '../types'; + +async function createSearchEngine({ + logger, + database, + config, +}: { + logger: Logger; + database: PluginDatabaseManager; + config: Config; +}): Promise { + if (config.has('search.elasticsearch')) { + return await ElasticSearchSearchEngine.fromConfig({ + logger, + config, + }); + } + + if (await PgSearchEngine.supported(database)) { + return await PgSearchEngine.from({ database }); + } + + return new LunrSearchEngine({ logger }); +} export default async function createPlugin({ logger, discovery, + config, + database, }: PluginEnvironment) { // Initialize a connection to a search engine. - const searchEngine = new LunrSearchEngine({ logger }); + const searchEngine = await createSearchEngine({ config, logger, database }); const indexBuilder = new IndexBuilder({ logger, searchEngine }); // Collators are responsible for gathering documents known to plugins. This diff --git a/packages/catalog-client/CHANGELOG.md b/packages/catalog-client/CHANGELOG.md index d417f74f96..d04d1dc936 100644 --- a/packages/catalog-client/CHANGELOG.md +++ b/packages/catalog-client/CHANGELOG.md @@ -1,5 +1,11 @@ # @backstage/catalog-client +## 0.3.18 + +### Patch Changes + +- 11c370af2: Support filtering entities via property existence + ## 0.3.17 ### Patch Changes diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index c356deed4f..173af9aef3 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -25,6 +25,11 @@ export type AddLocationResponse = { entities: Entity[]; }; +// Warning: (ae-missing-release-tag) "CATALOG_FILTER_EXISTS" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const CATALOG_FILTER_EXISTS: unique symbol; + // Warning: (ae-missing-release-tag) "CatalogApi" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -123,8 +128,8 @@ export class CatalogClient implements CatalogApi { // @public (undocumented) export type CatalogEntitiesRequest = { filter?: - | Record[] - | Record + | Record[] + | Record | undefined; fields?: string[] | undefined; }; diff --git a/packages/catalog-client/package.json b/packages/catalog-client/package.json index a114871a6e..cdbee32305 100644 --- a/packages/catalog-client/package.json +++ b/packages/catalog-client/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/catalog-client", - "version": "0.3.17", + "version": "0.3.18", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -35,7 +35,7 @@ "cross-fetch": "^3.0.6" }, "devDependencies": { - "@backstage/cli": "^0.7.5", + "@backstage/cli": "^0.7.6", "@types/jest": "^26.0.7", "msw": "^0.29.0" }, diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 4e3d95f99f..d2db98bb25 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { CatalogListResponse } from './types/api'; +import { CATALOG_FILTER_EXISTS, CatalogListResponse } from './types/api'; import { DiscoveryApi } from './types/discovery'; const server = setupServer(); @@ -83,7 +83,7 @@ describe('CatalogClient', () => { server.use( rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { expect(req.url.search).toBe( - '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2', + '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c', ); return res(ctx.json([])); }), @@ -100,6 +100,9 @@ describe('CatalogClient', () => { { a: '2', }, + { + c: CATALOG_FILTER_EXISTS, + }, ], }, { token }, @@ -113,7 +116,7 @@ describe('CatalogClient', () => { server.use( rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => { - expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D'); + expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D,c'); return res(ctx.json([])); }), ); @@ -124,6 +127,7 @@ describe('CatalogClient', () => { a: '1', b: ['2', '3'], ö: '=', + c: CATALOG_FILTER_EXISTS, }, }, { token }, diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 5dfcddde0c..ef90287c37 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -26,6 +26,7 @@ import { import { ResponseError } from '@backstage/errors'; import fetch from 'cross-fetch'; import { + CATALOG_FILTER_EXISTS, AddLocationRequest, AddLocationResponse, CatalogApi, @@ -69,9 +70,13 @@ export class CatalogClient implements CatalogApi { const filterParts: string[] = []; for (const [key, value] of Object.entries(filterItem)) { for (const v of [value].flat()) { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); + if (v === CATALOG_FILTER_EXISTS) { + filterParts.push(encodeURIComponent(key)); + } else if (typeof v === 'string') { + filterParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, + ); + } } } diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index ae3fd7b514..d30caa50d6 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -16,10 +16,12 @@ import { Entity, EntityName, Location } from '@backstage/catalog-model'; +export const CATALOG_FILTER_EXISTS = Symbol('CATALOG_FILTER_EXISTS'); + export type CatalogEntitiesRequest = { filter?: - | Record[] - | Record + | Record[] + | Record | undefined; fields?: string[] | undefined; }; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 6ec17fbae1..3bfcdd7fba 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -22,4 +22,5 @@ export type { CatalogListResponse, CatalogRequestOptions, } from './api'; +export { CATALOG_FILTER_EXISTS } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 437a0be02b..b9f46add6a 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -190,7 +190,7 @@ export type EntityEnvelope = { // // @public export function entityEnvelopeSchemaValidator< - T extends EntityEnvelope = EntityEnvelope + T extends EntityEnvelope = EntityEnvelope, >(schema?: unknown): (data: unknown) => T; // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen @@ -230,6 +230,7 @@ export type EntityMeta = JsonObject & { generation?: number; name: string; namespace?: string; + title?: string; description?: string; labels?: Record; annotations?: Record; @@ -333,9 +334,7 @@ export function getEntityName(entity: Entity): EntityName; // Warning: (ae-missing-release-tag) "getEntitySourceLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export function getEntitySourceLocation( - entity: Entity, -): { +export function getEntitySourceLocation(entity: Entity): { type: string; target: string; }; @@ -542,9 +541,7 @@ export function parseEntityRef( // Warning: (ae-missing-release-tag) "parseLocationReference" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export function parseLocationReference( - ref: string, -): { +export function parseLocationReference(ref: string): { type: string; target: string; }; @@ -726,10 +723,6 @@ export interface TemplateEntityV1beta2 extends Entity { // (undocumented) kind: 'Template'; // (undocumented) - metadata: EntityMeta & { - title?: string; - }; - // (undocumented) spec: { type: string; parameters?: JsonObject | JsonObject[]; diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index 8843f10f5f..681ac43b80 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -103,7 +103,13 @@ export type EntityMeta = JsonObject & { * The name of the entity. * * Must be unique within the catalog at any given point in time, for any - * given namespace + kind pair. + * given namespace + kind pair. This value is part of the technical + * identifier of the entity, and as such it will appear in URLs, database + * tables, entity references, and similar. It is subject to restrictions + * regarding what characters are allowed. + * + * If you want to use a different, more human readable string with fewer + * restrictions on it in user interfaces, see the `title` field below. */ name: string; @@ -112,6 +118,23 @@ export type EntityMeta = JsonObject & { */ namespace?: string; + /** + * A display name of the entity, to be presented in user interfaces instead + * of the `name` property above, when available. + * + * This field is sometimes useful when the `name` is cumbersome or ends up + * being perceived as overly technical. The title generally does not have + * as stringent format requirements on it, so it may contain special + * characters and be more explanatory. Do keep it very short though, and + * avoid situations where a title can be confused with the name of another + * entity, or where two entities share a title. + * + * Note that this is only for display purposes, and may be ignored by some + * parts of the code. Entity references still always make use of the `name` + * property, not the title. + */ + title?: string; + /** * A short (typically relatively few words, on one line) description of the * entity. diff --git a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts index 21539aaba4..c263689446 100644 --- a/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts +++ b/packages/catalog-model/src/entity/policies/SchemaValidEntityPolicy.test.ts @@ -58,7 +58,7 @@ describe('SchemaValidEntityPolicy', () => { // it('rejects wrong root type', async () => { - await expect(policy.enforce((7 as unknown) as Entity)).rejects.toThrow( + await expect(policy.enforce(7 as unknown as Entity)).rejects.toThrow( /object/, ); }); diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index 91d8547c6b..e2d9c5b203 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -18,9 +18,7 @@ import { EntityName, EntityRef } from '../types'; import { ENTITY_DEFAULT_NAMESPACE } from './constants'; import { Entity } from './Entity'; -function parseRefString( - ref: string, -): { +function parseRefString(ref: string): { kind?: string; namespace?: string; name: string; diff --git a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts index 737037e6b2..2bcd9c7040 100644 --- a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts @@ -30,6 +30,5 @@ export interface ApiEntityV1alpha1 extends Entity { }; } -export const apiEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const apiEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts index 4e08d4cce9..5312d87a35 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts @@ -33,6 +33,5 @@ export interface ComponentEntityV1alpha1 extends Entity { }; } -export const componentEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const componentEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts b/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts index c2f39de321..c832f35ee0 100644 --- a/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/DomainEntityV1alpha1.ts @@ -26,6 +26,5 @@ export interface DomainEntityV1alpha1 extends Entity { }; } -export const domainEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const domainEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts index 28f1503223..263ba311ad 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts @@ -34,6 +34,5 @@ export interface GroupEntityV1alpha1 extends Entity { }; } -export const groupEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const groupEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index 4a37340bee..44f04b2c39 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -28,6 +28,5 @@ export interface LocationEntityV1alpha1 extends Entity { }; } -export const locationEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const locationEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts index c8f96b87e3..211164f4c1 100644 --- a/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ResourceEntityV1alpha1.ts @@ -29,6 +29,5 @@ export interface ResourceEntityV1alpha1 extends Entity { }; } -export const resourceEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const resourceEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts b/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts index 7c719566ad..129e3fea50 100644 --- a/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/SystemEntityV1alpha1.ts @@ -27,6 +27,5 @@ export interface SystemEntityV1alpha1 extends Entity { }; } -export const systemEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const systemEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts index 2710fe1275..1d7aa2162c 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta2.ts @@ -15,16 +15,13 @@ */ import { JsonObject } from '@backstage/config'; -import type { Entity, EntityMeta } from '../entity/Entity'; +import type { Entity } from '../entity/Entity'; import schema from '../schema/kinds/Template.v1beta2.schema.json'; import { ajvCompiledJsonSchemaValidator } from './util'; export interface TemplateEntityV1beta2 extends Entity { apiVersion: 'backstage.io/v1beta2'; kind: 'Template'; - metadata: EntityMeta & { - title?: string; - }; spec: { type: string; parameters?: JsonObject | JsonObject[]; @@ -40,6 +37,5 @@ export interface TemplateEntityV1beta2 extends Entity { }; } -export const templateEntityV1beta2Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const templateEntityV1beta2Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts index 267a9f07fd..65ebdb3262 100644 --- a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts @@ -31,6 +31,5 @@ export interface UserEntityV1alpha1 extends Entity { }; } -export const userEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); +export const userEntityV1alpha1Validator = + ajvCompiledJsonSchemaValidator(schema); diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts index 431ee71218..91209e7a2a 100644 --- a/packages/catalog-model/src/location/helpers.ts +++ b/packages/catalog-model/src/location/helpers.ts @@ -26,9 +26,10 @@ import { LOCATION_ANNOTATION, SOURCE_LOCATION_ANNOTATION } from './annotation'; * @param ref A string-form location reference, e.g. 'url:https://host' * @returns A location reference, e.g. { type: 'url', target: 'https://host' } */ -export function parseLocationReference( - ref: string, -): { type: string; target: string } { +export function parseLocationReference(ref: string): { + type: string; + target: string; +} { if (typeof ref !== 'string') { throw new TypeError( `Unable to parse location reference '${ref}', unexpected argument ${typeof ref}`, @@ -91,9 +92,10 @@ export function stringifyLocationReference(ref: { * using the UrlReader from @backstage/backend-common. If it is not of type 'url', the caller * needs to have explicit handling of each location type or signal that it is not supported. */ -export function getEntitySourceLocation( - entity: Entity, -): { type: string; target: string } { +export function getEntitySourceLocation(entity: Entity): { + type: string; + target: string; +} { const locationRef = entity.metadata?.annotations?.[SOURCE_LOCATION_ANNOTATION] ?? entity.metadata?.annotations?.[LOCATION_ANNOTATION]; diff --git a/packages/catalog-model/src/schema/EntityMeta.schema.json b/packages/catalog-model/src/schema/EntityMeta.schema.json index ff0f9c84a8..c834bfb113 100644 --- a/packages/catalog-model/src/schema/EntityMeta.schema.json +++ b/packages/catalog-model/src/schema/EntityMeta.schema.json @@ -53,6 +53,12 @@ "examples": ["default", "admin"], "minLength": 1 }, + "title": { + "type": "string", + "description": "A display name of the entity, to be presented in user interfaces instead of the name property, when available.", + "examples": ["React SSR Template"], + "minLength": 1 + }, "description": { "type": "string", "description": "A short (typically relatively few words, on one line) description of the entity." diff --git a/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json index b14cdb8ee0..d68eac936d 100644 --- a/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json +++ b/packages/catalog-model/src/schema/kinds/Template.v1beta2.schema.json @@ -69,17 +69,6 @@ "kind": { "enum": ["Template"] }, - "metadata": { - "type": "object", - "properties": { - "title": { - "type": "string", - "description": "The nice display name for the template.", - "examples": ["React SSR Template"], - "minLength": 1 - } - } - }, "spec": { "type": "object", "required": ["type", "steps"], diff --git a/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts b/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts index 2fe74ea2a4..aa865bcf1c 100644 --- a/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts +++ b/packages/catalog-model/src/validation/entityEnvelopeSchemaValidator.ts @@ -41,7 +41,7 @@ import { compileAjvSchema, throwAjvError } from './ajv'; * @see https://github.com/backstage/backstage/tree/master/packages/catalog-model/src/schema */ export function entityEnvelopeSchemaValidator< - T extends EntityEnvelope = EntityEnvelope + T extends EntityEnvelope = EntityEnvelope, >(schema?: unknown): (data: unknown) => T { const validate = compileAjvSchema( schema ? (schema as Schema) : entityEnvelopeSchema, diff --git a/packages/catalog-model/src/validation/entitySchemaValidator.test.ts b/packages/catalog-model/src/validation/entitySchemaValidator.test.ts index 3dfe307f92..b486de1df2 100644 --- a/packages/catalog-model/src/validation/entitySchemaValidator.test.ts +++ b/packages/catalog-model/src/validation/entitySchemaValidator.test.ts @@ -30,6 +30,8 @@ describe('entitySchemaValidator', () => { generation: 13, name: 'test', namespace: 'ns', + title: 'My Component, Yay', + description: 'Yeah this is probably the best component so far', labels: { 'backstage.io/custom': 'ValueStuff', }, @@ -192,6 +194,21 @@ describe('entitySchemaValidator', () => { expect(() => validator(entity)).toThrow(/namespace/); }); + it('accepts missing title', () => { + delete entity.metadata.title; + expect(() => validator(entity)).not.toThrow(); + }); + + it('rejects bad title type', () => { + entity.metadata.title = 7; + expect(() => validator(entity)).toThrow(/title/); + }); + + it('rejects empty title', () => { + entity.metadata.title = ''; + expect(() => validator(entity)).toThrow(/title/); + }); + it('accepts missing description', () => { delete entity.metadata.description; expect(() => validator(entity)).not.toThrow(); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index b2877572fe..064438d17e 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,47 @@ # @backstage/cli +## 0.7.9 + +### Patch Changes + +- f3bba3d2b: Remove debug logging +- 8ea1e96b3: Fix file path handling in diff commands on Windows. +- 2518aab58: Compensate for error formatting mismatch between Webpack 5 and react-dev-utils +- 1ac2961c3: Reintroduce Node.js shims that were removed in the Webpack 5 migration. +- 8d07a8b03: Add Buffer to `ProvidePlugin` since this is no longer provided in `webpack@5` +- fe506a0cf: Remove Webpack deprecation message when running build. +- 485438a56: Fix `backstage-cli backend:dev` argument passing +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/config-loader@0.6.7 + +## 0.7.8 + +### Patch Changes + +- c4ef9181a: Migrate to using `webpack@5` 🎉 + +## 0.7.7 + +### Patch Changes + +- 6aa7c3db7: bump node-tar version to the latest +- e9d3983ee: Keep track of filtered configuration values when running frontend in development mode. +- Updated dependencies + - @backstage/config@0.1.6 + - @backstage/config-loader@0.6.6 + +## 0.7.6 + +### Patch Changes + +- 9d40fcb1e: - Bumping `material-ui/core` version to at least `4.12.2` as they made some breaking changes in later versions which broke `Pagination` of the `Table`. + - Switching out `material-table` to `@material-table/core` for support for the later versions of `material-ui/core` + - This causes a minor API change to `@backstage/core-components` as the interface for `Table` re-exports the `prop` from the underlying `Table` components. + - `onChangeRowsPerPage` has been renamed to `onRowsPerPageChange` + - `onChangePage` has been renamed to `onPageChange` + - Migration guide is here: https://material-table-core.com/docs/breaking-changes + ## 0.7.5 ### Patch Changes diff --git a/packages/cli/asset-types/asset-types.d.ts b/packages/cli/asset-types/asset-types.d.ts index 879e9b0b05..0bd0922004 100644 --- a/packages/cli/asset-types/asset-types.d.ts +++ b/packages/cli/asset-types/asset-types.d.ts @@ -68,6 +68,26 @@ declare module '*.svg' { export default src; } +declare module '*.eot' { + const src: string; + export default src; +} + +declare module '*.woff' { + const src: string; + export default src; +} + +declare module '*.woff2' { + const src: string; + export default src; +} + +declare module '*.ttf' { + const src: string; + export default src; +} + declare module '*.css' { const classes: { readonly [key: string]: string }; export default classes; diff --git a/packages/cli/package.json b/packages/cli/package.json index 1adaeaf725..09054cd351 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/cli", "description": "CLI for developing Backstage plugins and apps", - "version": "0.7.5", + "version": "0.7.9", "private": false, "publishConfig": { "access": "public" @@ -31,8 +31,8 @@ "@babel/core": "^7.4.4", "@babel/plugin-transform-modules-commonjs": "^7.4.4", "@backstage/cli-common": "^0.1.2", - "@backstage/config": "^0.1.5", - "@backstage/config-loader": "^0.6.5", + "@backstage/config": "^0.1.8", + "@backstage/config-loader": "^0.6.7", "@hot-loader/react-dom": "^16.13.0", "@lerna/package-graph": "^4.0.0", "@lerna/project": "^4.0.0", @@ -50,14 +50,13 @@ "@svgr/plugin-svgo": "5.4.x", "@svgr/rollup": "5.5.x", "@svgr/webpack": "5.5.x", - "@types/start-server-webpack-plugin": "^2.2.0", "@types/webpack-env": "^1.15.2", - "@types/webpack-node-externals": "^2.5.0", - "@typescript-eslint/eslint-plugin": "^v4.28.3", + "@typescript-eslint/eslint-plugin": "^v4.30.0", "@typescript-eslint/parser": "^v4.28.3", "@yarnpkg/lockfile": "^1.1.0", "babel-plugin-dynamic-import-node": "^2.3.3", "bfj": "^7.0.2", + "buffer": "^6.0.3", "chalk": "^4.0.0", "chokidar": "^3.3.1", "commander": "^6.1.0", @@ -76,18 +75,20 @@ "eslint-plugin-react-hooks": "^4.0.0", "express": "^4.17.1", "file-loader": "^6.2.0", - "fork-ts-checker-webpack-plugin": "^6.2.9", + "fork-ts-checker-webpack-plugin": "^4.0.5", "fs-extra": "9.1.0", "handlebars": "^4.7.3", - "html-webpack-plugin": "^4.3.0", + "html-webpack-plugin": "^5.3.1", "inquirer": "^7.0.4", "jest": "^26.0.1", "jest-css-modules": "^2.1.0", "json-schema": "^0.3.0", "lodash": "^4.17.19", - "mini-css-extract-plugin": "^0.9.0", + "mini-css-extract-plugin": "^1.4.1", + "node-libs-browser": "^2.2.1", "ora": "^5.3.0", "postcss": "^8.1.0", + "process": "^0.11.10", "raw-loader": "^4.0.1", "react": "^16.0.0", "react-dev-utils": "^11.0.4", @@ -100,17 +101,18 @@ "rollup-plugin-peer-deps-external": "^2.2.2", "rollup-plugin-postcss": "^4.0.0", "rollup-pluginutils": "^2.8.2", + "run-script-webpack-plugin": "^0.0.11", "semver": "^7.3.2", - "start-server-webpack-plugin": "^2.2.5", "style-loader": "^1.2.1", "sucrase": "^3.18.2", - "tar": "^6.0.1", - "terser-webpack-plugin": "^1.4.3", + "tar": "^6.1.2", + "terser-webpack-plugin": "^5.1.3", "ts-loader": "^8.0.17", "typescript": "^4.0.3", "url-loader": "^4.1.0", - "webpack": "^4.41.6", - "webpack-dev-server": "3.11.0", + "util": "^0.12.3", + "webpack": "^5.48.0", + "webpack-dev-server": "4.0.0-rc.0", "webpack-node-externals": "^3.0.0", "yaml": "^1.10.0", "yaml-jest": "^1.0.5", @@ -118,36 +120,37 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/backend-common": "^0.8.7", - "@backstage/config": "^0.1.5", - "@backstage/core-components": "^0.1.6", - "@backstage/core-plugin-api": "^0.1.3", - "@backstage/core-app-api": "^0.1.5", - "@backstage/dev-utils": "^0.2.3", - "@backstage/test-utils": "^0.1.15", - "@backstage/theme": "^0.2.8", + "@backstage/backend-common": "^0.9.0", + "@backstage/config": "^0.1.8", + "@backstage/core-components": "^0.3.3", + "@backstage/core-plugin-api": "^0.1.6", + "@backstage/core-app-api": "^0.1.10", + "@backstage/dev-utils": "^0.2.7", + "@backstage/test-utils": "^0.1.17", + "@backstage/theme": "^0.2.10", "@types/diff": "^5.0.0", "@types/express": "^4.17.6", "@types/fs-extra": "^9.0.1", - "@types/html-webpack-plugin": "^3.2.2", "@types/http-proxy": "^1.17.4", "@types/inquirer": "^7.3.1", - "@types/mini-css-extract-plugin": "^1.2.2", "@types/mock-fs": "^4.13.0", "@types/node": "^14.14.32", - "@types/react-dev-utils": "^9.0.4", "@types/recursive-readdir": "^2.2.0", "@types/rollup-plugin-peer-deps-external": "^2.2.0", "@types/rollup-plugin-postcss": "^2.0.0", "@types/tar": "^4.0.3", - "@types/webpack": "^4.41.7", - "@types/webpack-dev-server": "^3.11.0", + "@types/terser-webpack-plugin": "^5.0.4", + "@types/webpack": "^5.28.0", + "@types/webpack-dev-server": "^3.11.5", "@types/yarnpkg__lockfile": "^1.1.4", "del": "^6.0.0", "mock-fs": "^4.13.0", "nodemon": "^2.0.2", "ts-node": "^10.0.0" }, + "resolutions": { + "@types/webpack-dev-server/@types/webpack": "^5.28.0" + }, "files": [ "asset-types", "templates", diff --git a/packages/cli/src/commands/app/serve.ts b/packages/cli/src/commands/app/serve.ts index 85fbdd97a0..208042c719 100644 --- a/packages/cli/src/commands/app/serve.ts +++ b/packages/cli/src/commands/app/serve.ts @@ -59,6 +59,7 @@ export default async (cmd: Command) => { ...(await loadCliConfig({ args: cmd.config, fromPackage: name, + withFilteredKeys: true, })), }); diff --git a/packages/cli/src/commands/plugin/serve.ts b/packages/cli/src/commands/plugin/serve.ts index b9ba93bc62..a0200b7ce8 100644 --- a/packages/cli/src/commands/plugin/serve.ts +++ b/packages/cli/src/commands/plugin/serve.ts @@ -28,6 +28,7 @@ export default async (cmd: Command) => { ...(await loadCliConfig({ args: cmd.config, fromPackage: name, + withFilteredKeys: true, })), }); diff --git a/packages/cli/src/commands/versions/lint.ts b/packages/cli/src/commands/versions/lint.ts index 68d0766217..c542da6dfb 100644 --- a/packages/cli/src/commands/versions/lint.ts +++ b/packages/cli/src/commands/versions/lint.ts @@ -55,11 +55,9 @@ export default async (cmd: Command) => { lockfile.replaceVersions(result.newVersions); await lockfile.save(); } else { - const [ - newVersionsForbidden, - newVersionsAllowed, - ] = partition(result.newVersions, ({ name }) => - forbiddenDuplicatesFilter(name), + const [newVersionsForbidden, newVersionsAllowed] = partition( + result.newVersions, + ({ name }) => forbiddenDuplicatesFilter(name), ); if (newVersionsForbidden.length && !fix) { success = false; diff --git a/packages/cli/src/lib/builder/config.ts b/packages/cli/src/lib/builder/config.ts index 60a5f0efca..3c95a87bdd 100644 --- a/packages/cli/src/lib/builder/config.ts +++ b/packages/cli/src/lib/builder/config.ts @@ -80,7 +80,17 @@ export const makeConfigs = async ( postcss(), forwardFileImports({ exclude: /\.icon\.svg$/, - include: [/\.svg$/, /\.png$/, /\.gif$/, /\.jpg$/, /\.jpeg$/], + include: [ + /\.svg$/, + /\.png$/, + /\.gif$/, + /\.jpg$/, + /\.jpeg$/, + /\.eot$/, + /\.woff$/, + /\.woff2$/, + /\.ttf$/, + ], }), json(), yaml(), diff --git a/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts b/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts index aa6a6b4e2e..91615eddee 100644 --- a/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts +++ b/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts @@ -15,14 +15,14 @@ */ import { resolve as resolvePath } from 'path'; -import { ResolvePlugin } from 'webpack'; +import { WebpackPluginInstance } from 'webpack'; import { isChildPath } from '@backstage/cli-common'; import { LernaPackage } from './types'; // Enables proper resolution of packages when linking in external packages. // Without this the packages would depend on dependencies in the node_modules // of the external packages themselves, leading to module duplication -export class LinkedPackageResolvePlugin implements ResolvePlugin { +export class LinkedPackageResolvePlugin implements WebpackPluginInstance { constructor( private readonly targetModules: string, private readonly packages: LernaPackage[], diff --git a/packages/cli/src/lib/bundler/backend.ts b/packages/cli/src/lib/bundler/backend.ts index bddeac0de3..bd4ab42098 100644 --- a/packages/cli/src/lib/bundler/backend.ts +++ b/packages/cli/src/lib/bundler/backend.ts @@ -26,23 +26,16 @@ export async function serveBackend(options: BackendServeOptions) { isDev: true, }); - const compiler = webpack(config); - - const watcher = compiler.watch( - { - poll: true, - }, - (err: Error) => { - if (err) { - console.error(err); - } else console.log('Build succeeded'); - }, - ); + const compiler = webpack(config, (err: Error | undefined) => { + if (err) { + console.error(err); + } else console.log('Build succeeded'); + }); const waitForExit = async () => { for (const signal of ['SIGINT', 'SIGTERM'] as const) { process.on(signal, () => { - watcher.close(() => console.log('Stopped watcher')); + compiler.close(() => console.log('Stopped watcher')); // exit instead of resolve. The process is shutting down and resolving a promise here logs an error process.exit(); }); diff --git a/packages/cli/src/lib/bundler/bundle.ts b/packages/cli/src/lib/bundler/bundle.ts index 17a7398b2a..2f23d5cae6 100644 --- a/packages/cli/src/lib/bundler/bundle.ts +++ b/packages/cli/src/lib/bundler/bundle.ts @@ -69,6 +69,10 @@ export async function buildBundle(options: BuildOptions) { throw new Error(`Failed to compile.\n${error.message || error}`); }); + if (!stats) { + throw new Error('No stats returned'); + } + if (statsJsonEnabled) { // No @types/bfj await require('bfj').write( @@ -87,29 +91,49 @@ export async function buildBundle(options: BuildOptions) { } async function build(compiler: webpack.Compiler, isCi: boolean) { - const stats = await new Promise((resolve, reject) => { - compiler.run((err, buildStats) => { - if (err) { - if (err.message) { - const { errors } = formatWebpackMessages({ - errors: [err.message], - warnings: new Array(), - } as webpack.Stats.ToJsonOutput); + const stats = await new Promise( + (resolve, reject) => { + compiler.run((err, buildStats) => { + if (err) { + if (err.message) { + const { errors } = formatWebpackMessages({ + errors: [err.message], + warnings: new Array(), + _showErrors: true, + _showWarnings: true, + }); - throw new Error(errors[0]); + throw new Error(errors[0]); + } else { + reject(err); + } } else { - reject(err); + resolve(buildStats); } - } else { - resolve(buildStats); - } - }); - }); - - const { errors, warnings } = formatWebpackMessages( - stats.toJson({ all: false, warnings: true, errors: true }), + }); + }, ); + if (!stats) { + throw new Error('No stats provided'); + } + + const serializedStats = stats.toJson({ + all: false, + warnings: true, + errors: true, + }); + // NOTE(freben): The code below that extracts the message part of the errors, + // is due to react-dev-utils not yet being compatible with webpack 5. This + // may be possible to remove (just passing the serialized stats object + // directly into the format function) after a new release of react-dev-utils + // has been made available. + // See https://github.com/facebook/create-react-app/issues/9880 + const { errors, warnings } = formatWebpackMessages({ + errors: serializedStats.errors?.map(e => (e.message ? e.message : e)), + warnings: serializedStats.warnings?.map(e => (e.message ? e.message : e)), + }); + if (errors.length) { // Only keep the first error. Others are often indicative // of the same problem, but confuse the reader with noise. diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 75e94af716..6e3f445f4a 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -19,8 +19,8 @@ import { resolve as resolvePath } from 'path'; import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin'; import HtmlWebpackPlugin from 'html-webpack-plugin'; import ModuleScopePlugin from 'react-dev-utils/ModuleScopePlugin'; -import StartServerPlugin from 'start-server-webpack-plugin'; -import webpack from 'webpack'; +import { RunScriptWebpackPlugin } from 'run-script-webpack-plugin'; +import webpack, { ProvidePlugin } from 'webpack'; import nodeExternals from 'webpack-node-externals'; import { isChildPath } from '@backstage/cli-common'; import { optimization } from './optimization'; @@ -32,6 +32,7 @@ import { BundlingOptions, BackendBundlingOptions, LernaPackage } from './types'; import { version } from '../../lib/version'; import { paths as cliPaths } from '../../lib/paths'; import { runPlain } from '../run'; +import pickBy from 'lodash/pickBy'; export function resolveBaseUrl(config: Config): URL { const baseUrl = config.getString('app.baseUrl'); @@ -98,10 +99,9 @@ export async function createConfig( if (checksEnabled) { plugins.push( new ForkTsCheckerWebpackPlugin({ - typescript: { - configFile: paths.targetTsConfig, - }, - eslint: { + typescript: paths.targetTsConfig, + eslint: true, + eslintOptions: { files: ['**', '!**/__tests__/**', '!**/?(*.)(spec|test).*'], options: { parserOptions: { @@ -114,6 +114,16 @@ export async function createConfig( ); } + // TODO(blam): process is no longer auto polyfilled by webpack in v5. + // we use the provide plugin to provide this polyfill, but lets look + // to remove this eventually! + plugins.push( + new ProvidePlugin({ + process: 'process/browser', + Buffer: ['buffer', 'Buffer'], + }), + ); + plugins.push( new webpack.EnvironmentPlugin({ APP_CONFIG: options.frontendAppConfigs, @@ -156,27 +166,34 @@ export async function createConfig( return { mode: isDev ? 'development' : 'production', profile: false, - node: { - module: 'empty', - dgram: 'empty', - dns: 'mock', - fs: 'empty', - http2: 'empty', - net: 'empty', - tls: 'empty', - child_process: 'empty', - }, optimization: optimization(options), bail: false, performance: { hints: false, // we check the gzip size instead }, - devtool: isDev ? 'cheap-module-eval-source-map' : 'source-map', + devtool: isDev ? 'eval-cheap-module-source-map' : 'source-map', context: paths.targetPath, entry: [require.resolve('react-hot-loader/patch'), paths.targetEntry], resolve: { extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx'], mainFields: ['browser', 'module', 'main'], + fallback: { + ...pickBy(require('node-libs-browser')), + module: false, + dgram: false, + dns: false, + fs: false, + http2: false, + net: false, + tls: false, + child_process: false, + + /* new ignores */ + path: false, + https: false, + http: false, + util: require.resolve('util/'), + }, plugins: [ new LinkedPackageResolvePlugin(paths.rootNodeModules, externalPkgs), new ModuleScopePlugin( @@ -194,13 +211,13 @@ export async function createConfig( output: { path: paths.targetDist, publicPath: validBaseUrl.pathname, - filename: isDev ? '[name].js' : 'static/[name].[hash:8].js', + filename: isDev ? '[name].js' : 'static/[name].[fullhash:8].js', chunkFilename: isDev ? '[name].chunk.js' : 'static/[name].[chunkhash:8].chunk.js', ...(isDev ? { - devtoolModuleFilenameTemplate: info => + devtoolModuleFilenameTemplate: (info: any) => `file:///${resolvePath(info.absoluteResourcePath).replace( /\\/g, '/', @@ -237,7 +254,7 @@ export async function createBackendConfig( ? { watch: true, watchOptions: { - ignored: [/node_modules\/(?!\@backstage)/], + ignored: /node_modules\/(?!\@backstage)/, }, } : {}), @@ -259,7 +276,7 @@ export async function createBackendConfig( performance: { hints: false, // we check the gzip size instead }, - devtool: isDev ? 'cheap-module-eval-source-map' : 'source-map', + devtool: isDev ? 'eval-cheap-module-source-map' : 'source-map', context: paths.targetPath, entry: [ 'webpack/hot/poll?100', @@ -291,7 +308,7 @@ export async function createBackendConfig( : '[name].[chunkhash:8].chunk.js', ...(isDev ? { - devtoolModuleFilenameTemplate: info => + devtoolModuleFilenameTemplate: (info: any) => `file:///${resolvePath(info.absoluteResourcePath).replace( /\\/g, '/', @@ -300,18 +317,18 @@ export async function createBackendConfig( : {}), }, plugins: [ - new StartServerPlugin({ + new RunScriptWebpackPlugin({ name: 'main.js', nodeArgs: options.inspectEnabled ? ['--inspect'] : undefined, + args: process.argv.slice(3), // drop `node backstage-cli backend:dev` }), new webpack.HotModuleReplacementPlugin(), ...(checksEnabled ? [ new ForkTsCheckerWebpackPlugin({ - typescript: { - configFile: paths.targetTsConfig, - }, - eslint: { + typescript: paths.targetTsConfig, + eslint: true, + eslintOptions: { files: ['**', '!**/__tests__/**', '!**/?(*.)(spec|test).*'], options: { parserOptions: { @@ -347,11 +364,10 @@ function nodeExternalsWithResolve( }); return ( - context: string, - request: string, - callback: webpack.ExternalsFunctionCallback, + { context, request }: { context?: string; request?: string }, + callback: any, ) => { - currentContext = context; + currentContext = context!; return externals(context, request, callback); }; } diff --git a/packages/cli/src/lib/bundler/optimization.ts b/packages/cli/src/lib/bundler/optimization.ts index e97b2ae868..b8164d263d 100644 --- a/packages/cli/src/lib/bundler/optimization.ts +++ b/packages/cli/src/lib/bundler/optimization.ts @@ -14,14 +14,14 @@ * limitations under the License. */ -import { Options } from 'webpack'; +import { WebpackOptionsNormalized, WebpackPluginInstance } from 'webpack'; import TerserPlugin from 'terser-webpack-plugin'; import { BundlingOptions } from './types'; import { isParallelDefault } from '../parallel'; export const optimization = ( options: BundlingOptions, -): Options.Optimization => { +): WebpackOptionsNormalized['optimization'] => { const { isDev } = options; return { @@ -32,7 +32,7 @@ export const optimization = ( minimizer: [ new TerserPlugin({ parallel: options.parallel, - }), + }) as unknown as WebpackPluginInstance, ], } : {}), diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts index 088128c805..2f8e3125c1 100644 --- a/packages/cli/src/lib/bundler/server.ts +++ b/packages/cli/src/lib/bundler/server.ts @@ -42,25 +42,31 @@ export async function serveBundle(options: ServeOptions) { }); const compiler = webpack(config); - const server = new WebpackDevServer(compiler, { - hot: !process.env.CI, - contentBase: paths.targetPublic, - contentBasePublicPath: config.output?.publicPath, - publicPath: config.output?.publicPath, - historyApiFallback: { - // Paths with dots should still use the history fallback. - // See https://github.com/facebookincubator/create-react-app/issues/387. - disableDotRule: true, - }, - clientLogLevel: 'warning', - stats: 'errors-warnings', - https: url.protocol === 'https:', - host, - port, - proxy: pkg.proxy, - // When the dev server is behind a proxy, the host and public hostname differ - allowedHosts: [url.hostname], - }); + const server = new WebpackDevServer( + compiler as any, + { + hot: !process.env.CI, + devMiddleware: { + publicPath: config.output?.publicPath as string, + stats: 'errors-warnings', + }, + static: { + publicPath: config.output?.publicPath as string, + directory: paths.targetPublic ?? '/', + }, + historyApiFallback: { + // Paths with dots should still use the history fallback. + // See https://github.com/facebookincubator/create-react-app/issues/387. + disableDotRule: true, + }, + https: url.protocol === 'https:', + host, + port, + proxy: pkg.proxy, + // When the dev server is behind a proxy, the host and public hostname differ + allowedHosts: [url.hostname], + } as any, + ); await new Promise((resolve, reject) => { server.listen(port, host, (err?: Error) => { diff --git a/packages/cli/src/lib/bundler/transforms.ts b/packages/cli/src/lib/bundler/transforms.ts index 6eb4920326..b0a9b25466 100644 --- a/packages/cli/src/lib/bundler/transforms.ts +++ b/packages/cli/src/lib/bundler/transforms.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -import webpack, { Module, Plugin } from 'webpack'; +import webpack, { ModuleOptions, WebpackPluginInstance } from 'webpack'; import MiniCssExtractPlugin from 'mini-css-extract-plugin'; import { svgrTemplate } from '../svgrTemplate'; type Transforms = { - loaders: Module['rules']; - plugins: Plugin[]; + loaders: ModuleOptions['rules']; + plugins: WebpackPluginInstance[]; }; type TransformOptions = { @@ -51,6 +51,12 @@ export const transforms = (options: TransformOptions): Transforms => { production: !isDev, }, }, + { + test: /\.m?js/, + resolve: { + fullySpecified: false, + }, + }, { test: [/\.icon\.svg$/], use: [ @@ -83,6 +89,17 @@ export const transforms = (options: TransformOptions): Transforms => { name: 'static/[name].[hash:8].[ext]', }, }, + { + test: /\.(eot|woff|woff2|ttf)$/, + use: [ + { + loader: require.resolve('file-loader'), + options: { + name: 'static/[name].[hash:8].[ext]', + }, + }, + ], + }, { test: /\.ya?ml$/, use: require.resolve('yml-loader'), @@ -105,7 +122,7 @@ export const transforms = (options: TransformOptions): Transforms => { }, ]; - const plugins = new Array(); + const plugins = new Array(); if (isDev) { plugins.push(new webpack.HotModuleReplacementPlugin()); diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index db96ca7686..a93f040122 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -22,6 +22,7 @@ type Options = { args: string[]; fromPackage?: string; mockEnv?: boolean; + withFilteredKeys?: boolean; }; export async function loadCliConfig(options: Options) { @@ -57,6 +58,7 @@ export async function loadCliConfig(options: Options) { try { const frontendAppConfigs = schema.process(appConfigs, { visibility: ['frontend'], + withFilteredKeys: options.withFilteredKeys, }); const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs); diff --git a/packages/cli/src/lib/diff/handlers.ts b/packages/cli/src/lib/diff/handlers.ts index 5cbddde0ec..4a17125e5a 100644 --- a/packages/cli/src/lib/diff/handlers.ts +++ b/packages/cli/src/lib/diff/handlers.ts @@ -16,6 +16,7 @@ import chalk from 'chalk'; import { diffLines } from 'diff'; +import { sep, posix } from 'path'; import { FileDiff, PromptFunc, FileHandler, WriteFileFunc } from './types'; function sortObjectKeys(obj: Record) { @@ -281,7 +282,7 @@ export async function handleAllFiles( promptFunc: PromptFunc, ) { for (const file of files) { - const { path } = file; + const path = file.path.split(sep).join(posix.sep); const fileHandler = fileHandlers.find(handler => handler.patterns.some(pattern => typeof pattern === 'string' ? pattern === path : pattern.test(path), diff --git a/packages/cli/src/lib/versioning/Lockfile.ts b/packages/cli/src/lib/versioning/Lockfile.ts index 7b23652c33..d3517e3c83 100644 --- a/packages/cli/src/lib/versioning/Lockfile.ts +++ b/packages/cli/src/lib/versioning/Lockfile.ts @@ -137,9 +137,9 @@ export class Lockfile { } // Find all versions currently in use - const versions = Array.from( - new Set(entries.map(e => e.version)), - ).sort((v1, v2) => semver.rcompare(v1, v2)); + const versions = Array.from(new Set(entries.map(e => e.version))).sort( + (v1, v2) => semver.rcompare(v1, v2), + ); // If we're not using at least 2 different versions we're done if (versions.length < 2) { diff --git a/packages/cli/src/types.d.ts b/packages/cli/src/types.d.ts index 9088a86369..db5e508ec2 100644 --- a/packages/cli/src/types.d.ts +++ b/packages/cli/src/types.d.ts @@ -29,5 +29,230 @@ declare module '@svgr/rollup' { } declare module '@rollup/plugin-yaml'; +declare module 'react-dev-utils/formatWebpackMessages' { + export default function (stats: any): { + errors: string[]; + warnings: string[]; + }; +} -declare module 'terser-webpack-plugin'; +declare module 'react-dev-utils/openBrowser' { + export default function (url: string): boolean; +} + +declare module 'react-dev-utils/ModuleScopePlugin' { + import webpack = require('webpack'); + + export default class ModuleScopePlugin + implements webpack.WebpackPluginInstance + { + constructor( + appSrc: string | ReadonlyArray, + allowedFiles?: ReadonlyArray, + ); + apply: (resolver: any) => void; + } +} + +declare module 'react-dev-utils/FileSizeReporter' { + import webpack = require('webpack'); + + export interface OpaqueFileSizes { + root: string; + sizes: Record; + } + + /** + * Captures JS and CSS asset sizes inside the passed `buildFolder`. Save the + * result value to compare it after the build. + */ + export function measureFileSizesBeforeBuild( + buildFolder: string, + ): Promise; + + /** + * Prints the JS and CSS asset sizes after the build, and includes a size + * comparison with `previousFileSizes` that were captured earlier using + * `measureFileSizesBeforeBuild()`. `maxBundleGzipSize` and + * `maxChunkGzipSizemay` may optionally be specified to display a warning when + * the main bundle or a chunk exceeds the specified size (in bytes). + */ + export function printFileSizesAfterBuild( + webpackStats: webpack.Stats, + previousFileSizes: OpaqueFileSizes, + buildFolder: string, + maxBundleGzipSize?: number, + maxChunkGzipSize?: number, + ): void; +} + +declare module 'mini-css-extract-plugin' { + import webpack = require('webpack'); + + /** + * Lightweight CSS extraction webpack plugin. + * + * This plugin extracts CSS into separate files. It creates a CSS file per JS file which + * contains CSS. It supports On-Demand-Loading of CSS and SourceMaps. + * + * Configuration Detail: https://github.com/webpack-contrib/mini-css-extract-plugin#configuration + */ + export default class MiniCssExtractPlugin { + /** + * Webpack loader always used at the end of loaders list (ie. array index zero). + */ + static loader: string; + + constructor(options?: MiniCssExtractPlugin.PluginOptions); + + /** + * Apply the plugin + */ + apply(compiler: webpack.Compiler): void; + } + + namespace MiniCssExtractPlugin { + interface PluginOptions { + /** + * Works like [`output.filename`](https://webpack.js.org/configuration/output/#outputfilename). + */ + filename?: Required['output']['filename']; + /** + * Works like [`output.chunkFilename`](https://webpack.js.org/configuration/output/#outputchunkfilename). + */ + chunkFilename?: string; + /** + * For projects where CSS ordering has been mitigated through consistent + * use of scoping or naming conventions, the CSS order warnings can be + * disabled by setting this flag to true for the plugin. + */ + ignoreOrder?: boolean; + /** + * Specify where to insert the link tag. + * + * A string value specifies a DOM query for a parent element to attach to. + * + * A function allows to override default behavior for non-entry CSS chunks. + * This code will run in the browser alongside your application. It is recommend + * to only use ECMA 5 features and syntax. The function won't have access to the + * scope of the webpack configuration module. + * + * @default function() { document.head.appendChild(linkTag); } + */ + insert?: string | ((linkTag: any) => void); + /** + * Specify additional html attributes to add to the link tag. + * + * Note: These are only applied to dynamically loaded css chunks. To modify link + * attributes for entry CSS chunks, please use html-webpack-plugin. + */ + attributes?: Record; + /** + * This option allows loading asynchronous chunks with a custom link type, such as + * ``. + * + * `false` disables the link `type` attribute. + * + * @default 'text/css' + */ + linkType?: string | false | 'text/css'; + } + interface LoaderOptions { + /** + * Overrides [`output.publicPath`](https://webpack.js.org/configuration/output/#outputpublicpath). + * @default output.publicPath + */ + publicPath?: string | ((resourcePath: string, context: string) => string); + /** + * If false, the plugin will extract the CSS but **will not** emit the file + * @default true + */ + emit?: boolean; + /** + * By default, `mini-css-extract-plugin` generates JS modules that use the ES modules syntax. + * There are some cases in which using ES modules is beneficial, + * like in the case of module concatenation and tree shaking. + * @default true + */ + esModule?: boolean; + modules?: { + /** + * Enables/disables ES modules named export for locals. + * + * Names of locals are converted to camelCase. It is not allowed to use + * JavaScript reserved words in CSS class names. Options `esModule` and + * `modules.namedExport` in css-loader and MiniCssExtractPlugin.loader + * must be enabled. + * + * @default false + */ + namedExport?: boolean; + }; + } + } +} + +declare module 'fork-ts-checker-webpack-plugin/lib/ForkTsCheckerWebpackPlugin' {} +declare module 'webpack-node-externals' { + export default function webpackNodeExternals( + options?: webpackNodeExternals.Options, + ): any; + + namespace webpackNodeExternals { + type AllowlistOption = string | RegExp | AllowlistFunctionType; + type ImportTypeCallback = (moduleName: string) => string; + /** a function that accepts the module name and returns whether it should be included */ + type AllowlistFunctionType = (moduleName: string) => boolean; + interface ModulesFromFileType { + exclude?: string | string[]; + include?: string | string[]; + } + + interface Options { + /** + * An array for the externals to allow, so they will be included in the bundle. + * Can accept exact strings ('module_name'), regex patterns (/^module_name/), or a + * function that accepts the module name and returns whether it should be included. + * Important - if you have set aliases in your webpack config with the exact + * same names as modules in node_modules, you need to allowlist them so Webpack will know + * they should be bundled. + * @default [] + */ + allowlist?: AllowlistOption[] | AllowlistOption; + /** + * @default ['.bin'] + */ + binaryDirs?: string[]; + /** + * The method in which unbundled modules will be required in the code. Best to leave as + * 'commonjs' for node modules. + * @default 'commonjs' + */ + importType?: + | 'var' + | 'this' + | 'commonjs' + | 'amd' + | 'umd' + | ImportTypeCallback; + /** + * The folder in which to search for the node modules. + * @default 'node_modules' + */ + modulesDir?: string; + /** + * Additional folders to look for node modules. + */ + additionalModuleDirs?: string[]; + /** + * Read the modules from the package.json file instead of the node_modules folder. + * @default false + */ + modulesFromFile?: boolean | ModulesFromFileType; + /** + * @default false + */ + includeAbsolutePaths?: boolean; + } + } +} diff --git a/packages/cli/templates/default-plugin/package.json.hbs b/packages/cli/templates/default-plugin/package.json.hbs index dbba628055..fdac5a0c67 100644 --- a/packages/cli/templates/default-plugin/package.json.hbs +++ b/packages/cli/templates/default-plugin/package.json.hbs @@ -27,7 +27,7 @@ "@backstage/core-components": "^{{version '@backstage/core-components'}}", "@backstage/core-plugin-api": "^{{version '@backstage/core-plugin-api'}}", "@backstage/theme": "^{{version '@backstage/theme'}}", - "@material-ui/core": "^4.11.0", + "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "react": "^16.13.1", diff --git a/packages/codemods/CHANGELOG.md b/packages/codemods/CHANGELOG.md index 0c104d6c33..74a3022bf5 100644 --- a/packages/codemods/CHANGELOG.md +++ b/packages/codemods/CHANGELOG.md @@ -1,5 +1,48 @@ # @backstage/codemods +## 0.1.11 + +### Patch Changes + +- Updated dependencies + - @backstage/core-app-api@0.1.10 + - @backstage/core-components@0.3.3 + +## 0.1.10 + +### Patch Changes + +- Updated dependencies + - @backstage/core-app-api@0.1.9 + - @backstage/core-components@0.3.2 + +## 0.1.9 + +### Patch Changes + +- Updated dependencies + - @backstage/core-app-api@0.1.8 + - @backstage/core-components@0.3.1 + - @backstage/core-plugin-api@0.1.6 + +## 0.1.8 + +### Patch Changes + +- Updated dependencies + - @backstage/core-components@0.3.0 + - @backstage/core-plugin-api@0.1.5 + - @backstage/core-app-api@0.1.7 + +## 0.1.7 + +### Patch Changes + +- Updated dependencies + - @backstage/core-components@0.2.0 + - @backstage/core-app-api@0.1.6 + - @backstage/core-plugin-api@0.1.4 + ## 0.1.6 ### Patch Changes diff --git a/packages/codemods/package.json b/packages/codemods/package.json index 19c71c843c..04bf0ac014 100644 --- a/packages/codemods/package.json +++ b/packages/codemods/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/codemods", "description": "A collection of codemods for Backstage projects", - "version": "0.1.6", + "version": "0.1.11", "private": false, "publishConfig": { "access": "public", diff --git a/packages/config-loader/CHANGELOG.md b/packages/config-loader/CHANGELOG.md index d06568bf47..aee4cf453f 100644 --- a/packages/config-loader/CHANGELOG.md +++ b/packages/config-loader/CHANGELOG.md @@ -1,5 +1,22 @@ # @backstage/config-loader +## 0.6.7 + +### Patch Changes + +- 0ade9d02b: Include `devDependencies` and `optionalDependencies` in the detection of Backstage packages when collecting configuration schema. +- 9b8cec063: Add support for config file watching through a new group of `watch` options to `loadConfig`. +- Updated dependencies + - @backstage/config@0.1.7 + +## 0.6.6 + +### Patch Changes + +- e9d3983ee: Add option to populate the `filteredKeys` property when processing configuration with a schema. +- Updated dependencies + - @backstage/config@0.1.6 + ## 0.6.5 ### Patch Changes diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 37b9880448..38aa0b1ebb 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -37,6 +37,10 @@ export type LoadConfigOptions = { configPaths: string[]; env?: string; experimentalEnvFunc?: EnvFunc; + watch?: { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; + }; }; // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts @@ -59,7 +63,7 @@ export function readEnvConfig(env: { // Warnings were encountered during analysis: // -// src/lib/schema/types.d.ts:77:5 - (ae-forgotten-export) The symbol "ConfigProcessingOptions" needs to be exported by the entry point index.d.ts +// src/lib/schema/types.d.ts:83:5 - (ae-forgotten-export) The symbol "ConfigProcessingOptions" needs to be exported by the entry point index.d.ts // src/loader.d.ts:13:5 - (ae-forgotten-export) The symbol "EnvFunc" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 7eb0a78f3f..f810841d3e 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config-loader", "description": "Config loading functionality used by Backstage backend, and CLI", - "version": "0.6.5", + "version": "0.6.7", "private": false, "publishConfig": { "access": "public", @@ -31,9 +31,10 @@ }, "dependencies": { "@backstage/cli-common": "^0.1.1", - "@backstage/config": "^0.1.5", + "@backstage/config": "^0.1.7", "@types/json-schema": "^7.0.6", "ajv": "^7.0.3", + "chokidar": "^3.5.2", "fs-extra": "9.1.0", "json-schema": "^0.3.0", "json-schema-merge-allof": "^0.8.1", diff --git a/packages/config-loader/src/lib/schema/collect.test.ts b/packages/config-loader/src/lib/schema/collect.test.ts index 479e63c94c..ef9d38faa9 100644 --- a/packages/config-loader/src/lib/schema/collect.test.ts +++ b/packages/config-loader/src/lib/schema/collect.test.ts @@ -87,6 +87,8 @@ describe('collectConfigSchemas', () => { dependencies: { c1: '0.0.0', c2: '0.0.0', + }, + devDependencies: { '@backstage/mock': '0.0.0', }, configSchema: { ...mockSchema, title: 'b' }, diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index b53389d499..b363e8e5f3 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -74,6 +74,8 @@ export async function collectConfigSchemas( const pkg = await fs.readJson(pkgPath); const depNames = [ ...Object.keys(pkg.dependencies ?? {}), + ...Object.keys(pkg.devDependencies ?? {}), + ...Object.keys(pkg.optionalDependencies ?? {}), ...Object.keys(pkg.peerDependencies ?? {}), ]; diff --git a/packages/config-loader/src/lib/schema/filtering.test.ts b/packages/config-loader/src/lib/schema/filtering.test.ts index 5079afa876..d4b3cd55b6 100644 --- a/packages/config-loader/src/lib/schema/filtering.test.ts +++ b/packages/config-loader/src/lib/schema/filtering.test.ts @@ -63,43 +63,115 @@ const visibility = new Map( describe('filterByVisibility', () => { test.each<[ConfigVisibility[], JsonObject]>([ - [[], {}], + [ + [], + { + data: {}, + filteredKeys: [ + 'arr[0]', + 'arr[1]', + 'arr[2]', + 'objArr[0].f', + 'objArr[0].b', + 'objArr[0].s', + 'objArr[1].f', + 'objArr[1].b', + 'objArr[1].s', + 'obj.f', + 'obj.b.s', + 'arrF[0].never', + 'arrB[0].never', + 'arrS[0].never', + 'objF.never', + 'objB.never', + 'objS.never', + ], + }, + ], [ ['frontend'], { - arr: ['f'], - objArr: [{ f: 1 }, { f: 4 }], - obj: { f: 'a' }, - arrF: [], - objF: {}, + data: { + arr: ['f'], + objArr: [{ f: 1 }, { f: 4 }], + obj: { f: 'a' }, + arrF: [], + objF: {}, + }, + filteredKeys: [ + 'arr[1]', + 'arr[2]', + 'objArr[0].b', + 'objArr[0].s', + 'objArr[1].b', + 'objArr[1].s', + 'obj.b.s', + 'arrF[0].never', + 'arrB[0].never', + 'arrS[0].never', + 'objF.never', + 'objB.never', + 'objS.never', + ], }, ], [ ['backend'], { - arr: ['b'], - objArr: [{ b: 2 }, { b: 5 }], - obj: { b: {} }, - arrF: [{ never: 'here' }], - arrB: [{ never: 'here' }], - arrS: [{ never: 'here' }], - objF: { never: 'here' }, - objB: { never: 'here' }, - objS: { never: 'here' }, + data: { + arr: ['b'], + objArr: [{ b: 2 }, { b: 5 }], + obj: { b: {} }, + arrF: [{ never: 'here' }], + arrB: [{ never: 'here' }], + arrS: [{ never: 'here' }], + objF: { never: 'here' }, + objB: { never: 'here' }, + objS: { never: 'here' }, + }, + filteredKeys: [ + 'arr[0]', + 'arr[2]', + 'objArr[0].f', + 'objArr[0].s', + 'objArr[1].f', + 'objArr[1].s', + 'obj.f', + 'obj.b.s', + ], }, ], [ ['secret'], { - arr: ['s'], - objArr: [{ s: 3 }, { s: 6 }], - obj: { b: { s: true } }, - arrS: [], - objS: {}, + data: { + arr: ['s'], + objArr: [{ s: 3 }, { s: 6 }], + obj: { b: { s: true } }, + arrS: [], + objS: {}, + }, + filteredKeys: [ + 'arr[0]', + 'arr[1]', + 'objArr[0].f', + 'objArr[0].b', + 'objArr[1].f', + 'objArr[1].b', + 'obj.f', + 'arrF[0].never', + 'arrB[0].never', + 'arrS[0].never', + 'objF.never', + 'objB.never', + 'objS.never', + ], }, ], - [['frontend', 'backend', 'secret'], data], + [['frontend', 'backend', 'secret'], { data, filteredKeys: [] }], ])('should filter correctly with %p', (filter, expected) => { - expect(filterByVisibility(data, filter, visibility)).toEqual(expected); + expect( + filterByVisibility(data, filter, visibility, undefined, true), + ).toEqual(expected); }); }); diff --git a/packages/config-loader/src/lib/schema/filtering.ts b/packages/config-loader/src/lib/schema/filtering.ts index 74d367a30a..3533be01b7 100644 --- a/packages/config-loader/src/lib/schema/filtering.ts +++ b/packages/config-loader/src/lib/schema/filtering.ts @@ -30,9 +30,17 @@ export function filterByVisibility( includeVisibilities: ConfigVisibility[], visibilityByPath: Map, transformFunc?: TransformFunc, -): JsonObject { - function transform(jsonVal: JsonValue, path: string): JsonValue | undefined { - const visibility = visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY; + withFilteredKeys?: boolean, +): { data: JsonObject; filteredKeys?: string[] } { + const filteredKeys = new Array(); + + function transform( + jsonVal: JsonValue, + visibilityPath: string, // Matches the format we get from ajv + filterPath: string, // Matches the format of the ConfigReader + ): JsonValue | undefined { + const visibility = + visibilityByPath.get(visibilityPath) ?? DEFAULT_CONFIG_VISIBILITY; const isVisible = includeVisibilities.includes(visibility); if (typeof jsonVal !== 'object') { @@ -42,6 +50,9 @@ export function filterByVisibility( } return jsonVal; } + if (withFilteredKeys) { + filteredKeys.push(filterPath); + } return undefined; } else if (jsonVal === null) { return undefined; @@ -49,7 +60,11 @@ export function filterByVisibility( const arr = new Array(); for (const [index, value] of jsonVal.entries()) { - const out = transform(value, `${path}/${index}`); + const out = transform( + value, + `${visibilityPath}/${index}`, + `${filterPath}[${index}]`, + ); if (out !== undefined) { arr.push(out); } @@ -68,7 +83,11 @@ export function filterByVisibility( if (value === undefined) { continue; } - const out = transform(value, `${path}/${key}`); + const out = transform( + value, + `${visibilityPath}/${key}`, + filterPath ? `${filterPath}.${key}` : key, + ); if (out !== undefined) { outObj[key] = out; hasOutput = true; @@ -81,5 +100,8 @@ export function filterByVisibility( return undefined; } - return (transform(data, '') as JsonObject) ?? {}; + return { + filteredKeys: withFilteredKeys ? filteredKeys : undefined, + data: (transform(data, '', '') as JsonObject) ?? {}, + }; } diff --git a/packages/config-loader/src/lib/schema/load.test.ts b/packages/config-loader/src/lib/schema/load.test.ts index 4a2b719590..73fa55b90c 100644 --- a/packages/config-loader/src/lib/schema/load.test.ts +++ b/packages/config-loader/src/lib/schema/load.test.ts @@ -68,13 +68,19 @@ describe('loadConfigSchema', () => { schema.process(configs, { visibility: ['frontend'], valueTransform: () => 'X', + withFilteredKeys: true, }), - ).toEqual([{ data: { key1: 'X' }, context: 'test' }]); + ).toEqual([ + { data: { key1: 'X' }, context: 'test', filteredKeys: ['key2'] }, + ]); expect( schema.process(configs, { valueTransform: () => 'X', + withFilteredKeys: true, }), - ).toEqual([{ data: { key1: 'X', key2: 'X' }, context: 'test' }]); + ).toEqual([ + { data: { key1: 'X', key2: 'X' }, context: 'test', filteredKeys: [] }, + ]); const serialized = schema.serialize(); diff --git a/packages/config-loader/src/lib/schema/load.ts b/packages/config-loader/src/lib/schema/load.ts index ae5823c8cc..b67bd0002f 100644 --- a/packages/config-loader/src/lib/schema/load.ts +++ b/packages/config-loader/src/lib/schema/load.ts @@ -57,7 +57,7 @@ export async function loadConfigSchema( return { process( configs: AppConfig[], - { visibility, valueTransform } = {}, + { visibility, valueTransform, withFilteredKeys } = {}, ): AppConfig[] { const result = validate(configs); if (result.errors) { @@ -73,21 +73,23 @@ export async function loadConfigSchema( if (visibility) { processedConfigs = processedConfigs.map(({ data, context }) => ({ context, - data: filterByVisibility( + ...filterByVisibility( data, visibility, result.visibilityByPath, valueTransform, + withFilteredKeys, ), })); } else if (valueTransform) { processedConfigs = processedConfigs.map(({ data, context }) => ({ context, - data: filterByVisibility( + ...filterByVisibility( data, Array.from(CONFIG_VISIBILITIES), result.visibilityByPath, valueTransform, + withFilteredKeys, ), })); } diff --git a/packages/config-loader/src/lib/schema/types.ts b/packages/config-loader/src/lib/schema/types.ts index 30e47917bc..17090ad53c 100644 --- a/packages/config-loader/src/lib/schema/types.ts +++ b/packages/config-loader/src/lib/schema/types.ts @@ -96,6 +96,13 @@ type ConfigProcessingOptions = { * will be omitted. */ valueTransform?: TransformFunc; + + /** + * Whether or not to include the `filteredKeys` property in the output `AppConfig`s. + * + * Default: `false`. + */ + withFilteredKeys?: boolean; }; /** diff --git a/packages/config-loader/src/lib/transform/include.test.ts b/packages/config-loader/src/lib/transform/include.test.ts index 44bf07ab4d..91df723621 100644 --- a/packages/config-loader/src/lib/transform/include.test.ts +++ b/packages/config-loader/src/lib/transform/include.test.ts @@ -24,9 +24,11 @@ const substituteMe = '${MY_SUBSTITUTION}'; const mySubstitution = 'fooSubstitution'; const env = jest.fn(async (name: string) => { - return ({ - SECRET: 'my-secret', - } as { [name: string]: string })[name]; + return ( + { + SECRET: 'my-secret', + } as { [name: string]: string } + )[name]; }); const substitute: TransformFunc = async value => { @@ -44,14 +46,16 @@ const substitute: TransformFunc = async value => { }; const readFile = jest.fn(async (path: string) => { - const content = ({ - [resolvePath(root, 'my-secret')]: 'secret', - [resolvePath(root, 'my-data.json')]: '{"a":{"b":{"c":42}}}', - [resolvePath(root, 'my-data.yaml')]: 'some:\n yaml:\n key: 7', - [resolvePath(root, 'my-data.yml')]: 'different: { key: hello }', - [resolvePath(root, 'invalid.yaml')]: 'foo: [}', - [resolvePath(root, `${mySubstitution}/my-data.json`)]: '{"foo":"bar"}', - } as { [key: string]: string })[path]; + const content = ( + { + [resolvePath(root, 'my-secret')]: 'secret', + [resolvePath(root, 'my-data.json')]: '{"a":{"b":{"c":42}}}', + [resolvePath(root, 'my-data.yaml')]: 'some:\n yaml:\n key: 7', + [resolvePath(root, 'my-data.yml')]: 'different: { key: hello }', + [resolvePath(root, 'invalid.yaml')]: 'foo: [}', + [resolvePath(root, `${mySubstitution}/my-data.json`)]: '{"foo":"bar"}', + } as { [key: string]: string } + )[path]; if (!content) { throw new Error('File not found!'); diff --git a/packages/config-loader/src/lib/transform/substitution.test.ts b/packages/config-loader/src/lib/transform/substitution.test.ts index dc6ed47597..b03c39c174 100644 --- a/packages/config-loader/src/lib/transform/substitution.test.ts +++ b/packages/config-loader/src/lib/transform/substitution.test.ts @@ -17,10 +17,12 @@ import { createSubstitutionTransform } from './substitution'; const env = jest.fn(async (name: string) => { - return ({ - SECRET: 'my-secret', - TOKEN: 'my-token', - } as { [name: string]: string })[name]; + return ( + { + SECRET: 'my-secret', + TOKEN: 'my-token', + } as { [name: string]: string } + )[name]; }); const substituteTransform = createSubstitutionTransform(env); diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index b0e236f9c9..3f51e76936 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ +import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; +import fs from 'fs-extra'; describe('loadConfig', () => { - beforeAll(() => { + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; @@ -63,7 +65,7 @@ describe('loadConfig', () => { }); }); - afterAll(() => { + afterEach(() => { mockFs.restore(); }); @@ -170,4 +172,81 @@ describe('loadConfig', () => { }, ]); }); + + it('watches config files', async () => { + const onChange = defer(); + const stopSignal = defer(); + + await expect( + loadConfig({ + configRoot: '/root', + configPaths: [], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + await fs.writeJson('/root/app-config.yaml', { + app: { + title: 'New Title', + }, + }); + await expect(onChange.promise).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'New Title', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + + it('stops watching config files', async () => { + const stopSignal = defer(); + + await loadConfig({ + configRoot: '/root', + configPaths: [], + watch: { + onChange: () => { + expect('not').toBe('called'); + }, + stopSignal: stopSignal.promise, + }, + }); + + stopSignal.resolve(); + + await fs.writeJson('/root/app-config.yaml', { + app: { + title: 'New Title', + }, + }); + await new Promise(resolve => setTimeout(resolve, 1000)); + }); + + function defer() { + let resolve: (value: T) => void; + const promise = new Promise(_resolve => { + resolve = _resolve; + }); + return { promise, resolve: resolve! }; + } }); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 3846d6474b..4dbfc007a7 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -16,6 +16,7 @@ import fs from 'fs-extra'; import yaml from 'yaml'; +import chokidar from 'chokidar'; import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; import { AppConfig } from '@backstage/config'; import { @@ -42,13 +43,27 @@ export type LoadConfigOptions = { * @experimental This API is not stable and may change at any point */ experimentalEnvFunc?: EnvFunc; + + /** + * An optional configuration that enables watching of config files. + */ + watch?: { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; + }; }; export async function loadConfig( options: LoadConfigOptions, ): Promise { - const configs = []; - const { configRoot, experimentalEnvFunc: envFunc } = options; + const { configRoot, experimentalEnvFunc: envFunc, watch } = options; const configPaths = options.configPaths.slice(); // If no paths are provided, we default to reading @@ -64,7 +79,9 @@ export async function loadConfig( const env = envFunc ?? (async (name: string) => process.env[name]); - try { + const loadConfigFiles = async () => { + const configs = []; + for (const configPath of configPaths) { if (!isAbsolute(configPath)) { throw new Error(`Config load path is not absolute: '${configPath}'`); @@ -83,13 +100,50 @@ export async function loadConfig( configs.push({ data, context: basename(configPath) }); } + + return configs; + }; + + let fileConfigs; + try { + fileConfigs = await loadConfigFiles(); } catch (error) { throw new Error( `Failed to read static configuration file, ${error.message}`, ); } - configs.push(...readEnvConfig(process.env)); + const envConfigs = await readEnvConfig(process.env); - return configs; + // Set up config file watching if requested by the caller + if (watch) { + let currentSerializedConfig = JSON.stringify(fileConfigs); + + const watcher = chokidar.watch(configPaths, { + usePolling: process.env.NODE_ENV === 'test', + }); + watcher.on('change', async () => { + try { + const newConfigs = await loadConfigFiles(); + const newSerializedConfig = JSON.stringify(newConfigs); + + if (currentSerializedConfig === newSerializedConfig) { + return; + } + currentSerializedConfig = newSerializedConfig; + + watch.onChange([...newConfigs, ...envConfigs]); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + }); + + if (watch.stopSignal) { + watch.stopSignal.then(() => { + watcher.close(); + }); + } + } + + return [...fileConfigs, ...envConfigs]; } diff --git a/packages/config/CHANGELOG.md b/packages/config/CHANGELOG.md index 5cababf94e..742ae86a57 100644 --- a/packages/config/CHANGELOG.md +++ b/packages/config/CHANGELOG.md @@ -1,5 +1,23 @@ # @backstage/config +## 0.1.8 + +### Patch Changes + +- 47113f1f1: Only warn once per key when trying to read visibility-filtered values + +## 0.1.7 + +### Patch Changes + +- 90f25476a: Extended the `Config` interface to have an optional `subscribe` method that can be used be notified of updates to the configuration. + +## 0.1.6 + +### Patch Changes + +- e9d3983ee: Add warning when trying to access configuration values that have been filtered out by visibility. + ## 0.1.5 ### Patch Changes diff --git a/packages/config/api-report.md b/packages/config/api-report.md index 50d2af0f1f..91f37894c0 100644 --- a/packages/config/api-report.md +++ b/packages/config/api-report.md @@ -9,12 +9,16 @@ export type AppConfig = { context: string; data: JsonObject; + filteredKeys?: string[]; }; // Warning: (ae-missing-release-tag) "Config" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export type Config = { + subscribe?(onChange: () => void): { + unsubscribe: () => void; + }; has(key: string): boolean; keys(): string[]; get(key?: string): T; diff --git a/packages/config/package.json b/packages/config/package.json index 400e4548cd..6adee62ca3 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/config", "description": "Config API used by Backstage core, backend, and CLI", - "version": "0.1.5", + "version": "0.1.8", "private": false, "publishConfig": { "access": "public", diff --git a/packages/config/src/reader.test.ts b/packages/config/src/reader.test.ts index b1ba536797..7f3513abf6 100644 --- a/packages/config/src/reader.test.ts +++ b/packages/config/src/reader.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { withLogCollector } from '../../test-utils-core/src'; import { ConfigReader } from './reader'; const DATA = { @@ -63,9 +64,9 @@ function expectValidValues(config: ConfigReader) { strings: ['string1', 'string2'], }); expect(config.getConfig('nested').getString('string')).toBe('string'); - expect( - config.getOptionalConfig('nested')!.getStringArray('strings'), - ).toEqual(['string1', 'string2']); + expect(config.getOptionalConfig('nested')!.getStringArray('strings')).toEqual( + ['string1', 'string2'], + ); expect(config.getOptional('missing')).toBe(undefined); expect(config.getOptionalConfig('missing')).toBe(undefined); expect(config.getOptionalConfigArray('missing')).toBe(undefined); @@ -185,6 +186,85 @@ describe('ConfigReader', () => { const config = new ConfigReader(DATA, CTX); expectInvalidValues(config); }); + + it('should warn when accessing filtered keys in development mode', () => { + const oldEnv = process.env.NODE_ENV; + (process.env as any).NODE_ENV = 'development'; + + const config = ConfigReader.fromConfigs([ + { + data: DATA, + context: CTX, + filteredKeys: ['a', 'a2', 'b[0]'], + }, + ]); + + expect(withLogCollector(() => config.getOptional('a'))).toMatchObject({ + warn: [ + "Failed to read configuration value at 'a' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", + ], + }); + expect( + withLogCollector(() => config.getOptionalString('a2')), + ).toMatchObject({ + warn: [ + "Failed to read configuration value at 'a2' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", + ], + }); + expect( + withLogCollector(() => config.getOptionalConfigArray('b')), + ).toMatchObject({ + warn: [ + "Failed to read configuration array at 'b' as it does not have any visible elements. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", + ], + }); + + (process.env as any).NODE_ENV = oldEnv; + }); + + it('only warns once when accessing filtered keys in development mode', () => { + const oldEnv = process.env.NODE_ENV; + (process.env as any).NODE_ENV = 'development'; + + const config = ConfigReader.fromConfigs([ + { + data: DATA, + context: CTX, + filteredKeys: ['a'], + }, + ]); + + expect(withLogCollector(() => config.getOptional('a'))).toMatchObject({ + warn: [ + "Failed to read configuration value at 'a' as it is not visible. See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.", + ], + }); + expect(withLogCollector(() => config.getOptional('a'))).toMatchObject({ + warn: [], + }); + + (process.env as any).NODE_ENV = oldEnv; + }); + + it('should not warn when accessing filtered keys outside of development mode', () => { + const config = ConfigReader.fromConfigs([ + { + data: DATA, + context: CTX, + filteredKeys: ['a', 'b[0]'], + }, + ]); + + expect(withLogCollector(() => config.getOptional('a'))).toMatchObject({ + warn: [], + }); + expect(withLogCollector(() => config.getOptionalString('a'))).toMatchObject( + { warn: [] }, + ); + expect( + withLogCollector(() => config.getOptionalConfigArray('b')), + ).toMatchObject({ warn: [] }); + }); }); describe('ConfigReader with fallback', () => { diff --git a/packages/config/src/reader.ts b/packages/config/src/reader.ts index 11346d4a50..c6f69e5b29 100644 --- a/packages/config/src/reader.ts +++ b/packages/config/src/reader.ts @@ -55,6 +55,16 @@ const errors = { }; export class ConfigReader implements Config { + /** + * A set of key paths that where removed from the config due to not being visible. + * + * This was added as a mutable private member to avoid changes to the public API. + * Its only purpose of this is to warn users of missing visibility when running + * the frontend in development mode. + */ + private filteredKeys?: string[]; + private notifiedFilteredKeys = new Set(); + static fromConfigs(configs: AppConfig[]): ConfigReader { if (configs.length === 0) { return new ConfigReader(undefined); @@ -62,9 +72,14 @@ export class ConfigReader implements Config { // Merge together all configs into a single config with recursive fallback // readers, giving the first config object in the array the lowest priority. - return configs.reduce((previousReader, { data, context }) => { - return new ConfigReader(data, context, previousReader); - }, undefined!); + return configs.reduce( + (previousReader, { data, context, filteredKeys }) => { + const reader = new ConfigReader(data, context, previousReader); + reader.filteredKeys = filteredKeys; + return reader; + }, + undefined!, + ); } constructor( @@ -101,6 +116,22 @@ export class ConfigReader implements Config { const fallbackValue = this.fallback?.getOptional(key); if (value === undefined) { + if (process.env.NODE_ENV === 'development') { + if (fallbackValue === undefined && key) { + const fullKey = this.fullKey(key); + if ( + this.filteredKeys?.includes(fullKey) && + !this.notifiedFilteredKeys.has(fullKey) + ) { + this.notifiedFilteredKeys.add(fullKey); + // eslint-disable-next-line no-console + console.warn( + `Failed to read configuration value at '${fullKey}' as it is not visible. ` + + 'See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.', + ); + } + } + } return fallbackValue; } else if (fallbackValue === undefined) { return value as T; @@ -127,10 +158,9 @@ export class ConfigReader implements Config { getOptionalConfig(key: string): ConfigReader | undefined { const value = this.readValue(key); const fallbackConfig = this.fallback?.getOptionalConfig(key); - const prefix = this.fullKey(key); if (isObject(value)) { - return new ConfigReader(value, this.context, fallbackConfig, prefix); + return this.copy(value, key, fallbackConfig); } if (value !== undefined) { throw new TypeError( @@ -163,18 +193,24 @@ export class ConfigReader implements Config { }); if (!configs) { + if (process.env.NODE_ENV === 'development') { + const fullKey = this.fullKey(key); + if ( + this.filteredKeys?.some(k => k.startsWith(fullKey)) && + !this.notifiedFilteredKeys.has(key) + ) { + this.notifiedFilteredKeys.add(key); + // eslint-disable-next-line no-console + console.warn( + `Failed to read configuration array at '${key}' as it does not have any visible elements. ` + + 'See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.', + ); + } + } return undefined; } - return configs.map( - (obj, index) => - new ConfigReader( - obj, - this.context, - undefined, - this.fullKey(`${key}[${index}]`), - ), - ); + return configs.map((obj, index) => this.copy(obj, `${key}[${index}]`)); } getNumber(key: string): number { @@ -261,6 +297,17 @@ export class ConfigReader implements Config { return `${this.prefix}${this.prefix ? '.' : ''}${key}`; } + private copy(data: JsonObject, key: string, fallback?: ConfigReader) { + const reader = new ConfigReader( + data, + this.context, + fallback, + this.fullKey(key), + ); + reader.filteredKeys = this.filteredKeys; + return reader; + } + private readConfigValue( key: string, validate: ( @@ -270,6 +317,21 @@ export class ConfigReader implements Config { const value = this.readValue(key); if (value === undefined) { + if (process.env.NODE_ENV === 'development') { + const fullKey = this.fullKey(key); + if ( + this.filteredKeys?.includes(fullKey) && + !this.notifiedFilteredKeys.has(fullKey) + ) { + this.notifiedFilteredKeys.add(fullKey); + // eslint-disable-next-line no-console + console.warn( + `Failed to read configuration value at '${fullKey}' as it is not visible. ` + + 'See https://backstage.io/docs/conf/defining#visibility for instructions on how to make it visible.', + ); + } + } + return this.fallback?.readConfigValue(key, validate); } const result = validate(value); diff --git a/packages/config/src/types.ts b/packages/config/src/types.ts index d8f12f4f6e..84de5dae70 100644 --- a/packages/config/src/types.ts +++ b/packages/config/src/types.ts @@ -22,9 +22,21 @@ export type JsonValue = JsonObject | JsonArray | JsonPrimitive; export type AppConfig = { context: string; data: JsonObject; + filteredKeys?: string[]; }; export type Config = { + /** + * Subscribes to the configuration object in order to receive a notification + * whenever any value within the configuration has changed. + * + * This method is optional to implement, and consumers need to check if it is + * implemented before invoking it. + */ + subscribe?(onChange: () => void): { + unsubscribe: () => void; + }; + has(key: string): boolean; keys(): string[]; diff --git a/packages/core-app-api/CHANGELOG.md b/packages/core-app-api/CHANGELOG.md index 8e188a6577..db5158a697 100644 --- a/packages/core-app-api/CHANGELOG.md +++ b/packages/core-app-api/CHANGELOG.md @@ -1,5 +1,64 @@ # @backstage/core-app-api +## 0.1.10 + +### Patch Changes + +- cfcb486aa: Add system icons for the built-in entity types and use them in the entity list of the `catalog-import` plugin. +- 392b36fa1: Added support for using authenticating via GitHub Apps in addition to GitHub OAuth Apps. It used to be possible to use GitHub Apps, but they did not handle session refresh correctly. + + Note that GitHub Apps handle OAuth scope at the app installation level, meaning that the `scope` parameter for `getAccessToken` has no effect. When calling `getAccessToken` in open source plugins, one should still include the appropriate scope, but also document in the plugin README what scopes are required in the case of GitHub Apps. + + In addition, the `authHandler` and `signInResolver` options have been implemented for the GitHub provider in the auth backend. + +- Updated dependencies + - @backstage/core-components@0.3.3 + - @backstage/config@0.1.8 + +## 0.1.9 + +### Patch Changes + +- 72a31c29a: Add support for additional app origins +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/core-components@0.3.2 + - @backstage/theme@0.2.10 + +## 0.1.8 + +### Patch Changes + +- 362657623: Add support for serving the app with a base path other than `/`, which is enabled by including the path in `app.baseUrl`. +- 56c773909: Switched `@types/react` dependency to request `*` rather than a specific version. +- Updated dependencies + - @backstage/core-components@0.3.1 + - @backstage/core-plugin-api@0.1.6 + +## 0.1.7 + +### Patch Changes + +- Updated dependencies + - @backstage/core-components@0.3.0 + - @backstage/config@0.1.6 + - @backstage/core-plugin-api@0.1.5 + +## 0.1.6 + +### Patch Changes + +- 9d40fcb1e: - Bumping `material-ui/core` version to at least `4.12.2` as they made some breaking changes in later versions which broke `Pagination` of the `Table`. + - Switching out `material-table` to `@material-table/core` for support for the later versions of `material-ui/core` + - This causes a minor API change to `@backstage/core-components` as the interface for `Table` re-exports the `prop` from the underlying `Table` components. + - `onChangeRowsPerPage` has been renamed to `onRowsPerPageChange` + - `onChangePage` has been renamed to `onPageChange` + - Migration guide is here: https://material-table-core.com/docs/breaking-changes +- Updated dependencies + - @backstage/core-components@0.2.0 + - @backstage/core-plugin-api@0.1.4 + - @backstage/theme@0.2.9 + ## 0.1.5 ### Patch Changes diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index a7a3845dd7..20d8a5a770 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -68,9 +68,7 @@ export class AlertApiForwarder implements AlertApi { // // @public (undocumented) export type ApiFactoryHolder = { - get( - api: ApiRef, - ): + get(api: ApiRef): | ApiFactory< T, T, @@ -86,9 +84,7 @@ export type ApiFactoryHolder = { // @public export class ApiFactoryRegistry implements ApiFactoryHolder { // (undocumented) - get( - api: ApiRef, - ): + get(api: ApiRef): | ApiFactory< T, T, @@ -105,7 +101,7 @@ export class ApiFactoryRegistry implements ApiFactoryHolder { Impl extends Api, Deps extends { [name in string]: unknown; - } + }, >(scope: ApiFactoryScope, factory: ApiFactory): boolean; } @@ -211,7 +207,7 @@ export type AppOptions = { export type AppRouteBinder = < ExternalRoutes extends { [name: string]: ExternalRouteRef; - } + }, >( externalRoutes: ExternalRoutes, targetRoutes: PartialKeys< @@ -398,7 +394,7 @@ export type GithubSession = { providerInfo: { accessToken: string; scopes: Set; - expiresAt: Date; + expiresAt?: Date; }; profile: ProfileInfo; backstageIdentity: BackstageIdentity; @@ -469,7 +465,8 @@ export class OAuth2 OpenIdConnectApi, ProfileInfoApi, BackstageIdentityApi, - SessionApi { + SessionApi +{ // Warning: (ae-forgotten-export) The symbol "Options" needs to be exported by the entry point index.d.ts constructor(options: Options); // Warning: (ae-forgotten-export) The symbol "CreateOptions" needs to be exported by the entry point index.d.ts @@ -561,7 +558,8 @@ export class OneLoginAuth { // // @public (undocumented) export class SamlAuth - implements ProfileInfoApi, BackstageIdentityApi, SessionApi { + implements ProfileInfoApi, BackstageIdentityApi, SessionApi +{ // Warning: (ae-forgotten-export) The symbol "SamlSession" needs to be exported by the entry point index.d.ts constructor(sessionManager: SessionManager); // Warning: (ae-forgotten-export) The symbol "AuthApiCreateOptions" needs to be exported by the entry point index.d.ts diff --git a/packages/core-app-api/package.json b/packages/core-app-api/package.json index 41e7c52393..90c962be3e 100644 --- a/packages/core-app-api/package.json +++ b/packages/core-app-api/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core-app-api", "description": "Core app API used by Backstage apps", - "version": "0.1.5", + "version": "0.1.10", "private": false, "publishConfig": { "access": "public", @@ -29,13 +29,13 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/core-components": "^0.1.6", - "@backstage/config": "^0.1.3", - "@backstage/core-plugin-api": "^0.1.3", - "@backstage/theme": "^0.2.8", - "@material-ui/core": "^4.11.0", + "@backstage/core-components": "^0.3.3", + "@backstage/config": "^0.1.8", + "@backstage/core-plugin-api": "^0.1.6", + "@backstage/theme": "^0.2.10", + "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", - "@types/react": "^16.9", + "@types/react": "*", "@types/prop-types": "^15.7.3", "prop-types": "^15.7.2", "react": "^16.12.0", @@ -44,9 +44,9 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/cli": "^0.7.5", - "@backstage/test-utils": "^0.1.15", - "@backstage/test-utils-core": "^0.1.1", + "@backstage/cli": "^0.7.9", + "@backstage/test-utils": "^0.1.17", + "@backstage/test-utils-core": "^0.1.2", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^11.2.5", "@testing-library/react-hooks": "^3.4.2", diff --git a/packages/core-app-api/src/apis/implementations/FeatureFlagsApi/LocalStorageFeatureFlags.test.tsx b/packages/core-app-api/src/apis/implementations/FeatureFlagsApi/LocalStorageFeatureFlags.test.tsx index afc25b2e8c..1193592e96 100644 --- a/packages/core-app-api/src/apis/implementations/FeatureFlagsApi/LocalStorageFeatureFlags.test.tsx +++ b/packages/core-app-api/src/apis/implementations/FeatureFlagsApi/LocalStorageFeatureFlags.test.tsx @@ -194,8 +194,7 @@ describe('FeatureFlags', () => { it('throws an error if length is greater than 150 characters', () => { expect(() => featureFlags.registerFlag({ - name: - 'loremipsumdolorsitametconsecteturadipiscingelitnuncvitaeportaexaullamcorperturpismaurisutmattisnequemorbisediaculisauguevivamuspulvinarcursuseratblandithendreritquisqueuttinciduntmagnavestibulumblanditaugueat', + name: 'loremipsumdolorsitametconsecteturadipiscingelitnuncvitaeportaexaullamcorperturpismaurisutmattisnequemorbisediaculisauguevivamuspulvinarcursuseratblandithendreritquisqueuttinciduntmagnavestibulumblanditaugueat', pluginId: 'plugin-three', }), ).toThrow(/not exceed 150 characters/i); diff --git a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts index 21ebc5699a..85e3e79ad5 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts @@ -29,15 +29,17 @@ import { import { SessionManager } from '../../../../lib/AuthSessionManager/types'; import { AuthSessionStore, + RefreshingAuthSessionManager, StaticAuthSessionManager, } from '../../../../lib/AuthSessionManager'; import { OAuthApiCreateOptions } from '../types'; +import { OptionalRefreshSessionManagerMux } from '../../../../lib/AuthSessionManager/OptionalRefreshSessionManagerMux'; export type GithubAuthResponse = { providerInfo: { accessToken: string; scope: string; - expiresInSeconds: number; + expiresInSeconds?: number; }; profile: ProfileInfo; backstageIdentity: BackstageIdentity; @@ -68,27 +70,46 @@ class GithubAuth implements OAuthApi, SessionApi { providerInfo: { accessToken: res.providerInfo.accessToken, scopes: GithubAuth.normalizeScope(res.providerInfo.scope), - expiresAt: new Date( - Date.now() + res.providerInfo.expiresInSeconds * 1000, - ), + expiresAt: res.providerInfo.expiresInSeconds + ? new Date(Date.now() + res.providerInfo.expiresInSeconds * 1000) + : undefined, }, }; }, }); - const sessionManager = new StaticAuthSessionManager({ + const refreshingSessionManager = new RefreshingAuthSessionManager({ connector, defaultScopes: new Set(defaultScopes), sessionScopes: (session: GithubSession) => session.providerInfo.scopes, + sessionShouldRefresh: (session: GithubSession) => { + const { expiresAt } = session.providerInfo; + if (!expiresAt) { + return false; + } + const expiresInSec = (expiresAt.getTime() - Date.now()) / 1000; + return expiresInSec < 60 * 5; + }, }); - const authSessionStore = new AuthSessionStore({ - manager: sessionManager, + const staticSessionManager = new AuthSessionStore({ + manager: new StaticAuthSessionManager({ + connector, + defaultScopes: new Set(defaultScopes), + sessionScopes: (session: GithubSession) => session.providerInfo.scopes, + }), storageKey: `${provider.id}Session`, sessionScopes: (session: GithubSession) => session.providerInfo.scopes, }); - return new GithubAuth(authSessionStore); + const sessionManagerMux = new OptionalRefreshSessionManagerMux({ + refreshingSessionManager, + staticSessionManager, + sessionCanRefresh: session => + session.providerInfo.expiresAt !== undefined, + }); + + return new GithubAuth(sessionManagerMux); } constructor(private readonly sessionManager: SessionManager) {} diff --git a/packages/core-app-api/src/apis/implementations/auth/github/types.ts b/packages/core-app-api/src/apis/implementations/auth/github/types.ts index 88df25b49d..f5dae3a064 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/types.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/types.ts @@ -20,7 +20,7 @@ export type GithubSession = { providerInfo: { accessToken: string; scopes: Set; - expiresAt: Date; + expiresAt?: Date; }; profile: ProfileInfo; backstageIdentity: BackstageIdentity; diff --git a/packages/core-app-api/src/apis/implementations/auth/oauth2/OAuth2.ts b/packages/core-app-api/src/apis/implementations/auth/oauth2/OAuth2.ts index 8c27e0aa63..24db8d2560 100644 --- a/packages/core-app-api/src/apis/implementations/auth/oauth2/OAuth2.ts +++ b/packages/core-app-api/src/apis/implementations/auth/oauth2/OAuth2.ts @@ -65,7 +65,8 @@ class OAuth2 OpenIdConnectApi, ProfileInfoApi, BackstageIdentityApi, - SessionApi { + SessionApi +{ static create({ discoveryApi, environment = 'development', diff --git a/packages/core-app-api/src/apis/system/ApiProvider.test.tsx b/packages/core-app-api/src/apis/system/ApiProvider.test.tsx index 5afb39a2ec..b94aea6b67 100644 --- a/packages/core-app-api/src/apis/system/ApiProvider.test.tsx +++ b/packages/core-app-api/src/apis/system/ApiProvider.test.tsx @@ -185,9 +185,10 @@ describe('ApiProvider', () => { }); describe('v1 consumer', () => { - const ApiContext = getGlobalSingleton< - Context> - >('api-context'); + const ApiContext = + getGlobalSingleton>>( + 'api-context', + ); function useMockApiV1(apiRef: ApiRef): T { const impl = useContext(ApiContext)?.atVersion(1)?.get(apiRef); diff --git a/packages/core-app-api/src/app/App.tsx b/packages/core-app-api/src/app/App.tsx index a95554709a..c8bd8fec8f 100644 --- a/packages/core-app-api/src/app/App.tsx +++ b/packages/core-app-api/src/app/App.tsx @@ -107,6 +107,20 @@ export function generateBoundRoutes(bindRoutes: AppOptions['bindRoutes']) { return result; } +/** + * Get the app base path from the configured app baseUrl. + * + * The returned path does not have a trailing slash. + */ +function getBasePath(configApi: Config) { + let { pathname } = new URL( + configApi.getOptionalString('app.baseUrl') ?? '/', + 'http://dummy.dev', // baseUrl can be specified as just a path + ); + pathname = pathname.replace(/\/*$/, ''); + return pathname; +} + type FullAppOptions = { apis: Iterable; icons: NonNullable; @@ -216,37 +230,33 @@ export class PrivateAppImpl implements BackstageApp { [], ); - const { - routePaths, - routeParents, - routeObjects, - featureFlags, - } = useMemo(() => { - const result = traverseElementTree({ - root: children, - discoverers: [childDiscoverer, routeElementDiscoverer], - collectors: { - routePaths: routePathCollector, - routeParents: routeParentCollector, - routeObjects: routeObjectCollector, - collectedPlugins: pluginCollector, - featureFlags: featureFlagCollector, - }, - }); + const { routePaths, routeParents, routeObjects, featureFlags } = + useMemo(() => { + const result = traverseElementTree({ + root: children, + discoverers: [childDiscoverer, routeElementDiscoverer], + collectors: { + routePaths: routePathCollector, + routeParents: routeParentCollector, + routeObjects: routeObjectCollector, + collectedPlugins: pluginCollector, + featureFlags: featureFlagCollector, + }, + }); - validateRoutes(result.routePaths, result.routeParents); + validateRoutes(result.routePaths, result.routeParents); - // TODO(Rugvip): Restructure the public API so that we can get an immediate view of - // the app, rather than having to wait for the provider to render. - // For now we need to push the additional plugins we find during - // collection and then make sure we initialize things afterwards. - result.collectedPlugins.forEach(plugin => this.plugins.add(plugin)); - this.verifyPlugins(this.plugins); + // TODO(Rugvip): Restructure the public API so that we can get an immediate view of + // the app, rather than having to wait for the provider to render. + // For now we need to push the additional plugins we find during + // collection and then make sure we initialize things afterwards. + result.collectedPlugins.forEach(plugin => this.plugins.add(plugin)); + this.verifyPlugins(this.plugins); - // Initialize APIs once all plugins are available - this.getApiHolder(); - return result; - }, [children]); + // Initialize APIs once all plugins are available + this.getApiHolder(); + return result; + }, [children]); const loadedConfig = useConfigLoader( this.configLoader, @@ -302,6 +312,7 @@ export class PrivateAppImpl implements BackstageApp { routeParents={routeParents} routeObjects={routeObjects} routeBindings={generateBoundRoutes(this.bindRoutes)} + basePath={getBasePath(loadedConfig.api)} > {children} @@ -314,10 +325,8 @@ export class PrivateAppImpl implements BackstageApp { } getRouter(): ComponentType<{}> { - const { - Router: RouterComponent, - SignInPage: SignInPageComponent, - } = this.components; + const { Router: RouterComponent, SignInPage: SignInPageComponent } = + this.components; // This wraps the sign-in page and waits for sign-in to be completed before rendering the app const SignInPageWrapper = ({ @@ -339,14 +348,7 @@ export class PrivateAppImpl implements BackstageApp { const AppRouter = ({ children }: PropsWithChildren<{}>) => { const configApi = useApi(configApiRef); - - let { pathname } = new URL( - configApi.getOptionalString('app.baseUrl') ?? '/', - 'http://dummy.dev', // baseUrl can be specified as just a path - ); - if (pathname.endsWith('/')) { - pathname = pathname.replace(/\/$/, ''); - } + const mountPath = `${getBasePath(configApi)}/*`; // If the app hasn't configured a sign-in page, we just continue as guest. if (!SignInPageComponent) { @@ -361,7 +363,7 @@ export class PrivateAppImpl implements BackstageApp { return ( - {children}} /> + {children}} /> ); @@ -371,7 +373,7 @@ export class PrivateAppImpl implements BackstageApp { - {children}} /> + {children}} /> diff --git a/packages/core-app-api/src/app/AppContext.test.tsx b/packages/core-app-api/src/app/AppContext.test.tsx index d87bf7d228..6ce164b9f0 100644 --- a/packages/core-app-api/src/app/AppContext.test.tsx +++ b/packages/core-app-api/src/app/AppContext.test.tsx @@ -22,9 +22,10 @@ import { AppContext as AppContextV1 } from './types'; import { AppContextProvider } from './AppContext'; describe('v1 consumer', () => { - const AppContext = getGlobalSingleton< - Context> - >('app-context'); + const AppContext = + getGlobalSingleton>>( + 'app-context', + ); function useMockAppV1(): AppContextV1 { const impl = useContext(AppContext)?.atVersion(1); diff --git a/packages/core-app-api/src/app/createApp.tsx b/packages/core-app-api/src/app/createApp.tsx index b7c49939e2..1d74f93add 100644 --- a/packages/core-app-api/src/app/createApp.tsx +++ b/packages/core-app-api/src/app/createApp.tsx @@ -60,7 +60,7 @@ export const defaultConfigLoader: AppConfigLoader = async ( if (!Array.isArray(appConfig)) { throw new Error('Static configuration has invalid format'); } - const configs = (appConfig.slice() as unknown) as AppConfig[]; + const configs = appConfig.slice() as unknown as AppConfig[]; // Avoiding this string also being replaced at runtime if ( diff --git a/packages/core-app-api/src/app/icons.tsx b/packages/core-app-api/src/app/icons.tsx index bf45155096..a102026b29 100644 --- a/packages/core-app-api/src/app/icons.tsx +++ b/packages/core-app-api/src/app/icons.tsx @@ -15,17 +15,22 @@ */ import { IconComponent } from '@backstage/core-plugin-api'; -import MuiMenuBookIcon from '@material-ui/icons/MenuBook'; +import MuiApartmentIcon from '@material-ui/icons/Apartment'; import MuiBrokenImageIcon from '@material-ui/icons/BrokenImage'; +import MuiCategoryIcon from '@material-ui/icons/Category'; import MuiChatIcon from '@material-ui/icons/Chat'; import MuiDashboardIcon from '@material-ui/icons/Dashboard'; +import MuiDocsIcon from '@material-ui/icons/Description'; import MuiEmailIcon from '@material-ui/icons/Email'; +import MuiExtensionIcon from '@material-ui/icons/Extension'; import MuiGitHubIcon from '@material-ui/icons/GitHub'; import MuiHelpIcon from '@material-ui/icons/Help'; +import MuiLocationOnIcon from '@material-ui/icons/LocationOn'; +import MuiMemoryIcon from '@material-ui/icons/Memory'; +import MuiMenuBookIcon from '@material-ui/icons/MenuBook'; import MuiPeopleIcon from '@material-ui/icons/People'; import MuiPersonIcon from '@material-ui/icons/Person'; import MuiWarningIcon from '@material-ui/icons/Warning'; -import MuiDocsIcon from '@material-ui/icons/Description'; type AppIconsKey = | 'brokenImage' @@ -37,6 +42,13 @@ type AppIconsKey = | 'github' | 'group' | 'help' + | 'kind:api' + | 'kind:component' + | 'kind:domain' + | 'kind:group' + | 'kind:location' + | 'kind:system' + | 'kind:user' | 'user' | 'warning'; @@ -53,6 +65,13 @@ export const defaultAppIcons: AppIcons = { github: MuiGitHubIcon, group: MuiPeopleIcon, help: MuiHelpIcon, + 'kind:api': MuiExtensionIcon, + 'kind:component': MuiMemoryIcon, + 'kind:domain': MuiApartmentIcon, + 'kind:group': MuiPeopleIcon, + 'kind:location': MuiLocationOnIcon, + 'kind:system': MuiCategoryIcon, + 'kind:user': MuiPersonIcon, user: MuiPersonIcon, warning: MuiWarningIcon, }; diff --git a/packages/core-app-api/src/app/types.ts b/packages/core-app-api/src/app/types.ts index 836f1b7fb0..30ce45abe9 100644 --- a/packages/core-app-api/src/app/types.ts +++ b/packages/core-app-api/src/app/types.ts @@ -105,14 +105,14 @@ type KeysWithType = { */ type PartialKeys< Map extends { [name in string]: any }, - Keys extends keyof Map + Keys extends keyof Map, > = Partial> & Required>; /** * Creates a map of target routes with matching parameters based on a map of external routes. */ type TargetRouteMap< - ExternalRoutes extends { [name: string]: ExternalRouteRef } + ExternalRoutes extends { [name: string]: ExternalRouteRef }, > = { [name in keyof ExternalRoutes]: ExternalRoutes[name] extends ExternalRouteRef< infer Params, @@ -123,7 +123,7 @@ type TargetRouteMap< }; export type AppRouteBinder = < - ExternalRoutes extends { [name: string]: ExternalRouteRef } + ExternalRoutes extends { [name: string]: ExternalRouteRef }, >( externalRoutes: ExternalRoutes, targetRoutes: PartialKeys< diff --git a/packages/core-app-api/src/extensions/extensions.tsx b/packages/core-app-api/src/extensions/extensions.tsx index 063f34c659..199430c619 100644 --- a/packages/core-app-api/src/extensions/extensions.tsx +++ b/packages/core-app-api/src/extensions/extensions.tsx @@ -28,7 +28,7 @@ const ERROR_MESSAGE = 'Import this from @backstage/core-plugin-api'; /** @deprecated Import from @backstage/core-plugin-api instead */ export function createRoutableExtension< - T extends (props: any) => JSX.Element | null + T extends (props: any) => JSX.Element | null, >(_options: { component: () => Promise; mountPoint: RouteRef; @@ -38,14 +38,14 @@ export function createRoutableExtension< /** @deprecated Import from @backstage/core-plugin-api instead */ export function createComponentExtension< - T extends (props: any) => JSX.Element | null + T extends (props: any) => JSX.Element | null, >(_options: { component: ComponentLoader }): Extension { throw new Error(ERROR_MESSAGE); } /** @deprecated Import from @backstage/core-plugin-api instead */ export function createReactExtension< - T extends (props: any) => JSX.Element | null + T extends (props: any) => JSX.Element | null, >(_options: { component: ComponentLoader; data?: Record; diff --git a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts index 524a0c5709..28125f6fa8 100644 --- a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts +++ b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.test.ts @@ -126,8 +126,7 @@ describe('DefaultAuthConnector', () => { expect(popupSpy).toBeCalledTimes(1); expect(popupSpy.mock.calls[0][0]).toMatchObject({ - url: - 'http://my-host/api/auth/my-provider/start?scope=a%20b&env=production', + url: 'http://my-host/api/auth/my-provider/start?scope=a%20b&origin=http%3A%2F%2Flocalhost&env=production', }); await expect(sessionPromise).resolves.toEqual({ @@ -175,8 +174,7 @@ describe('DefaultAuthConnector', () => { expect(popupSpy).toBeCalledTimes(1); expect(popupSpy.mock.calls[0][0]).toMatchObject({ - url: - 'http://my-host/api/auth/my-provider/start?scope=-ab-&env=production', + url: 'http://my-host/api/auth/my-provider/start?scope=-ab-&origin=http%3A%2F%2Flocalhost&env=production', }); }); }); diff --git a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts index 81da04010c..261a008d4b 100644 --- a/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts +++ b/packages/core-app-api/src/lib/AuthConnector/DefaultAuthConnector.ts @@ -61,7 +61,8 @@ function defaultJoinScopes(scopes: Set) { * via the OAuthRequestApi. */ export class DefaultAuthConnector - implements AuthConnector { + implements AuthConnector +{ private readonly discoveryApi: DiscoveryApi; private readonly environment: string; private readonly provider: AuthProvider & { id: string }; @@ -151,7 +152,10 @@ export class DefaultAuthConnector private async showPopup(scopes: Set): Promise { const scope = this.joinScopesFunc(scopes); - const popupUrl = await this.buildUrl('/start', { scope }); + const popupUrl = await this.buildUrl('/start', { + scope, + origin: location.origin, + }); const payload = await showLoginPopup({ url: popupUrl, diff --git a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts index 4ceadd51ba..ded26c59e5 100644 --- a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts +++ b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.test.ts @@ -128,6 +128,19 @@ describe('GheAuth AuthSessionStore', () => { expect(manager.removeSession).toHaveBeenCalled(); }); + it('should set session', async () => { + const manager = new MockManager(); + const store = new AuthSessionStore({ manager, ...defaultOptions }); + + await expect(store.getSession({ optional: true })).resolves.toBe(undefined); + expect(localStorage.getItem('my-key')).toBe(null); + expect(manager.setSession).not.toHaveBeenCalled(); + store.setSession('123'); + expect(manager.setSession).toHaveBeenCalled(); + expect(localStorage.getItem('my-key')).toBe('"123"'); + await expect(store.getSession({ optional: true })).resolves.toBe('123'); + }); + it('should forward sessionState calls', () => { const manager = new MockManager(); const store = new AuthSessionStore({ manager, ...defaultOptions }); diff --git a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts index 057a70e58d..fc15da7af8 100644 --- a/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts +++ b/packages/core-app-api/src/lib/AuthSessionManager/AuthSessionStore.ts @@ -15,7 +15,6 @@ */ import { - SessionManager, MutableSessionManager, SessionScopesFunc, SessionShouldRefreshFunc, @@ -40,7 +39,7 @@ type Options = { * * Session is serialized to JSON with special support for following types: Set. */ -export class AuthSessionStore implements SessionManager { +export class AuthSessionStore implements MutableSessionManager { private readonly manager: MutableSessionManager; private readonly storageKey: string; private readonly sessionShouldRefreshFunc: SessionShouldRefreshFunc; @@ -63,6 +62,11 @@ export class AuthSessionStore implements SessionManager { }); } + setSession(session: T | undefined): void { + this.manager.setSession(session); + this.saveSession(session); + } + async getSession(options: GetSessionOptions): Promise { const { scopes } = options; const session = this.loadSession(); diff --git a/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.test.ts b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.test.ts new file mode 100644 index 0000000000..919948f531 --- /dev/null +++ b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.test.ts @@ -0,0 +1,138 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable, SessionState } from '@backstage/core-plugin-api'; +import { OptionalRefreshSessionManagerMux } from './OptionalRefreshSessionManagerMux'; +import { MutableSessionManager, SessionManager } from './types'; + +class MockManager implements MutableSessionManager { + constructor(public session?: string) {} + + setSession(session: string | undefined): void { + this.session = session; + } + async getSession(): Promise { + return this.session; + } + async removeSession(): Promise { + delete this.session; + } + sessionState$(): Observable { + throw new Error('Method not implemented.'); + } +} + +function trackState(manager: SessionManager) { + const states = new Array(); + manager + .sessionState$() + .subscribe(state => states.push(state === SessionState.SignedIn)); + return states; +} + +describe('OptionalRefreshSessionManagerMux', () => { + it('finds no session', async () => { + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager: new MockManager(), + refreshingSessionManager: new MockManager(), + sessionCanRefresh: () => false, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe(undefined); + expect(states).toEqual([false]); + }); + + it('prioritizes a static session', async () => { + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager: new MockManager('static'), + refreshingSessionManager: new MockManager('refreshing'), + sessionCanRefresh: () => false, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('static'); + expect(states).toEqual([false, true]); + }); + + it('transfers a refreshing session to the static manager', async () => { + const staticSessionManager = new MockManager(); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => false, + }); + + const states = trackState(mux); + expect(staticSessionManager.session).toBeUndefined(); + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBe('refreshing'); + expect(states).toEqual([false, true]); + }); + + it('relies on the refreshing manager if refresh is available', async () => { + const staticSessionManager = new MockManager(); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => true, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBeUndefined(); + expect(states).toEqual([false, true]); + }); + + it('can switch between refreshing and static sessions', async () => { + let canRefresh = true; + const staticSessionManager = new MockManager(); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => canRefresh, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBeUndefined(); + canRefresh = false; + await expect(mux.getSession({})).resolves.toBe('refreshing'); + expect(staticSessionManager.session).toBe('refreshing'); + + expect(states).toEqual([false, true]); + }); + + it('removes sessions from both managers', async () => { + const staticSessionManager = new MockManager('static'); + const refreshingSessionManager = new MockManager('refreshing'); + const mux = new OptionalRefreshSessionManagerMux({ + staticSessionManager, + refreshingSessionManager, + sessionCanRefresh: () => true, + }); + + const states = trackState(mux); + await expect(mux.getSession({})).resolves.toBe('static'); + await mux.removeSession(); + expect(staticSessionManager.session).toBeUndefined(); + expect(refreshingSessionManager.session).toBeUndefined(); + expect(states).toEqual([false, true, false]); + }); +}); diff --git a/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.ts b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.ts new file mode 100644 index 0000000000..221a7635ea --- /dev/null +++ b/packages/core-app-api/src/lib/AuthSessionManager/OptionalRefreshSessionManagerMux.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable, SessionState } from '@backstage/core-plugin-api'; +import { + SessionManager, + MutableSessionManager, + GetSessionOptions, +} from './types'; +import { SessionStateTracker } from './SessionStateTracker'; + +type Options = { + /** + * A callback that is called to determine whether a given session supports refresh + */ + sessionCanRefresh: (session: T) => boolean; + + /** + * The session manager that is used if the a session does not support refresh. + */ + staticSessionManager: MutableSessionManager; + + /** + * The session manager that is used if the a session supports refresh. + */ + refreshingSessionManager: SessionManager; +}; + +/** + * OptionalRefreshSessionManagerMux wraps two different session managers, one for + * static session storage and another one that supports refresh. For each session + * that is retrieved is checked for whether it supports refresh. If it does, the + * refreshing session manager is used, otherwise the static session manager is used. + */ +export class OptionalRefreshSessionManagerMux implements SessionManager { + private readonly stateTracker = new SessionStateTracker(); + + private readonly sessionCanRefresh: (session: T) => boolean; + private readonly staticSessionManager: MutableSessionManager; + private readonly refreshingSessionManager: SessionManager; + + constructor(options: Options) { + this.sessionCanRefresh = options.sessionCanRefresh; + this.staticSessionManager = options.staticSessionManager; + this.refreshingSessionManager = options.refreshingSessionManager; + } + + async getSession(options: GetSessionOptions): Promise { + // First we check if there is an existing static session, using an optional request + const staticSession = await this.staticSessionManager.getSession({ + ...options, + optional: true, + }); + if (staticSession) { + this.stateTracker.setIsSignedIn(true); + return staticSession; + } + + // If there is no static session available, we ask the refresh manager to get a session + const session = await this.refreshingSessionManager.getSession(options); + + // Handling the case where the session request is optional + if (!session) { + this.stateTracker.setIsSignedIn(false); + return undefined; + } + + // Next we check if the session we received from the refreshing manager can actually + // be refreshed. If it can, we use this session without storing it in the static manager. + if (this.sessionCanRefresh(session)) { + this.stateTracker.setIsSignedIn(true); + return session; + } + + // If the session can't be refreshed, we store it in the static manager + this.staticSessionManager.setSession(session); + this.stateTracker.setIsSignedIn(true); + return session; + } + + async removeSession(): Promise { + await Promise.all([ + this.refreshingSessionManager.removeSession(), + this.staticSessionManager.removeSession(), + ]); + this.stateTracker.setIsSignedIn(false); + } + + sessionState$(): Observable { + return this.stateTracker.sessionState$(); + } +} diff --git a/packages/core-app-api/src/lib/loginPopup.test.ts b/packages/core-app-api/src/lib/loginPopup.test.ts index 1eb7c3f8b9..db4d9eb8eb 100644 --- a/packages/core-app-api/src/lib/loginPopup.test.ts +++ b/packages/core-app-api/src/lib/loginPopup.test.ts @@ -30,8 +30,7 @@ describe('showLoginPopup', () => { const removeEventListenerSpy = jest.spyOn(window, 'removeEventListener'); const payloadPromise = showLoginPopup({ - url: - 'my-origin/api/backend/auth/start?scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fa%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fb', + url: 'my-origin/api/backend/auth/start?scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fa%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fb', name: 'test-popup', origin: 'my-origin', }); diff --git a/packages/core-app-api/src/lib/subjects.ts b/packages/core-app-api/src/lib/subjects.ts index 0391050bda..e122da819b 100644 --- a/packages/core-app-api/src/lib/subjects.ts +++ b/packages/core-app-api/src/lib/subjects.ts @@ -29,7 +29,8 @@ import ObservableImpl from 'zen-observable'; * See http://reactivex.io/documentation/subject.html */ export class PublishSubject - implements Observable, ZenObservable.SubscriptionObserver { + implements Observable, ZenObservable.SubscriptionObserver +{ private isClosed = false; private terminatingError?: Error; @@ -121,7 +122,8 @@ export class PublishSubject * See http://reactivex.io/documentation/subject.html */ export class BehaviorSubject - implements Observable, ZenObservable.SubscriptionObserver { + implements Observable, ZenObservable.SubscriptionObserver +{ private isClosed = false; private currentValue: T; private terminatingError?: Error; diff --git a/packages/core-app-api/src/lib/versionedValues.ts b/packages/core-app-api/src/lib/versionedValues.ts index 3d0a4a41ae..3b3064fc7e 100644 --- a/packages/core-app-api/src/lib/versionedValues.ts +++ b/packages/core-app-api/src/lib/versionedValues.ts @@ -29,7 +29,7 @@ export type VersionedValue = { * Creates a container for a map of versioned values that implements VersionedValue. */ export function createVersionedValueMap< - Versions extends { [version: number]: any } + Versions extends { [version: number]: any }, >(versions: Versions): VersionedValue { Object.freeze(versions); return { diff --git a/packages/core-app-api/src/routing/FlatRoutes.test.tsx b/packages/core-app-api/src/routing/FlatRoutes.test.tsx index d5201a6f57..0df296cc59 100644 --- a/packages/core-app-api/src/routing/FlatRoutes.test.tsx +++ b/packages/core-app-api/src/routing/FlatRoutes.test.tsx @@ -37,11 +37,11 @@ function makeRouteRenderer(node: ReactNode) { ({ NotFoundErrorPage: () => <>Not Found, }), - } as unknown) as AppContext + } as unknown as AppContext } > diff --git a/packages/core-app-api/src/routing/RouteResolver.test.ts b/packages/core-app-api/src/routing/RouteResolver.test.ts index b47a18ecce..1b7eb10003 100644 --- a/packages/core-app-api/src/routing/RouteResolver.test.ts +++ b/packages/core-app-api/src/routing/RouteResolver.test.ts @@ -65,7 +65,7 @@ const externalRef4 = createExternalRouteRef({ describe('RouteResolver', () => { it('should not resolve anything with an empty resolver', () => { - const r = new RouteResolver(new Map(), new Map(), [], new Map()); + const r = new RouteResolver(new Map(), new Map(), [], new Map(), ''); expect(r.resolve(ref1, '/')?.()).toBe(undefined); expect(r.resolve(ref2, '/')?.({ x: '1x' })).toBe(undefined); @@ -85,6 +85,7 @@ describe('RouteResolver', () => { new Map(), [{ routeRefs: new Set([ref1]), path: '/my-route', ...rest }], new Map(), + '', ); expect(r.resolve(ref1, '/')?.()).toBe('/my-route'); @@ -99,6 +100,29 @@ describe('RouteResolver', () => { expect(r.resolve(externalRef4, '/')?.({ x: '6x' })).toBe(undefined); }); + it('should resolve an absolute route and an app base path', () => { + const r = new RouteResolver( + new Map([[ref1, '/my-route']]), + new Map(), + [{ routeRefs: new Set([ref1]), path: '/my-route', ...rest }], + new Map(), + '/base', + ); + + expect(r.resolve(ref1, '/')?.()).toBe('/base/my-route'); + expect(r.resolve(ref2, '/')?.({ x: '1x' })).toBe(undefined); + expect(r.resolve(subRef1, '/')?.()).toBe('/base/my-route/foo'); + expect(r.resolve(subRef2, '/')?.({ a: '2a' })).toBe( + '/base/my-route/foo/2a', + ); + expect(r.resolve(subRef3, '/')?.({ x: '3x' })).toBe(undefined); + expect(r.resolve(subRef4, '/')?.({ x: '4x', a: '4a' })).toBe(undefined); + expect(r.resolve(externalRef1, '/')?.()).toBe(undefined); + expect(r.resolve(externalRef2, '/')?.()).toBe(undefined); + expect(r.resolve(externalRef3, '/')?.({ x: '5x' })).toBe(undefined); + expect(r.resolve(externalRef4, '/')?.({ x: '6x' })).toBe(undefined); + }); + it('should resolve an absolute route with a param and with a parent', () => { const r = new RouteResolver( new Map([ @@ -122,6 +146,7 @@ describe('RouteResolver', () => { [externalRef3, ref2], [externalRef4, subRef3], ]), + '', ); expect(r.resolve(ref1, '/')?.()).toBe('/my-route'); @@ -179,6 +204,7 @@ describe('RouteResolver', () => { }, ], new Map(), + '', ); expect(r.resolve(ref2, '/')?.({ x: 'x' })).toBe('/root/x'); @@ -232,6 +258,7 @@ describe('RouteResolver', () => { [externalRef3, ref2], [externalRef4, subRef3], ]), + '', ); const l = '/my-grandparent/my-y/my-parent/my-x'; diff --git a/packages/core-app-api/src/routing/RouteResolver.ts b/packages/core-app-api/src/routing/RouteResolver.ts index e09da28df8..b5be5f893e 100644 --- a/packages/core-app-api/src/routing/RouteResolver.ts +++ b/packages/core-app-api/src/routing/RouteResolver.ts @@ -187,6 +187,7 @@ export class RouteResolver { ExternalRouteRef, RouteRef | SubRouteRef >, + private readonly appBasePath: string, // base path without a trailing slash ) {} resolve( @@ -209,13 +210,15 @@ export class RouteResolver { // Next we figure out the base path, which is the combination of the common parent path // between our current location and our target location, as well as the additional path // that is the difference between the parent path and the base of our target location. - const basePath = resolveBasePath( - targetRef, - sourceLocation, - this.routePaths, - this.routeParents, - this.routeObjects, - ); + const basePath = + this.appBasePath + + resolveBasePath( + targetRef, + sourceLocation, + this.routePaths, + this.routeParents, + this.routeObjects, + ); const routeFunc: RouteFunc = (...[params]) => { return basePath + generatePath(targetPath, params); diff --git a/packages/core-app-api/src/routing/RoutingProvider.test.tsx b/packages/core-app-api/src/routing/RoutingProvider.test.tsx index 499b81a743..47f9c87c96 100644 --- a/packages/core-app-api/src/routing/RoutingProvider.test.tsx +++ b/packages/core-app-api/src/routing/RoutingProvider.test.tsx @@ -152,6 +152,7 @@ function withRoutingProvider( routeParents={routeParents} routeObjects={routeObjects} routeBindings={new Map(routeBindings)} + basePath="" > {root} @@ -330,9 +331,10 @@ describe('discovery', () => { }); describe('v1 consumer', () => { - const RoutingContext = getGlobalSingleton< - Context> - >('routing-context'); + const RoutingContext = + getGlobalSingleton>>( + 'routing-context', + ); function useMockRouteRefV1( routeRef: AnyRouteRef, @@ -367,6 +369,7 @@ describe('v1 consumer', () => { routeParents={new Map()} routeObjects={[]} routeBindings={new Map()} + basePath="/base" children={children} /> ), @@ -375,8 +378,8 @@ describe('v1 consumer', () => { expect(renderedHook.result.current).toBe(undefined); renderedHook.rerender({ routeRef: routeRef2 }); - expect(renderedHook.result.current?.()).toBe('/foo'); + expect(renderedHook.result.current?.()).toBe('/base/foo'); renderedHook.rerender({ routeRef: routeRef3 }); - expect(renderedHook.result.current?.({ x: 'my-x' })).toBe('/bar/my-x'); + expect(renderedHook.result.current?.({ x: 'my-x' })).toBe('/base/bar/my-x'); }); }); diff --git a/packages/core-app-api/src/routing/RoutingProvider.tsx b/packages/core-app-api/src/routing/RoutingProvider.tsx index ea8e8f76f8..2192f5df26 100644 --- a/packages/core-app-api/src/routing/RoutingProvider.tsx +++ b/packages/core-app-api/src/routing/RoutingProvider.tsx @@ -38,6 +38,7 @@ type ProviderProps = { routeParents: Map; routeObjects: BackstageRouteObject[]; routeBindings: Map; + basePath?: string; children: ReactNode; }; @@ -46,6 +47,7 @@ export const RoutingProvider = ({ routeParents, routeObjects, routeBindings, + basePath = '', children, }: ProviderProps) => { const resolver = new RouteResolver( @@ -53,6 +55,7 @@ export const RoutingProvider = ({ routeParents, routeObjects, routeBindings, + basePath, ); const versionedValue = createVersionedValueMap({ 1: resolver }); diff --git a/packages/core-app-api/src/routing/types.ts b/packages/core-app-api/src/routing/types.ts index 6561a0da70..a12e914aba 100644 --- a/packages/core-app-api/src/routing/types.ts +++ b/packages/core-app-api/src/routing/types.ts @@ -75,7 +75,7 @@ export function isSubRouteRef( export function isExternalRouteRef< Params extends AnyParams, - Optional extends boolean + Optional extends boolean, >( routeRef: | RouteRef diff --git a/packages/core-components/CHANGELOG.md b/packages/core-components/CHANGELOG.md index 8372b25563..1b56b033b5 100644 --- a/packages/core-components/CHANGELOG.md +++ b/packages/core-components/CHANGELOG.md @@ -1,5 +1,91 @@ # @backstage/core-components +## 0.3.3 + +### Patch Changes + +- d041655a7: Fix accessibility issue in ``. The component doesn't render anymore an hidden `textarea` containing the text to be copied. +- 6d76bca85: Handle changes to nodes passed into `` correctly. +- Updated dependencies + - @backstage/config@0.1.8 + +## 0.3.2 + +### Patch Changes + +- a3f3cff3b: Change the default hover experience for the sidebar to be not jumpy & add visual separation between sidebar & Entity Page tabs for dark mode. +- 6b1afe8c0: Add a configurable `palette.bursts.gradient` property to the Backstage theme, to support customizing the gradients in the `ItemCard` header. +- Updated dependencies + - @backstage/config@0.1.7 + - @backstage/theme@0.2.10 + +## 0.3.1 + +### Patch Changes + +- 56c773909: Switched `@types/react` dependency to request `*` rather than a specific version. +- 55a5dbd54: Fix for `SidebarItem` matching the active route too broadly. +- Updated dependencies + - @backstage/core-plugin-api@0.1.6 + +## 0.3.0 + +### Minor Changes + +- 7bf006210: Remove unused props from InfoCard prop type + +### Patch Changes + +- c4d8ff963: Switched frontend identity code to use `token` instead of the deprecated `idToken` field +- 7b8aa8d0d: Move the `CreateComponentButton` from the catalog plugin to the `core-components` & rename it to `CreateButton` to be reused inside the api-docs plugin & scaffolder plugin, but also future plugins. Additionally, improve responsiveness of `CreateButton` & `SupportButton` by shrinking them to `IconButtons` on smaller screens. +- 260c053b9: Fix All Material UI Warnings +- Updated dependencies + - @backstage/config@0.1.6 + - @backstage/core-plugin-api@0.1.5 + +## 0.2.0 + +### Minor Changes + +- 9d40fcb1e: - Bumping `material-ui/core` version to at least `4.12.2` as they made some breaking changes in later versions which broke `Pagination` of the `Table`. + - Switching out `material-table` to `@material-table/core` for support for the later versions of `material-ui/core` + - This causes a minor API change to `@backstage/core-components` as the interface for `Table` re-exports the `prop` from the underlying `Table` components. + - `onChangeRowsPerPage` has been renamed to `onRowsPerPageChange` + - `onChangePage` has been renamed to `onPageChange` + - Migration guide is here: https://material-table-core.com/docs/breaking-changes + +### Patch Changes + +- 19d9995b6: Improve accessibility of core & catalog components by adjusting them with non-breaking changes. +- 224e54484: Added an `EntityProcessingErrorsPanel` component to show any errors that occurred when refreshing an entity from its source location. + + If upgrading, this should be added to your `EntityPage` in your Backstage application: + + ```diff + // packages/app/src/components/catalog/EntityPage.tsx + + const overviewContent = ( + ... + + + + + + + + + + + + + + + + + + + + ``` + + Additionally, `WarningPanel` now changes color based on the provided severity. + +- Updated dependencies + - @backstage/core-plugin-api@0.1.4 + - @backstage/theme@0.2.9 + ## 0.1.6 ### Patch Changes diff --git a/packages/core-components/api-report.md b/packages/core-components/api-report.md index ccafa4cfa7..f73eee1f13 100644 --- a/packages/core-components/api-report.md +++ b/packages/core-components/api-report.md @@ -11,11 +11,10 @@ import { Breadcrumbs as Breadcrumbs_2 } from '@material-ui/core'; import { ButtonProps } from '@material-ui/core'; import { ButtonTypeMap } from '@material-ui/core'; import { CardHeaderProps } from '@material-ui/core'; -import { Column } from 'material-table'; +import { Column } from '@material-table/core'; import { CommonProps } from '@material-ui/core/OverridableComponent'; import { ComponentClass } from 'react'; import { ComponentProps } from 'react'; -import { ComponentType } from 'react'; import { Context } from 'react'; import { default as CSS_2 } from 'csstype'; import { CSSProperties } from 'react'; @@ -26,8 +25,9 @@ import { IconComponent } from '@backstage/core-plugin-api'; import { LinearProgressProps } from '@material-ui/core'; import { LinkProps as LinkProps_2 } from '@material-ui/core'; import { LinkProps as LinkProps_3 } from 'react-router-dom'; -import { MaterialTableProps } from 'material-table'; +import { MaterialTableProps } from '@material-table/core'; import { NavLinkProps } from 'react-router-dom'; +import { Palette } from '@material-ui/core/styles/createPalette'; import { ProfileInfoApi } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; import PropTypes from 'prop-types'; @@ -80,7 +80,7 @@ export const Avatar: ({ // Warning: (ae-missing-release-tag) "Breadcrumbs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const Breadcrumbs: ({ children, ...props }: Props_25) => JSX.Element; +export const Breadcrumbs: ({ children, ...props }: Props_24) => JSX.Element; // Warning: (ae-missing-release-tag) "BrokenImageIcon" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -441,7 +441,7 @@ export const Content: ({ noPadding, children, ...props -}: PropsWithChildren) => JSX.Element; +}: PropsWithChildren) => JSX.Element; // Warning: (ae-forgotten-export) The symbol "ContentHeaderProps" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "ContentHeader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -467,6 +467,15 @@ export const CopyTextButton: { }; }; +// Warning: (ae-forgotten-export) The symbol "CreateButtonProps" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "CreateButton" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const CreateButton: ({ + title, + to, +}: CreateButtonProps) => JSX.Element | null; + // Warning: (ae-missing-release-tag) "DashboardIcon" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -695,7 +704,7 @@ export const Header: ({ tooltip, type, typeLink, -}: PropsWithChildren) => JSX.Element; +}: PropsWithChildren) => JSX.Element; // Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "HeaderIconLinkRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -777,7 +786,7 @@ export const InfoCard: ({ className, noPadding, titleTypographyProps, -}: Props_20) => JSX.Element; +}: Props_19) => JSX.Element; // Warning: (ae-missing-release-tag) "InfoCardVariants" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -1200,7 +1209,7 @@ export const OverflowTooltip: (props: Props_12) => JSX.Element; export const Page: ({ themeId, children, -}: PropsWithChildren) => JSX.Element; +}: PropsWithChildren) => JSX.Element; // Warning: (ae-forgotten-export) The symbol "PageWithHeaderProps" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "PageWithHeader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -1291,7 +1300,7 @@ export const Sidebar: ({ openDelayMs, closeDelayMs, children, -}: PropsWithChildren) => JSX.Element; +}: PropsWithChildren) => JSX.Element; // Warning: (ae-missing-release-tag) "SIDEBAR_INTRO_LOCAL_STORAGE" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -2448,7 +2457,7 @@ export const SidebarSpacer: React_2.ComponentType< // Warning: (ae-missing-release-tag) "SignInPage" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const SignInPage: (props: Props_23) => JSX.Element; +export const SignInPage: (props: Props_22) => JSX.Element; // Warning: (ae-missing-release-tag) "SignInProviderConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -2592,7 +2601,7 @@ export const TabbedCard: ({ deepLink, value, onChange, -}: PropsWithChildren) => JSX.Element; +}: PropsWithChildren) => JSX.Element; // Warning: (ae-missing-release-tag) "TabbedLayout" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -2715,11 +2724,17 @@ export const WarningIcon: IconComponent; // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (tsdoc-param-tag-with-invalid-optional-name) The @param should not include a JSDoc-style optional name; it must not be enclosed in '[ ]' brackets. // Warning: (tsdoc-param-tag-with-invalid-type) The @param block should not include a JSDoc-style '{type}' -// Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "WarningProps" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "WarningPanel" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public -export const WarningPanel: (props: Props_17) => JSX.Element; +export const WarningPanel: ({ + severity, + title, + message, + children, + defaultExpanded, +}: WarningProps) => JSX.Element; // Warnings were encountered during analysis: // diff --git a/packages/core-components/package.json b/packages/core-components/package.json index 944002c74f..ae6daece4d 100644 --- a/packages/core-components/package.json +++ b/packages/core-components/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/core-components", "description": "Core components used by Backstage plugins and apps", - "version": "0.1.6", + "version": "0.3.3", "private": false, "publishConfig": { "access": "public", @@ -29,17 +29,18 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/config": "^0.1.5", - "@backstage/core-plugin-api": "^0.1.2", + "@backstage/config": "^0.1.8", + "@backstage/core-plugin-api": "^0.1.6", "@backstage/errors": "^0.1.1", - "@backstage/theme": "^0.2.8", - "@material-ui/core": "^4.11.0", + "@backstage/theme": "^0.2.10", + "@material-table/core": "^3.1.0", + "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "@testing-library/react-hooks": "^3.4.2", "@types/dagre": "^0.7.44", "@types/prop-types": "^15.7.3", - "@types/react": "^16.9", + "@types/react": "*", "@types/react-sparklines": "^1.7.0", "@types/react-text-truncate": "^0.14.0", "classnames": "^2.2.6", @@ -50,7 +51,6 @@ "dagre": "^0.8.5", "immer": "^9.0.1", "lodash": "^4.17.15", - "material-table": "^1.69.1", "pluralize": "^8.0.0", "prop-types": "^15.7.2", "qs": "^6.9.4", @@ -58,7 +58,7 @@ "react": "^16.12.0", "react-dom": "^16.12.0", "react-helmet": "6.1.0", - "react-hook-form": "^6.15.4", + "react-hook-form": "^7.12.2", "react-markdown": "^5.0.2", "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", @@ -70,9 +70,9 @@ "zen-observable": "^0.8.15" }, "devDependencies": { - "@backstage/core-app-api": "^0.1.5", - "@backstage/cli": "^0.7.5", - "@backstage/test-utils": "^0.1.15", + "@backstage/core-app-api": "^0.1.10", + "@backstage/cli": "^0.7.9", + "@backstage/test-utils": "^0.1.17", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^11.2.5", "@testing-library/user-event": "^13.1.8", @@ -84,6 +84,7 @@ "@types/jest": "^26.0.7", "@types/node": "^14.14.32", "@types/react-helmet": "^6.1.0", + "@types/react-syntax-highlighter": "^13.5.2", "@types/zen-observable": "^0.8.0" }, "files": [ diff --git a/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx b/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx index 298a8d5780..a4ffd3462b 100644 --- a/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx +++ b/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx @@ -15,8 +15,6 @@ */ import React from 'react'; -import { fireEvent } from '@testing-library/react'; -import { act } from 'react-dom/test-utils'; import { renderInTestApp } from '@backstage/test-utils'; import { CodeSnippet } from './CodeSnippet'; @@ -56,19 +54,4 @@ describe('', () => { expect(getByText('2')).toBeInTheDocument(); expect(getByText('3')).toBeInTheDocument(); }); - - it('copy code using button', async () => { - jest.useFakeTimers(); - document.execCommand = jest.fn(); - const { getByTitle } = await renderInTestApp( - , - ); - const button = getByTitle('Text copied to clipboard'); - fireEvent.click(button); - act(() => { - jest.runAllTimers(); - }); - expect(document.execCommand).toHaveBeenCalled(); - jest.useRealTimers(); - }); }); diff --git a/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx b/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx index acd0fa809c..b84d3919e8 100644 --- a/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx +++ b/packages/core-components/src/components/CopyTextButton/CopyTextButton.test.tsx @@ -19,8 +19,9 @@ import { fireEvent } from '@testing-library/react'; import { act } from 'react-dom/test-utils'; import { renderInTestApp } from '@backstage/test-utils'; import { CopyTextButton } from './CopyTextButton'; -import { ApiRegistry, ApiProvider } from '@backstage/core-app-api'; +import { ApiProvider, ApiRegistry } from '@backstage/core-app-api'; import { errorApiRef, ErrorApi } from '@backstage/core-plugin-api'; +import { useCopyToClipboard } from 'react-use'; jest.mock('popper.js', () => { const PopperJS = jest.requireActual('popper.js'); @@ -33,6 +34,17 @@ jest.mock('popper.js', () => { }; }); +jest.mock('react-use', () => { + const original = jest.requireActual('react-use'); + + return { + ...original, + useCopyToClipboard: jest + .fn() + .mockImplementation(original.useCopyToClipboard), + }; +}); + const props = { text: 'mockText', tooltipDelay: 2, @@ -43,9 +55,7 @@ const apiRegistry = ApiRegistry.from([ [ errorApiRef, { - post(error) { - throw error; - }, + post: jest.fn(), error$: jest.fn(), } as ErrorApi, ], @@ -53,17 +63,22 @@ const apiRegistry = ApiRegistry.from([ describe('', () => { it('renders without exploding', async () => { - const { getByDisplayValue } = await renderInTestApp( + const { getByTitle, queryByText } = await renderInTestApp( , ); - getByDisplayValue('mockText'); + expect(getByTitle('mockTooltip')).toBeInTheDocument(); + expect(queryByText('mockTooltip')).not.toBeInTheDocument(); }); - it('displays tooltip on click', async () => { + it('displays tooltip and copy the text on click', async () => { jest.useFakeTimers(); - document.execCommand = jest.fn(); + + const spy = useCopyToClipboard as jest.Mock; + const copy = jest.fn(); + spy.mockReturnValue([{}, copy]); + const rendered = await renderInTestApp( @@ -74,8 +89,22 @@ describe('', () => { act(() => { jest.runAllTimers(); }); - expect(document.execCommand).toHaveBeenCalled(); + expect(copy).toHaveBeenCalledWith('mockText'); rendered.getByText('mockTooltip'); jest.useRealTimers(); }); + + it('reports copy errors', async () => { + const spy = useCopyToClipboard as jest.Mock; + + const error = new Error('just an error'); + spy.mockReturnValue([{ error }, jest.fn()]); + + await renderInTestApp( + + + , + ); + expect(apiRegistry.get(errorApiRef)?.post).toHaveBeenCalledWith(error); + }); }); diff --git a/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx b/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx index b5bc516d1a..cc0ecaf76f 100644 --- a/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx +++ b/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx @@ -18,7 +18,8 @@ import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { IconButton, Tooltip } from '@material-ui/core'; import CopyIcon from '@material-ui/icons/FileCopy'; import PropTypes from 'prop-types'; -import React, { MouseEventHandler, useRef, useState } from 'react'; +import React, { MouseEventHandler, useEffect, useState } from 'react'; +import { useCopyToClipboard } from 'react-use'; /** * Copy text button with visual feedback in the form of @@ -52,30 +53,23 @@ export const CopyTextButton = (props: Props) => { ...props, }; const errorApi = useApi(errorApiRef); - const inputRef = useRef(null); const [open, setOpen] = useState(false); + const [{ error }, copyToClipboard] = useCopyToClipboard(); + + useEffect(() => { + if (error) { + errorApi.post(error); + } + }, [error, errorApi]); const handleCopyClick: MouseEventHandler = e => { e.stopPropagation(); setOpen(true); - - try { - if (inputRef.current) { - inputRef.current.select(); - document.execCommand('copy'); - } - } catch (error) { - errorApi.post(error); - } + copyToClipboard(text); }; return ( <> -