Merge branch 'master' into feature/ldap-less-memory

.changeset/afraid-trainers-know.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-auth-backend': patch
+---
+
+Fix a bug preventing an access token to be refreshed a second time with the GitLab provider.

.changeset/angry-ghosts-report.md

@@ -1,16 +0,0 @@
----
-'@backstage/backend-common': patch
----
-
-Support a `ensureExists` config option to skip ensuring a configured database exists. This allows deployment scenarios where
-limited permissions are given for provisioned databases without privileges to create new databases. If set to `false`, the
-database connection will not be validated prior to use which means the backend will not attempt to create the database if it
-doesn't exist. You can configure this in your app-config.yaml:
-
-```yaml
-backend:
-  database:
-    ensureExists: false
-```
-
-This defaults to `true` if unspecified. You can also configure this per plugin connection and will override the base option.

.changeset/angry-rules-fail.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-catalog': patch
----
-
-Add unstable prop for disabling unregister entity menu

.changeset/brave-eggs-rush.md

@@ -0,0 +1,6 @@
+---
+'@backstage/core-components': patch
+---
+
+Change the styling of the `<DependencyGraph>` to have more contrast in light
+mode. Nodes now have a design similar to material UI buttons.

.changeset/cuddly-rabbits-look.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Action needed: If you are using the templates located at https://github.com/backstage/backstage/tree/master/ in your Backstage app directly using the URL via the `app-config.yaml`, you should copy over the templates inside your org and import from there. The templates have now been moved to https://github.com/backstage/software-templates. See https://github.com/backstage/backstage/issues/6415 for explanation.

.changeset/curly-badgers-sit.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-catalog-react': patch
----
-
-Store filter values set in `EntityListProvider` in query parameters. This allows selected filters to be restored when returning to pages that list catalog entities.

.changeset/curvy-timers-sit.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+replace the deprecated file-loader for fonts with assets module

.changeset/fast-actors-grow.md

@@ -0,0 +1,16 @@
+---
+'@backstage/plugin-auth-backend': minor
+---
+
+Bump `passport-saml` to version 3. This is a breaking change, in that it [now requires](https://github.com/node-saml/passport-saml/pull/548) the `auth.saml.cert` parameter to be set. If you are not using SAML auth, you can ignore this.
+
+To update your settings, add something similar to the following to your app-config:
+
+```yaml
+auth:
+  saml:
+    # ... other settings ...
+    cert: 'MIICizCCAfQCCQCY8tKaMc0BMjANBgkqh ... W=='
+```
+
+For more information, see the [library README](https://github.com/node-saml/passport-saml#security-and-signatures).

.changeset/gorgeous-pugs-deliver.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Set the default TZ (Timezone) env for the test command to be UTC so any date related tests are consistent across timezones.

.changeset/green-vans-peel.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-catalog-react': patch
----
-
-Fix `EntityTypeFilter` so it produces unique case-insensitive set of available types

.changeset/grumpy-beds-design.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-user-settings': patch
+---
+
+Fix import for `createPlugin` in example snippet

.changeset/grumpy-dolls-call.md

@@ -1,10 +0,0 @@
----
-'@backstage/plugin-catalog-react': minor
-'@backstage/plugin-scaffolder': patch
----
-
-Updated the software templates list page (`ScaffolderPage`) to use the `useEntityListProvider` hook from #5643. This reduces the code footprint, making it easier to customize the display of this page, and consolidates duplicate approaches to querying the catalog with filters.
-
-- The `useEntityTypeFilter` hook has been updated along with the underlying `EntityTypeFilter` to work with multiple values, to allow more flexibility for different user interfaces. It's unlikely that this change affects you; however, if you're using either of these directly, you'll need to update your usage.
-- `SearchToolbar` was renamed to `EntitySearchBar` and moved to `catalog-react` to be usable by other entity list pages
-- `UserListPicker` now has an `availableTypes` prop to restrict which user-related options to present

.changeset/hot-otters-taste.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-newrelic': patch
+---
+
+Added pagination and searching to the table

.changeset/late-mangos-allow.md

@@ -0,0 +1,28 @@
+---
+'@backstage/plugin-home': minor
+---
+
+The homepage `<Header />` is now part of the composable canvas (allowing you to add the <HomepageTimer />, for example).
+
+You will need to wrap your existing composed `<HomePage />` component in `<Page />`, `<Header />`, and `<Content />` components, like this:
+
+```diff
+// app/src/components/home/HomePage.tsx
+
++ import { Content, Header, Page, HomePageTimer } from '@backstage/core-components';
+
+export const HomePage = () => (
++  <Page themeId="home">
++    <Header title="Home">
++      <HomepageTimer />
++    </Header>
++    <Content>
+  <Grid container spacing={3}>
+    <Grid item xs={12}>
+      <HomePageSearchBar />
+    </Grid>
+    // ...
++    </Content>
++  </Page>
+);
+```

.changeset/lemon-crabs-confess.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-scaffolder-backend': patch
----
-
-Updating fs-extra to 10.0.0 to handle broken symbolic links correctly

.changeset/lemon-dancers-taste.md

@@ -1,9 +0,0 @@
----
-'@backstage/plugin-catalog-backend': patch
----
-
-The codeowners processor extracts the username of the primary owner and uses this as the owner field.
-Given the kind isn't specified this is assumed to be a group and so the link to the owner in the about card
-doesn't work. This change specifies the kind where the entity is a user. e.g:
-
-`@iain-b` -> `user:iain-b`

.changeset/neat-mugs-add.md

@@ -0,0 +1,7 @@
+---
+'@backstage/integration': patch
+---
+
+Fix Azure URL handling to properly support both repo shorthand (`/owner/_git/project`) and full URLs (`/owner/project/_git/repo`).
+
+Fix Azure DevOps Server URL handling by being able to parse URLs with hosts other than `dev.azure.com`. Note that the `api-version` used for API requests is currently `6.0`, meaning you need to support at least this version in your Azure DevOps Server instance.

.changeset/nice-bugs-beg.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-user-settings': minor
----
-
-Exported and renamed components from the `@backstage/plugin-user-settings` plugin , to be able to use it in the consumer side and customize the `SettingPage`

.changeset/perfect-keys-hide.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Fill in most missing type exports.

.changeset/polite-dragons-allow.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-auth-backend': patch
+---
+
+Use a more informative error message if the configured OIDC identity provider does not provide a `userinfo_endpoint` in its metadata.

.changeset/polite-spies-judge.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-scaffolder-backend-module-rails': patch
----
-
-updated paths to consider differences between platform (windows corrected)

.changeset/poor-otters-buy.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-scaffolder-backend': patch
----
-
-Add new `fetch:template` action which handles the same responsibilities as `fetch:cookiecutter` without the external dependency on `cookiecutter`. For information on migrating from `fetch:cookiecutter` to `fetch:template`, see the [migration guide](https://backstage.io/docs/features/software-templates/builtin-actions#migrating-from-fetch-cookiecutter-to-fetch-template) in the docs.

.changeset/poor-weeks-act.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder': patch
+---
+
+Change the Categories filter to a dropdown component

.changeset/pretty-drinks-serve.md

@@ -1,12 +0,0 @@
----
-'@backstage/plugin-scaffolder-backend': patch
----
-
-Scaffolder: Added an 'eq' handlebars helper for use in software template YAML files. This can be used to execute a step depending on the value of an input, e.g.:
-
-```yaml
-steps:
-  id: 'conditional-step'
-  action: 'custom-action'
-  if: '{{ eq parameters.myvalue "custom" }}',
-```

.changeset/real-beans-collect.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Fix Azure `readTree` and `search` handling to properly support paths.

.changeset/real-plums-vanish.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-auth-backend': patch
----
-
-Add Sign In and Handler resolver for Microsoft provider

.changeset/rotten-pears-live.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-elasticsearch': patch
+---
+
+Added rejectUnauthorized config option

.changeset/search-mighty-mice-collect.md

@@ -1,6 +0,0 @@
----
-'@backstage/plugin-search-backend-node': minor
----
-
-Change return value of `SearchEngine.index` to `Promise<void>` to support
-implementation of external search engines.

.changeset/short-eggs-confess.md

@@ -1,5 +0,0 @@
----
-'@backstage/plugin-catalog-backend-module-ldap': patch
----
-
-Expose missing types used by the custom transformers

.changeset/short-mugs-drive.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+Improve UX for Login pop-up

.changeset/shy-rules-design.md

@@ -1,7 +0,0 @@
----
-'@backstage/plugin-scaffolder-backend': minor
----
-
-Updated inputs for the `publish:github:pull-request` action.
-
-Now requires a `repoUrl` instead of separate `owner` and `repo` inputs. This aligns with the output of the `RepoUrlPicker` ui field used by the pull-request sample template.

.changeset/silly-trainers-clean.md

@@ -0,0 +1,6 @@
+---
+'@backstage/core-app-api': patch
+'@backstage/core-plugin-api': patch
+---
+
+Switch to using utilities from `@backstage/version-bridge'.

.changeset/six-crabs-sit.md

@@ -0,0 +1,5 @@
+---
+'@backstage/core-components': patch
+---
+
+remove hard coded min height in page header

.changeset/strong-geese-hope.md

@@ -0,0 +1,13 @@
+---
+'@backstage/backend-common': patch
+'@backstage/integration': patch
+'@backstage/techdocs-common': patch
+'@backstage/plugin-catalog': patch
+'@backstage/plugin-catalog-backend': patch
+'@backstage/plugin-catalog-import': patch
+'@backstage/plugin-scaffolder': patch
+'@backstage/plugin-scaffolder-backend': patch
+'@backstage/plugin-techdocs': patch
+---
+
+Upgrade git-parse-url to v11.6.0

.changeset/techdocs-small-bugs-exist.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-home': patch
+---
+
+Adjust dependencies to `@types/react` and `react-router` to follow the pattern
+used by all other Backstage packages.

.changeset/techdocs-typescript-isnt-fun.md

@@ -1,7 +0,0 @@
----
-'@backstage/plugin-techdocs-backend': patch
----
-
-It is no longer required to provide a generator and a preparer to the TechDocs
-router factory when running TechDocs in the "recommended" (e.g. externally
-prepared and generated docs) configuration.

.changeset/ten-dolls-ring.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-explore': patch
+---
+
+Make styling of groups tab in the explore page use the Backstage theme instead
+of hard coded colors. Change the layout to be full screen.

.changeset/tidy-shoes-occur.md

@@ -0,0 +1,5 @@
+---
+'@backstage/integration': patch
+---
+
+Handle pagination on return results when fetch GitHub API to list selected repos

.changeset/unlucky-tables-repair.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog': patch
+---
+
+Display entity titles on `EntityLayout` if defined

.changeset/wicked-jars-move.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-cloudbuild': patch
+---
+
+- Swaps over the plugin CloudBuild from using MomentJS to using Luxon for date formatting.
+- Fixes some React warnings generated by nested <p> elements.

.changeset/wise-rockets-smoke.md

@@ -1,5 +0,0 @@
----
-'@backstage/backend-common': patch
----
-
-Document the default behavior of `statusCheck` option in `createStatusCheckRouter`.

.changeset/young-tables-reply.md

@@ -1,16 +0,0 @@
----
-'@backstage/backend-common': patch
-'@backstage/cli': patch
-'@backstage/config-loader': patch
-'@backstage/create-app': patch
-'@backstage/techdocs-common': patch
-'@backstage/plugin-app-backend': patch
-'@backstage/plugin-auth-backend': patch
-'@backstage/plugin-catalog-backend': patch
-'@backstage/plugin-kubernetes-backend': patch
-'@backstage/plugin-rollbar-backend': patch
-'@backstage/plugin-scaffolder-backend': patch
-'@backstage/plugin-techdocs-backend': patch
----
-
-Revert the upgrade to `fs-extra@10.0.0` as that seemed to have broken all installs inexplicably.

.github/CODEOWNERS

@@ -8,17 +8,18 @@
 /docs/features/techdocs                     @backstage/techdocs-core
 /docs/features/search                       @backstage/techdocs-core
 /docs/assets/search                         @backstage/techdocs-core
-/plugins/code-coverage                      @alde @nissayeva
-/plugins/code-coverage-backend              @alde @nissayeva
+/plugins/code-coverage                      @backstage/reviewers @alde @nissayeva
+/plugins/code-coverage-backend              @backstage/reviewers @alde @nissayeva
 /plugins/cost-insights                      @backstage/silver-lining
-/plugins/cloudbuild                         @trivago/ebarrios
+/plugins/cloudbuild                         @backstage/reviewers @trivago/ebarrios
 /plugins/search                             @backstage/techdocs-core
 /plugins/search-*                           @backstage/techdocs-core
 /plugins/techdocs                           @backstage/techdocs-core
 /plugins/techdocs-backend                   @backstage/techdocs-core
-/plugins/ilert                              @yacut
+/plugins/ilert                              @backstage/reviewers @yacut
+/plugins/home                               @backstage/techdocs-core
 /packages/search-common                     @backstage/techdocs-core
 /packages/techdocs-common                   @backstage/techdocs-core
-/.changeset/cost-insights-*                 @backstage/silver-lining
+/.changeset/cost-insights-*                 @backstage/reviewers @backstage/silver-lining
 /.changeset/search-*                        @backstage/techdocs-core
 /.changeset/techdocs-*                      @backstage/techdocs-core

.github/styles/vocab.txt

@@ -1,5 +1,6 @@
 abc
 accessors
+Anddddd
 Apdex
 api
 Api
@@ -14,6 +15,7 @@ Avro
 backrub
 Bigtable
 Billett
+Bitbucket
 Bitrise
 Blackbox
 bool
@@ -66,7 +68,6 @@ devs
 discoverability
 Discoverability
 dls
-docgen
 Dockerfile
 dockerfiles
 Dockerize
@@ -74,6 +75,7 @@ dockerode
 Docusaurus
 env
 Env
+elasticsearch
 esbuild
 eslint
 etag
@@ -82,6 +84,8 @@ facto
 failover
 Fargate
 Figma
+firehydrant
+FireHydrant
 Firekube
 Fiverr
 gitbeaker
@@ -137,6 +141,7 @@ maintainership
 makefile
 md
 memcache
+memoize
 memoized
 microservice
 microservices
@@ -210,8 +215,10 @@ repo
 Repo
 repos
 rerender
+rerenders
 Reusability
 reusability
+roadmaps
 rollbar
 Rollbar
 Rollup
@@ -220,6 +227,7 @@ Routable
 rst
 rsync
 ruleset
+runbook
 sam
 sanitization
 scaffolded

.github/workflows/chromatic-storybook-test.yml

@@ -5,6 +5,7 @@ on:
       - '.github/workflows/chromatic-storybook-test.yml'
       - 'packages/storybook/**'
       - 'packages/core-components/src/**'
+      - '**/*.stories.tsx'
 
 jobs:
   chromatic:
@@ -50,4 +51,5 @@ jobs:
           # projetToken intentionally shared to allow collaborators to run Chromatic on forks
           # https://www.chromatic.com/docs/custom-ci-provider#run-chromatic-on-external-forks-of-open-source-projects
           projectToken: 9tzak77m9nj
-          storybookBuildDir: 'packages/storybook/dist'
+          workingDir: 'packages/storybook'
+          storybookBuildDir: 'dist'

.github/workflows/ci.yml

@@ -105,11 +105,8 @@ jobs:
         run: git diff --quiet origin/master HEAD -- yarn.lock
         continue-on-error: true
 
-      - name: verify doc links
-        run: node scripts/verify-links.js
-
       - name: prettier
-        run: yarn prettier:check
+        run: yarn prettier:check '!ADOPTERS.md'
 
       - name: lock
         run: yarn lock:check
@@ -123,8 +120,12 @@ jobs:
       - name: type checking and declarations
         run: yarn tsc:full
 
-      - name: check api reports
-        run: yarn build:api-reports:only --ci
+        # We need to generate the API references as well, so that we can verify the doc links
+      - name: check api reports and generate API reference
+        run: yarn build:api-reports:only --ci --docs
+
+      - name: verify doc links
+        run: node scripts/verify-links.js
 
       - name: build changed packages
         if: ${{ steps.yarn-lock.outcome == 'success' }}

.github/workflows/e2e-win.yml

@@ -25,6 +25,13 @@ jobs:
 
     name: Node ${{ matrix.node-version }} on ${{ matrix.os }}
     steps:
+      # In order to have the create-app template function as if it was downloaded from NPM
+      # we need to make sure we checkout files with LF line endings only
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+
       - uses: actions/checkout@v2
 
       - name: use node.js ${{ matrix.node-version }}

.github/workflows/microsite-build-check.yml

@@ -27,9 +27,6 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: verify doc links
-        run: node scripts/verify-links.js
-
         # Skip caching of microsite dependencies, it keeps the global cache size
         # smaller, which make Windows builds a lot faster for the rest of the project.
       - name: yarn install

.github/workflows/microsite-with-storybook-deploy.yml

@@ -4,12 +4,6 @@ on:
   push:
     branches:
       - master
-    paths:
-      - '.github/workflows/microsite-with-storybook-deploy.yml'
-      - 'packages/storybook/**'
-      - 'packages/core-components/src/**'
-      - 'microsite/**'
-      - 'docs/**'
 
 jobs:
   deploy-microsite-and-storybook:
@@ -41,6 +35,9 @@ jobs:
         run: yarn install --frozen-lockfile
         working-directory: microsite
 
+      - name: build API reference
+        run: yarn build:api-docs
+
       - name: build microsite
         run: yarn build
         working-directory: microsite

.github/workflows/prettify.yml

@@ -0,0 +1,54 @@
+name: Prettier
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  autofix-markdown:
+    name: Autofix Markdown files using Prettier
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # Fetch changes to previous commit - required for 'only_changed' in Prettier action
+          fetch-depth: 0
+
+      # Beginning of yarn setup, keep in sync between all workflows, see ci.yml
+      - name: use node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+          registry-url: https://registry.npmjs.org/ # Needed for auth
+      - name: cache all node_modules
+        id: cache-modules
+        uses: actions/cache@v2
+        with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-v${{ matrix.node-version }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }}
+      - name: find location of global yarn cache
+        id: yarn-cache
+        if: steps.cache-modules.outputs.cache-hit != 'true'
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: cache global yarn cache
+        uses: actions/cache@v2
+        if: steps.cache-modules.outputs.cache-hit != 'true'
+        with:
+          path: ${{ steps.yarn-cache.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - name: yarn install
+        run: yarn install --frozen-lockfile
+      # End of yarn setup
+
+      - name: Run Prettier on ADOPTERS.md
+        uses: creyD/prettier_action@v3.1
+        with:
+          # Modifies commit only if prettier autofixed the ADOPTERS.md
+          prettier_options: --config docs/prettier.config.js --write ADOPTERS.md
+          only_changed: true
+          commit_message: 'Autofix ADOPTERS.md using Prettier'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/snyk-monitor.yml

@@ -0,0 +1,51 @@
+name: Snyk Monitoring
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+    paths:
+      - '.github/workflows/snyk-monitor.yml'
+      - '**/.snyk'
+      - '**/package.json'
+      - 'yarn.lock'
+
+# This workflow synchronizes the packages in this repo along with policies in
+# each .snyk file with the remote state in our snyk org. It allows us to define
+# ignore policies in the .snyk files and then have them show up in the snyk web
+# UI, and also automatically adds any new packages that are created.
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Monitor and Synchronize Snyk Policies
+        uses: snyk/actions/node@master
+        with:
+          command: monitor
+          args: >
+            --yarn-workspaces
+            --org=backstage-dgh
+            --strict-out-of-sync=false
+            --remote-repo-url=https://github.com/backstage/backstage
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+
+      # Above we run the `monitor` command, this runs the `test` command which is
+      # the one that generates the SARIF report that we can upload to GitHub.
+      - name: Create Snyk report
+        uses: snyk/actions/node@master
+        continue-on-error: true # To make sure that SARIF upload gets called
+        with:
+          args: >
+            --yarn-workspaces
+            --org=backstage-dgh
+            --strict-out-of-sync=false
+            --sarif-file-output=snyk.sarif
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      - name: Upload Snyk report
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: snyk.sarif

.imgbotconfig

@@ -1,5 +1,3 @@
 {
-  "ignoredFiles": [
-    "docs/assets/**/*.svg"
-  ]
+  "ignoredFiles": ["docs/assets/**/*.svg"]
 }

ADOPTERS.md

@@ -1,39 +1,47 @@
-| Organization                                      | Contact                                                                                                                                                                                             | Description of Use                                                                                                                                                                                    |
-| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [Spotify](https://www.spotify.com)                | [@leemills83](https://github.com/leemills83)                                                                                                                                                        | Main interface towards all of Spotify's infrastructure and technical documentation.                                                                                                                   |
-| [bol.com](https://www.bol.com)                    | [@sagacity](https://github.com/sagacity)                                                                                                                                                            | Initial work being done to unify platform tooling.                                                                                                                                                    |
-| [DFDS](https://www.dfds.com)                      | [@carlsendk](https://github.com/carlsendk)                                                                                                                                                          | V2 self-service platform.                                                                                                                                                                             |
-| [Roadie](https://roadie.io)                       | [@dtuite](https://github.com/dtuite)                                                                                                                                                                | Hosted, managed Backstage with easy set-up                                                                                                                                                            |
-| [Roku](https://www.roku.com)                      | [@timurista](https://github.com/timurista)                                                                                                                                                          | Initial work on Cloud engineering service platform.                                                                                                                                                   |
-| [SDA SE](https://sda.se)                          | [@Fox32](https://github.com/Fox32)                                                                                                                                                                  | Central place for developing and sharing services in our insurance ecosystem.                                                                                                                         |
-| [H-E-B](https://www.heb.com)                      | [@german-j-rodriguez](https://github.com/german-j-rodriguez)                                                                                                                                        | Initial work on Engineering Portal service platform.                                                                                                                                                  |
-| [American Airlines](https://www.aa.com)           | [@paulpach](https://github.com/paulpach)                                                                                                                                                            | Central place for developers to develop and maintain applications                                                                                                                                     |
-| [Kiwi.com](https://kiwi.com)                      | [@aexvir](https://github.com/aexvir)                                                                                                                                                                | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry.                                                                                                      |
-| [Voi](https://www.voiscooters.com/)               | [@K-Phoen](https://github.com/K-Phoen)                                                                                                                                                              | Developer portal, main gateway to our infrastructure, documentation and internal tooling.                                                                                                             |
-| [Talkdesk](https://www.talkdesk.com)              | [@jaime-talkdesk](https://github.com/jaime-talkdesk)                                                                                                                                                | Initial work for Engineering Portal and Self Provisioning to R&D                                                                                                                                      |
-| [Wealthsimple](https://www.wealthsimple.com)      | [@andrewthauer](https://github.com/andrewthauer)                                                                                                                                                    | Developer portal, service catalog, documentation and tooling                                                                                                                                          |
-| [Grab](https://www.grab.com)                      | [@althafh](https://github.com/althafh)                                                                                                                                                              | Initial work as a unified interface for all of Grab's internal tooling                                                                                                                                |
-| [Telenor Sweden](https://www.telenor.se)          | [@O5ten](https://github.com/O5ten)                                                                                                                                                                  | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks                                                                                    |
-| [Fiverr](https://www.fiverr.com)                  | [@nirga](https://github.com/nirga)                                                                                                                                                                  | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform.                                                                |
-| [Zalando SE](https://www.zalando.de)              | [@leviferreira](https://github.com/leviferreira)                                                                                                                                                    | Building V2 of the Internal Development Portal.                                                                                                                                                       |
-| [LegalZoom](https://legalzoom.com)                | [@backjo](https://github.com/backjo)                                                                                                                                                                | Developer portal - hub for all engineering projects and metadata.                                                                                                                                     |
-| [Expedia Group](https://www.expediagroup.com)     | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit                                                                                                                                                                           |
-| [Paddle.com](https://paddle.com)                  | [Ioannis Georgoulas](https://github.com/geototti21)                                                                                                                                                 | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go                                                                       |
-| [Acast.com](https://acast.com)                    | [Olle Lundberg](https://github.com/lndbrg)                                                                                                                                                          | Developer portal with tech docs, service catalog and a bunch of other internal tooling                                                                                                                |
-| [Lunar](https://lunar.app)                        | [Jacob Valdemar](https://github.com/JacobValdemar)                                                                                                                                                  | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's.                                                                        |
-| [Trendyol](https://trendyol.com)                  | [Erdogan Oksuz](https://github.com/erdoganoksuz)                                                                                                                                                    | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme.                                                         |
-| [Peloton](https://www.onepeloton.com/)            | [Jim Haughwout](https://github.com/JimHaughwout)                                                                                                                                                    | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well.                                                                                |
-| [TELUS](https://telus.com)                        | [Seb Barre](https://github.com/sbarre)                                                                                                                                                              | The Go-to place to find answers about development and delivery at TELUS.                                                                                                                              |
-| [Brex](https://www.brex.com/)                     | [Vamsi Chitters](https://github.com/vamsikc)                                                                                                                                                        | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies.                                                                        |
-| [Oriflame](https://www.oriflame.com/)             | [Oriflame](https://github.com/oriflame)                                                                                                                                                             | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more.                                                             |
-| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH)                                                                                                                                                  | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments.            |
-| [Netflix](https://www.netflix.com/)               | [bleathem](https://github.com/bleathem)                                                                                                                                                             | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support.               |
-| [b.well](https://www.icanbwell.com/)              | [Jacob Rosales](https://github.com/jrosales)                                                                                                                                                        | Foundation for our engineering portal and cloud insights.                                                                                                                                             |
-| [PagerDuty](https://www.pagerduty.com/)           | [Mark Shaw](https://github.com/markshawtoronto)                                                                                                                                                     | Developer portal, initially focused on software templates and tech-docs.                                                                                                                              |
-| [MoonShiner](https://moonshiner.at)               | [Fabian Hippmann](https://github.com/FabianHippmann)                                                                                                                                                | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀                                                                   |
-| [FundApps](https://www.fundapps.co/)              | [Elliot Greenwood](https://github.com/egnwd)                                                                                                                                                        | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes                                                                                      |
-| [DAZN](https://dazn.com/)                         | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com)                                                  | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). |
-| [HelloFresh](https://www.hellofresh.de/)          | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni)                                                    | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally.                                                                                                        |
-| [FactSet](https://www.factset.com/)               | [@kuangp](https://github.com/kuangp)                                                                                                                                                                | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools.                           |
-| [Workrise](https://www.workrise.com/)             | [Michael Rode](https://github.com/michaelrode)                                                                                                                                                      | Developer portal, main gateway to our infrastructure, documentation and internal tooling.                                                                                                             |
-| [RedVentures](https://www.redventures.com/)       | [Chris Diaz](https://github.com/codingdiaz)                                                                                                                                                         | Developer portal that brings everything an engineer needs to provide value into a single pane of glass.                                                                                               |
+| Organization                                                             | Contact                                                                                                                                                                                             | Description of Use                                                                                                                                                                                    |
+| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Spotify](https://www.spotify.com)                                       | [@leemills83](https://github.com/leemills83)                                                                                                                                                        | Main interface towards all of Spotify's infrastructure and technical documentation.                                                                                                                   |
+| [bol.com](https://www.bol.com)                                           | [@sagacity](https://github.com/sagacity)                                                                                                                                                            | Initial work being done to unify platform tooling.                                                                                                                                                    |
+| [DFDS](https://www.dfds.com)                                             | [@carlsendk](https://github.com/carlsendk)                                                                                                                                                          | V2 self-service platform.                                                                                                                                                                             |
+| [Roadie](https://roadie.io)                                              | [@dtuite](https://github.com/dtuite)                                                                                                                                                                | Hosted, managed Backstage with easy set-up                                                                                                                                                            |
+| [Roku](https://www.roku.com)                                             | [@timurista](https://github.com/timurista)                                                                                                                                                          | Initial work on Cloud engineering service platform.                                                                                                                                                   |
+| [SDA SE](https://sda.se)                                                 | [@Fox32](https://github.com/Fox32)                                                                                                                                                                  | Central place for developing and sharing services in our insurance ecosystem.                                                                                                                         |
+| [H-E-B](https://www.heb.com)                                             | [@german-j-rodriguez](https://github.com/german-j-rodriguez)                                                                                                                                        | Initial work on Engineering Portal service platform.                                                                                                                                                  |
+| [American Airlines](https://www.aa.com)                                  | [@paulpach](https://github.com/paulpach)                                                                                                                                                            | Central place for developers to develop and maintain applications                                                                                                                                     |
+| [Kiwi.com](https://kiwi.com)                                             | [@aexvir](https://github.com/aexvir)                                                                                                                                                                | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry.                                                                                                      |
+| [Voi](https://www.voiscooters.com/)                                      | [@K-Phoen](https://github.com/K-Phoen)                                                                                                                                                              | Developer portal, main gateway to our infrastructure, documentation and internal tooling.                                                                                                             |
+| [Talkdesk](https://www.talkdesk.com)                                     | [@jaime-talkdesk](https://github.com/jaime-talkdesk)                                                                                                                                                | Initial work for Engineering Portal and Self Provisioning to R&D                                                                                                                                      |
+| [Wealthsimple](https://www.wealthsimple.com)                             | [@andrewthauer](https://github.com/andrewthauer)                                                                                                                                                    | Developer portal, service catalog, documentation and tooling                                                                                                                                          |
+| [Grab](https://www.grab.com)                                             | [@althafh](https://github.com/althafh)                                                                                                                                                              | Initial work as a unified interface for all of Grab's internal tooling                                                                                                                                |
+| [Telenor Sweden](https://www.telenor.se)                                 | [@O5ten](https://github.com/O5ten)                                                                                                                                                                  | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks                                                                                    |
+| [Fiverr](https://www.fiverr.com)                                         | [@nirga](https://github.com/nirga)                                                                                                                                                                  | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform.                                                                |
+| [Zalando SE](https://www.zalando.de)                                     | [@leviferreira](https://github.com/leviferreira)                                                                                                                                                    | Building V2 of the Internal Development Portal.                                                                                                                                                       |
+| [LegalZoom](https://legalzoom.com)                                       | [@backjo](https://github.com/backjo)                                                                                                                                                                | Developer portal - hub for all engineering projects and metadata.                                                                                                                                     |
+| [Expedia Group](https://www.expediagroup.com)                            | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit                                                                                                                                                                           |
+| [Paddle.com](https://paddle.com)                                         | [Ioannis Georgoulas](https://github.com/geototti21)                                                                                                                                                 | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go                                                                       |
+| [Acast.com](https://acast.com)                                           | [Olle Lundberg](https://github.com/lndbrg)                                                                                                                                                          | Developer portal with tech docs, service catalog and a bunch of other internal tooling                                                                                                                |
+| [Lunar](https://lunar.app)                                               | [Jacob Valdemar](https://github.com/JacobValdemar)                                                                                                                                                  | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's.                                                                        |
+| [Trendyol](https://trendyol.com)                                         | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb)                                                                                                        | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme.                                                         |
+| [Peloton](https://www.onepeloton.com/)                                   | [Jim Haughwout](https://github.com/JimHaughwout)                                                                                                                                                    | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well.                                                                                |
+| [TELUS](https://telus.com)                                               | [Seb Barre](https://github.com/sbarre)                                                                                                                                                              | The Go-to place to find answers about development and delivery at TELUS.                                                                                                                              |
+| [Brex](https://www.brex.com/)                                            | [Vamsi Chitters](https://github.com/vamsikc)                                                                                                                                                        | A centralized UI to understand how a service fits in the whole Brex architecture and manage a team’s engineering dependencies.                                                                        |
+| [Oriflame](https://www.oriflame.com/)                                    | [Oriflame](https://github.com/oriflame)                                                                                                                                                             | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more.                                                             |
+| [Booz Allen Hamilton](https://www.boozallen.com/)                        | [Jason Miller](https://github.com/JasonMiller-BAH)                                                                                                                                                  | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments.            |
+| [Netflix](https://www.netflix.com/)                                      | [bleathem](https://github.com/bleathem)                                                                                                                                                             | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support.               |
+| [b.well](https://www.icanbwell.com/)                                     | [Jacob Rosales](https://github.com/jrosales)                                                                                                                                                        | Foundation for our engineering portal and cloud insights.                                                                                                                                             |
+| [PagerDuty](https://www.pagerduty.com/)                                  | [Mark Shaw](https://github.com/markshawtoronto)                                                                                                                                                     | Developer portal, initially focused on software templates and tech-docs.                                                                                                                              |
+| [MoonShiner](https://moonshiner.at)                                      | [Fabian Hippmann](https://github.com/FabianHippmann)                                                                                                                                                | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process 🌕🚀🧑‍🚀                                                                   |
+| [FundApps](https://www.fundapps.co/)                                     | [Elliot Greenwood](https://github.com/egnwd)                                                                                                                                                        | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes                                                                                      |
+| [DAZN](https://dazn.com/)                                                | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com)                                                  | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). |
+| [HelloFresh](https://www.hellofresh.de/)                                 | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni)                                                    | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally.                                                                                                        |
+| [FactSet](https://www.factset.com/)                                      | [@kuangp](https://github.com/kuangp)                                                                                                                                                                | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools.                           |
+| [Workrise](https://www.workrise.com/)                                    | [Michael Rode](https://github.com/michaelrode)                                                                                                                                                      | Developer portal, main gateway to our infrastructure, documentation and internal tooling.                                                                                                             |
+| [RedVentures](https://www.redventures.com/)                              | [Chris Diaz](https://github.com/codingdiaz)                                                                                                                                                         | Developer portal that brings everything an engineer needs to provide value into a single pane of glass.                                                                                               |
+| [MavTek](https://www.mavtek.com/)                                        | [@fgascon](https://github.com/fgascon)                                                                                                                                                              | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices.                                                                                 |
+| [QuintoAndar](https://www.quintoandar.com.br/)                           | [@quintoandar](https://github.com/quintoandar)                                                                                                                                                      | Developer portal, services catalog and centralization of service metrics.                                                                                                                             |
+| [empathy.co](https://empathy.co/)                                        | [@guillermotti](https://github.com/guillermotti)                                                                                                                                                    | Developer portal for tech docs, service catalog, plugin discovery and much more.                                                                                                                      |
+| [creditas.com](https://creditas.com/)                                    | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas)                                                                                                       | Centralization of all services, standards, documentation, etc. We started the deployment process.                                                                                                     |
+| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl)                                                                                                                                                    | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration                                                                                      |
+| [Powerspike](https://powerspike.tv/)                                     | [@trelore](https://github.com/trelore)                                                                                                                                                              | Developer portal for documentation of core libraries and repositories.                                                                                                                                |
+| [2U](https://2u.com)                                                     | [Andrew Thal](https://github.com/athal7)      | Development team home-base, promoting service discoverability, resource dependencies, and tech radar |
+| [Taxfix](https://taxfix.de/)                                                     | [Sami Ur Rehman](https://github.com/samiurrehman92)      | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins.  |

CONTRIBUTING.md

@@ -116,7 +116,7 @@ In general, changesets are not needed for the documentation, build utilities, co
 
 1. Run `yarn changeset`
 2. Select which packages you want to include a changeset for
-3. Select impact of change that you're introducing (patch, minor, or major)
+3. Select impact of change that you're introducing, using `minor` for breaking changes and `patch` otherwise. We do not use `major` changes while packages are at version `0.x`.
 4. Add generated changeset to Git
 5. Push the commit with your changeset to the branch associated with your PR
 6. Accept our gratitude for making the release process easier on the maintainers

README.md

@@ -2,8 +2,6 @@
 
 # [Backstage](https://backstage.io)
 
-_During the month of July the majority of the maintainers will be on summer vacation 🏖️ Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_
-
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects)
 [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22)
@@ -63,3 +61,9 @@ Check out [the documentation](https://backstage.io/docs/getting-started) on how
 Copyright 2020-2021 © The Backstage Authors. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page: https://www.linuxfoundation.org/trademark-usage
 
 Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
+
+## Security
+
+Please report sensitive security issues via Spotify's [bug-bounty program](https://hackerone.com/spotify) rather than GitHub.
+
+For further details please see our complete [security release process](SECURITY.md).

SECURITY.md

@@ -9,3 +9,30 @@
 ## Reporting a Vulnerability
 
 Please report sensitive security issues via Spotify's [bug-bounty program](https://hackerone.com/spotify) rather than GitHub.
+
+If you have questions about a potential vulnerability, please reach out on Discord by asking for a maintainer in the `#support` channel, or via direct message to a maintainer.
+
+## Remediation and Notification Process
+
+Vulnerabilities are handled and published through [GitHub Security Advisories](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories).
+
+In the event of a vulnerability the runbook for the maintainers is as follows:
+
+1. Create a [new draft security advisory](https://github.com/backstage/backstage/security/advisories/new). The values and descriptions don't need to be perfect to begin with as they can be edited later. For severity, use the "Assess severity using CVSS" and refer to [the guide](https://www.first.org/cvss/v3.1/user-guide) for help.
+2. Request a CVE identification number. It can take up to 72h for one to be assigned so be sure to do this early on.
+3. If there is not already a patch for the vulnerability, collaborate on one in a private fork:
+   1. Head to the security advisory on GitHub and [create a private fork](https://docs.github.com/en/code-security/security-advisories/collaborating-in-a-temporary-private-fork-to-resolve-a-security-vulnerability)
+   1. [Invite any collaborators](https://docs.github.com/en/code-security/security-advisories/adding-a-collaborator-to-a-security-advisory) from outside the maintainer team that should be part of creating a fix.
+   1. Create one or multiple Pull Requests with fixes towards the fork. Note that these PRs will not have CI checks run on them, so more care should be taken to run local validation. The PRs are also not merged like normal PRs, but are instead merged straight into the main repo all at once when the merge button is hit on the security advisory.
+4. Once the fix is ready in a PR or private fork and it is time to release it, there are a couple of options. Either merge into the main branch with a changeset and wait for a regular release, or do a quick release by manually bumping the version in `package.json` of the affected package, along with a manual `CHANGELOG.md` entry. Note that a quick release will only work if the package does not have any other pending changes that depend on pending changes in other packages, so be sure to manually check that first, and fall back to an early regular release if needed. In general it's best to stick with the regular release flow, with the quick release being used only for time sensitive fixes.
+5. Finalize and publish the security advisory. Note that once you hit the publish button it's no longer possible to edit the advisory. Just like the CVE number this can take up to 72h, and expect it to be slower than the CVE number request.
+
+## Vulnerability Policies
+
+Backstage uses Snyk vulnerability scans in order to make sure we minimize vulnerabilities in our dependencies and get notified of new vulnerabilities.
+
+There are many situations where a vulnerability does not affect a particular dependency because of how the vulnerable package is used. In that situation the package authors may choose to stay at the current version rather than bumping the dependency, leading to a warning in the vulnerability scans but no actual vulnerability.
+
+To work around this and other similar issues, Snyk provides a method to ignore vulnerabilities. In order to provide the best visibility and most utility to adopters of Backstage, we store these ignore rules in `.snyk` policy files. This allows adopters to rely on our ignore policies if they wish to do so.
+
+Adding a new ignore policy is done by creating or modifying an existing `.snyk` file within a package root. See the [Snyk Documentation](https://support.snyk.io/hc/en-us/articles/360007487097-The-snyk-file) for details on the syntax. Always include a description, full path, and time limit of the ignore policy.

app-config.yaml

@@ -101,8 +101,10 @@ organization:
 # https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-basic-to-recommended-deployment-approach
 techdocs:
   builder: 'local' # Alternatives - 'external'
-  generators:
-    techdocs: 'docker' # Alternatives - 'local'
+  generator:
+    runIn: 'docker'
+    # dockerImage: my-org/techdocs # use a custom docker image
+    # pullImage: true # or false to disable automatic pulling of image (e.g. if custom docker login is required)
   publisher:
     type: 'local' # Alternatives - 'googleGcs' or 'awsS3' or 'azureBlobStorage' or 'openStackSwift'. Read documentation for using alternatives.
 
@@ -131,6 +133,9 @@ kafka:
       brokers:
         - localhost:9092
 
+allure:
+  baseUrl: http://localhost:5050/allure-docker-service
+
 integrations:
   github:
     - host: github.com
@@ -150,12 +155,21 @@ integrations:
     - host: bitbucket.org
       username: ${BITBUCKET_USERNAME}
       appPassword: ${BITBUCKET_APP_PASSWORD}
+    ### Example for how to add your bitbucket server instance using the API:
+    # - host: server.bitbucket.com
+    #   apiBaseUrl: server.bitbucket.com
+    #   username: ${BITBUCKET_SERVER_USERNAME}
+    #   appPassword: ${BITBUCKET_SERVER_APP_PASSWORD}
   azure:
     - host: dev.azure.com
       token: ${AZURE_TOKEN}
-#  googleGcs:
-#    clientEmail: 'example@example.com'
-#    privateKey: ${GCS_PRIVATE_KEY}
+  #  googleGcs:
+  #    clientEmail: 'example@example.com'
+  #    privateKey: ${GCS_PRIVATE_KEY}
+  awsS3:
+    - host: amazonaws.com
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
 
 catalog:
   rules:
@@ -254,6 +268,8 @@ scaffolder:
   # defaultAuthor:
   #   name: Scaffolder
   #   email: scaffolder@backstage.io
+  # Use to customize the default commit message when new components are created
+  # defaultCommitMessage: 'Initial commit'
   github:
     token: ${GITHUB_TOKEN}
     visibility: public # or 'internal' or 'private'
@@ -296,6 +312,7 @@ auth:
     saml:
       entryPoint: 'http://localhost:7001/'
       issuer: 'passport-saml'
+      cert: 'fake-cert-base64'
     okta:
       development:
         clientId: ${AUTH_OKTA_CLIENT_ID}
@@ -312,13 +329,20 @@ auth:
         #
         # scope: saml-login-selector openid profile email
     oidc:
+      # Note that you must define a session secret (see above) since the oidc provider requires session support.
+      # Note that by default, this provider will use the 'none' prompt which assumes that your are already logged on in the IDP.
+      # You should set prompt to:
+      # - auto: will let the IDP decide if you need to log on or if you can skip login when you have an active SSO session
+      # - login: will force the IDP to always present a login form to the user
       development:
         metadataUrl: ${AUTH_OIDC_METADATA_URL}
         clientId: ${AUTH_OIDC_CLIENT_ID}
         clientSecret: ${AUTH_OIDC_CLIENT_SECRET}
         authorizationUrl: ${AUTH_OIDC_AUTH_URL}
         tokenUrl: ${AUTH_OIDC_TOKEN_URL}
-        tokenSignedResponseAlg: ${AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG}
+        tokenSignedResponseAlg: ${AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG} # default='RS256'
+        scope: ${AUTH_OIDC_SCOPE} # default='openid profile email'
+        prompt: ${AUTH_OIDC_PROMPT} # default=none (allowed values: auto, none, consent, login)
     auth0:
       development:
         clientId: ${AUTH_AUTH0_CLIENT_ID}
@@ -370,3 +394,9 @@ homepage:
       timezone: 'Asia/Tokyo'
 pagerduty:
   eventsBaseUrl: 'https://events.pagerduty.com/v2'
+jenkins:
+  instances:
+    - name: default
+      baseUrl: https://jenkins.example.com
+      username: backstage-bot
+      apiKey: 123456789abcdef0123456789abcedf012

catalog-info.yaml

@@ -15,7 +15,7 @@ metadata:
       url: https://discord.com/invite/EBHEGzX
   annotations:
     github.com/project-slug: backstage/backstage
-    backstage.io/techdocs-ref: url:https://github.com/backstage/backstage
+    backstage.io/techdocs-ref: dir:.
     lighthouse.com/website-url: https://backstage.io
 spec:
   type: library

contrib/docker/frontend-with-nginx/Dockerfile.hostbuild

@@ -8,7 +8,6 @@
 # simply copies in the build output into the image. If you want to also perform
 # the build itself inside docker, use Dockerfile.build in this folder instead.
 
-
 # USAGE:
 #
 # - Copy this file and the "docker" folder from this directory to your project
@@ -23,7 +22,7 @@
 #
 #       yarn install
 #       yarn tsc
-#       yarn build --config <config1> --config <config2> ...
+#       yarn workspace app build --config <config1> --config <config2> ...
 #       docker build -t backstage-frontend -f Dockerfile.hostbuild .
 
 
@@ -38,4 +37,3 @@ COPY docker/default.conf.template /etc/nginx/templates/default.conf.template
 COPY docker/inject-config.sh /docker-entrypoint.d/40-inject-config.sh
 
 ENV PORT 80
-

contrib/docs/tutorials/aws-deployment.md

@@ -5,7 +5,7 @@ image](https://backstage.io/docs/getting-started/deployment-docker); this
 tutorial shows how to deploy that Docker image to AWS using Elastic Container
 Registry (ECR) and Elastic Kubernetes Service (EKS). Amazon also supports
 deployments with Helm, covered in the [Helm
-Kubernetes](../kubernetes/basic_kubernetes_example_with_helm) example.
+Kubernetes](../../kubernetes/basic_kubernetes_example_with_helm) example.
 
 The basic workflow for this method is to build a Backstage Docker image, upload
 the new version to a container registry, and update a Kubernetes deployment with

docs/.gitignore

@@ -0,0 +1,2 @@
+# This is generated by build:api-docs in the root
+reference

docs/api/utility-apis.md

@@ -12,25 +12,32 @@ however always be a need for plugins to communicate outside of its boundaries,
 both with other plugins and the app itself.
 
 Backstage provides two primary methods for plugins to communicate across their
-boundaries in client-side code. The first one being the `createPlugin` API and
-the registration hooks passed to the `register` method, and the second one being
-Utility APIs. While the `createPlugin` API is focused on the initialization
-plugins and the app, the Utility APIs provide ways for plugins to communicate
-during their entire life cycle.
+boundaries in client-side code. The first one being the
+[createPlugin](../reference/core-plugin-api.createplugin.md) API along with the
+extensions that it can provide, and the second one being Utility APIs. While the
+[createPlugin](../reference/core-plugin-api.createplugin.md) API is focused on
+the initialization plugins and the app, the Utility APIs provide ways for
+plugins to communicate during their entire life cycle.
 
 ## Consuming APIs
 
-Each Utility API is tied to an `ApiRef` instance, which is a global singleton
-object without any additional state or functionality, its only purpose is to
-reference Utility APIs. `ApiRef`s are created using `createApiRef`, which is
-exported by `@backstage/core-plugin-api`. There are many
-[predefined Utility APIs](../reference/utility-apis/README.md) defined in
-`@backstage/core-plugin-api`, and they're all exported with a name of the
-pattern `*ApiRef`, for example `errorApiRef`.
+Each Utility API is tied to an [ApiRef](../reference/core-plugin-api.apiref.md)
+instance, which is a global singleton object without any additional state or
+functionality, its only purpose is to reference Utility APIs.
+[ApiRef](../reference/core-plugin-api.apiref.md)s are created using
+[createApiRef](../reference/core-plugin-api.createapiref.md), which is exported
+by [@backstage/core-plugin-api](../reference/core-plugin-api.md). There are also
+many predefined Utility APIs in
+[@backstage/core-plugin-api](../reference/core-plugin-api.md), and they're all
+exported with a name of the pattern `*ApiRef`, for example
+[errorApiRef](../reference/core-plugin-api.errorapiref.md).
 
-To access one of the Utility APIs inside a React component, use the `useApi`
-hook exported by `@backstage/core-plugin-api`, or the `withApis` HOC if you
-prefer class components. For example, the `ErrorApi` can be accessed like this:
+To access one of the Utility APIs inside a React component, use the
+[useApi](../reference/core-plugin-api.useapi.md) hook exported by
+[@backstage/core-plugin-api](../reference/core-plugin-api.md), or the
+[withApis](../reference/core-plugin-api.withapis.md) HOC if you prefer class
+components. For example, the
+[ErrorApi](../reference/core-plugin-api.errorapi.md) can be accessed like this:
 
 ```tsx
 import React from 'react';
@@ -48,24 +55,31 @@ export const MyComponent = () => {
 };
 ```
 
-Note that there is no explicit type given for `ErrorApi`. This is because the
-`errorApiRef` has the type embedded, and `useApi` is able to infer the type.
+Note that there is no explicit type given for
+[ErrorApi](../reference/core-plugin-api.errorapi.md). This is because the
+[errorApiRef](../reference/core-plugin-api.errorapiref.md) has the type
+embedded, and [useApi](../reference/core-plugin-api.useapi.md) is able to infer
+the type.
 
 Also note that consuming Utility APIs is not limited to plugins, it can be done
 from any component inside Backstage, including the ones in
-`@backstage/core-plugin-api`. The only requirement is that they are beneath the
-`AppProvider` in the react tree.
+[@backstage/core-plugin-api](../reference/core-plugin-api.md). The only
+requirement is that they are beneath the `AppProvider` in the react tree.
 
 ## Supplying APIs
 
 ### API Factories
 
-APIs are registered in the form of `ApiFactories`, which encapsulate the process
-of instantiating an API. It is a collection of three things: the `ApiRef` of the
-API to instantiate, a list of all required dependencies, and a factory function
-that returns a new API instance.
+APIs are registered in the form of
+[ApiFactories](../reference/core-plugin-api.apifactory.md), which encapsulate
+the process of instantiating an API. It is a collection of three things: the
+[ApiRef](../reference/core-plugin-api.apiref.md) of the API to instantiate, a
+list of all required dependencies, and a factory function that returns a new API
+instance.
 
-For example, this is the default `ApiFactory` for the `ErrorApi`:
+For example, this is the default
+[ApiFactory](../reference/core-plugin-api.apifactory.md) for the
+[ErrorApi](../reference/core-plugin-api.errorapi.md):
 
 ```ts
 createApiFactory({
@@ -79,18 +93,25 @@ createApiFactory({
 });
 ```
 
-In this example the `errorApiRef` is our API, which encapsulates the `ErrorApi`
-type. The `alertApiRef` is our single dependency, which we give the name
-`alertApi`, and is then passed on to the factory function, which returns an
-implementation of the `ErrorApi`.
+In this example the [errorApiRef](../reference/core-plugin-api.errorapiref.md)
+is our API, which encapsulates the
+[ErrorApi](../reference/core-plugin-api.errorapi.md) type. The
+[alertApiRef](../reference/core-plugin-api.alertapiref.md) is our single
+dependency, which we give the name `alertApi`, and is then passed on to the
+factory function, which returns an implementation of the
+[ErrorApi](../reference/core-plugin-api.errorapi.md).
 
-The `createApiFactory` function is a thin wrapper that enables TypeScript type
-inference. You may notice that there are no type annotations in the above
-example, and that is because we're able to infer all types from the `ApiRef`s.
-TypeScript will make sure that the return value of the `factory` function
-matches the type embedded in `api`'s `ApiRef`, in this case the `ErrorApi`. It
-will also match the types between the `deps` and the parameters of the `factory`
-function, again using the type embedded within the `ApiRef`s.
+The [createApiFactory](../reference/core-plugin-api.createapifactory.md)
+function is a thin wrapper that enables TypeScript type inference. You may
+notice that there are no type annotations in the above example, and that is
+because we're able to infer all types from the
+[ApiRef](../reference/core-plugin-api.apiref.md)s. TypeScript will make sure
+that the return value of the `factory` function matches the type embedded in
+`api`'s [ApiRef](../reference/core-plugin-api.apiref.md), in this case the
+[ErrorApi](../reference/core-plugin-api.errorapi.md). It will also match the
+types between the `deps` and the parameters of the `factory` function, again
+using the type embedded within the
+[ApiRef](../reference/core-plugin-api.apiref.md)s.
 
 ## Registering API Factories
 
@@ -102,24 +123,27 @@ app, and the app itself.
 
 Starting with the Backstage core library, it provides implementations for all of
 the core APIs. The core APIs are the ones exported by
-`@backstage/core-plugin-api`, such as the `errorApiRef` and `configApiRef`. You
-can find a full list of them [here](../reference/utility-apis/README.md).
+[@backstage/core-plugin-api](../reference/core-plugin-api.md), such as the
+[errorApiRef](../reference/core-plugin-api.errorapiref.md) and
+[configApiRef](../reference/core-plugin-api.configapiref.md).
 
-The core APIs are loaded for any app created with `createApp` from
-`@backstage/core-plugin-api`, which means that there is no step that needs to be
-taken to include these APIs in an app.
+The core APIs are loaded for any app created with
+[createApp](../reference/core-app-api.createapp.md) from
+[@backstage/core-plugin-api](../reference/core-plugin-api.md), which means that
+there is no step that needs to be taken to include these APIs in an app.
 
 ### Plugin APIs
 
 In addition to the core APIs, plugins can define and export their own APIs.
 While doing so they should usually also provide default implementations of their
 own APIs, for example, the `catalog` plugin exports `catalogApiRef`, and also
-supplies a default `ApiFactory` of that API using the `CatalogClient`. There is
-one restriction to plugin-provided API Factories: plugins may not supply
-factories for core APIs, trying to do so will cause the app to refuse to start.
+supplies a default [ApiFactory](../reference/core-plugin-api.apifactory.md) of
+that API using the `CatalogClient`. There is one restriction to plugin-provided
+API Factories: plugins may not supply factories for core APIs, trying to do so
+will cause the app to refuse to start.
 
-Plugins supply their APIs through the `apis` option of `createPlugin`, for
-example:
+Plugins supply their APIs through the `apis` option of
+[createPlugin](../reference/core-plugin-api.createplugin.md), for example:
 
 ```ts
 export const techdocsPlugin = createPlugin({
@@ -144,7 +168,8 @@ Lastly, the app itself is the final point where APIs can be added, and what has
 the final say in what APIs will be loaded at runtime. The app may override the
 factories for any of the core or plugin APIs, with the exception of the config,
 app theme, and identity APIs. These are static APIs that are tied into the
-`createApp` implementation, and therefore not possible to override.
+[createApp](../reference/core-app-api.createapp.md) implementation, and
+therefore not possible to override.
 
 Overriding APIs is useful for apps that want to switch out behavior to tailor it
 to their environment. In some cases plugins may also export multiple
@@ -206,16 +231,19 @@ const app = createApp({
 ```
 
 Note that the above line will cause an error if `IgnoreErrorApi` does not fully
-implement the `ErrorApi`, as it is checked by the type embedded in the
-`errorApiRef` at compile time.
+implement the [ErrorApi](../reference/core-plugin-api.errorapi.md), as it is
+checked by the type embedded in the
+[errorApiRef](../reference/core-plugin-api.errorapiref.md) at compile time.
 
 ## Defining custom Utility APIs
 
 Plugins are free to define their own Utility APIs. Simply define the TypeScript
-interface for the API, and create an `ApiRef` using `createApiRef` exported from
-`@backstage/core-plugin-api`. Also be sure to provide at least one
-implementation of the API, and to declare a default factory for the API in
-`createPlugin`.
+interface for the API, and create an
+[ApiRef](../reference/core-plugin-api.apiref.md) using
+[createApiRef](../reference/core-plugin-api.createapiref.md) exported from
+[@backstage/core-plugin-api](../reference/core-plugin-api.md). Also be sure to
+provide at least one implementation of the API, and to declare a default factory
+for the API in [createPlugin](../reference/core-plugin-api.createplugin.md).
 
 Custom Utility APIs can be either public or private, which is up to the plugin
 to choose. Private APIs do not expose an external API surface, and it's
@@ -226,15 +254,18 @@ plugin to override the API in the app. It is however important to maintain
 backwards compatibility of public APIs, as you may otherwise break apps that are
 using your plugin.
 
-To make an API public, simply export the `ApiRef` of the API, and any associated
-types. To make an API private, just avoid exporting the `ApiRef`, but still be
-sure to supply a default factory to `createPlugin`.
+To make an API public, simply export the
+[ApiRef](../reference/core-plugin-api.apiref.md) of the API, and any associated
+types. To make an API private, just avoid exporting the
+[ApiRef](../reference/core-plugin-api.apiref.md), but still be sure to supply a
+default factory to [createPlugin](../reference/core-plugin-api.createplugin.md).
 
 Private APIs are useful for plugins that want to depend on other APIs outside of
 React components, but not have to expose an entire API surface to maintain. When
 using private APIs, it is fine to use the `typeof` of an implementing class as
-the type parameter passed to `createApiRef`, while public APIs should always
-define a separate TypeScript interface type.
+the type parameter passed to
+[createApiRef](../reference/core-plugin-api.createapiref.md), while public APIs
+should always define a separate TypeScript interface type.
 
 Plugins may depend on APIs from other plugins, both in React components and as
 dependencies to API factories. Do however be sure to not cause circular
@@ -242,13 +273,14 @@ dependencies between plugins.
 
 ## Architecture
 
-The `ApiRef` instances mentioned above provide a point of indirection between
-consumers and producers of Utility APIs. It allows for plugins and components to
-depend on APIs in a type-safe way, without having a direct reference to a
-concrete implementation of the APIs. The Apps are also given a lot of
-flexibility in what implementations to provide. As long as they adhere to the
-contract established by an `ApiRef`, they are free to choose any implementation
-they want.
+The [ApiRef](../reference/core-plugin-api.apiref.md) instances mentioned above
+provide a point of indirection between consumers and producers of Utility APIs.
+It allows for plugins and components to depend on APIs in a type-safe way,
+without having a direct reference to a concrete implementation of the APIs. The
+Apps are also given a lot of flexibility in what implementations to provide. As
+long as they adhere to the contract established by an
+[ApiRef](../reference/core-plugin-api.apiref.md), they are free to choose any
+implementation they want.
 
 The figure below shows the relationship between
 <span style="color: #82b366">different Apps</span>, that provide
@@ -271,14 +303,17 @@ directly tied to React.
 The indirection provided by Utility APIs also makes it straightforward to test
 components that depend on APIs, and to provide a standard common development
 environment for plugins. A proper test wrapper with mocked API implementations
-is not yet ready, but it will be provided as a part of `@backstage/test-utils`.
-It will provide mocked variants of APIs, with additional methods for asserting a
-component's interaction with the API.
+is not yet ready, but it will be provided as a part of
+[@backstage/test-utils](../reference/test-utils.md). It will provide mocked
+variants of APIs, with additional methods for asserting a component's
+interaction with the API.
 
 The common development environment for plugins is included in
-`@backstage/dev-utils`, where the exported `createDevApp` function creates an
+[@backstage/dev-utils](../reference/dev-utils.md), where the exported
+[createDevApp](../reference/dev-utils.createdevapp.md) function creates an
 application with implementations for all core APIs already present. Contrary to
 the method for wiring up Utility API implementations in an app created with
-`createApp`, `createDevApp` uses automatic dependency injection. This is to make
-it possible to replace any API implementation, and having that be reflected in
-dependents of that API.
+[createApp](../reference/core-app-api.createapp.md),
+[createDevApp](../reference/dev-utils.createdevapp.md) uses automatic dependency
+injection. This is to make it possible to replace any API implementation, and
+having that be reflected in dependents of that API.

docs/auth/add-auth-provider.md

@@ -244,9 +244,7 @@ export interface OAuthProviderHandlers {
     req: express.Request,
     options: Record<string, string>,
   ): Promise<RedirectInfo>;
-  handler(
-    req: express.Request,
-  ): Promise<{
+  handler(req: express.Request): Promise<{
     response: AuthResponse<OAuthProviderInfo>;
     refreshToken?: string;
   }>;

docs/auth/github/provider.md

@@ -10,11 +10,16 @@ that can authenticate users using GitHub or GitHub Enterprise OAuth.
 
 ## Create an OAuth App on GitHub
 
-To add GitHub authentication, you must create an OAuth App from the GitHub
+To add GitHub authentication, you must create either a GitHub App, or an OAuth
+App from the GitHub
 [developer settings](https://github.com/settings/developers). The `Homepage URL`
 should point to Backstage's frontend, while the `Authorization callback URL`
 will point to the auth backend.
 
+Note that if you're using a GitHub App, the allowed scopes are configured as
+part of that app. This means you need to verify what scopes the plugins you use
+require, so be sure to check the plugin READMEs for that information.
+
 Settings for local development:
 
 - Application name: Backstage (or your custom app name)
@@ -51,3 +56,11 @@ The GitHub provider is a structure with three configuration keys:
 To add the provider to the frontend, add the `githubAuthApi` reference and
 `SignInPage` component as shown in
 [Adding the provider to the sign-in page](../index.md#adding-the-provider-to-the-sign-in-page).
+
+## Difference between GitHub Apps and GitHub OAuth Apps
+
+GitHub Apps handle OAuth scope at the app installation level, meaning that the
+`scope` parameter for the call to `getAccessToken` in the frontend has no
+effect. When calling `getAccessToken` in open source plugins, one should still
+include the appropriate scope, but also document in the plugin README what
+scopes are required for GitHub Apps.

docs/auth/identity-resolver.md

@@ -111,6 +111,48 @@ export default async function createPlugin({
 ...
 ```
 
+## Resolving membership through the catalog
+
+If you want to provide additional claims through Sign-In resolvers but still
+have the software catalog handle group (and transitive group) membership, you
+can do this using the `CatalogIdentityClient` provided as context to Sign-In
+resolvers:
+
+```ts
+export default async function createPlugin({
+  ...
+}: PluginEnvironment): Promise<Router> {
+  return await createRouter({
+    ...
+    providerFactories: {
+      google: createGoogleProvider({
+        signIn: {
+          resolver: async ({ profile: { email } }, ctx) => {
+            const [id] = email?.split('@') ?? '';
+            // Fetch from an external system that returns entity claims like:
+            // ['user:default/breanna.davison', ...]
+            const ent = await externalSystemClient.getUsernames(email);
+
+            // Resolve group membership from the Backstage catalog
+            const fullEnt = await ctx.catalogIdentityClient.resolveCatalogMembership({
+              entityRefs: [id].concat(ent),
+              logger: ctx.logger,
+            });
+            const token = await ctx.tokenIssuer.issueToken({
+              claims: { sub: id, ent: fullEnt },
+            });
+            return { id, token };
+          },
+        },
+      }),
+      ...
+```
+
+The `resolveCatalogMembership` method will retrieve the referenced entities from
+the catalog, if possible, and check for
+[memberOf](../features/software-catalog/well-known-relations.md#memberof-and-hasmember)
+relations to add additional entity claims.
+
 ## AuthHandler
 
 Similar to a custom sign-in resolver, you can also write a custom auth handler

docs/auth/index.md

@@ -60,9 +60,8 @@ small update to show this provider as a login option. The `SignInPage` component
 handles this, and takes either a `provider` or `providers` (array) prop of
 `SignInProviderConfig` definitions.
 
-These reference the [ApiRef](../reference/utility-apis/README.md) exported by
-the provider. Again, an example using GitHub that can be adapted to any of the
-built-in providers:
+These reference the `ApiRef` exported by the provider. Again, an example using
+GitHub that can be adapted to any of the built-in providers:
 
 ```diff
 # packages/app/src/App.tsx

docs/auth/microsoft/provider.md

@@ -44,7 +44,7 @@ The Microsoft provider is a structure with three configuration keys:
 
 - `clientId`: Application (client) ID, found on App Registration > Overview
 - `clientSecret`: Secret, found on App Registration > Certificates & secrets
-- `tenentId`: Directory (tenant) ID, found on App Registration > Overview
+- `tenantId`: Directory (tenant) ID, found on App Registration > Overview
 
 ## Adding the provider to the Backstage frontend
 

docs/auth/using-auth.md

@@ -28,7 +28,8 @@ OAuth helps in that regard.
 The method with which frontend plugins request access to third party services is
 through [Utility APIs](../api/utility-apis.md) for each service provider. For a
 full list of providers, see the
-[Utility API References](../reference/utility-apis/README.md).
+[@backstage/core-plugin-api](../reference/core-plugin-api.md#variables)
+reference.
 
 ### Identity - WIP
 

docs/conf/reading.md

@@ -7,7 +7,7 @@ description: Documentation on Reading Backstage Configuration
 ## Config API
 
 There's a common configuration API for by both frontend and backend plugins. An
-API reference can be found [here](../reference/utility-apis/Config.md).
+API reference can be found [here](../reference/config.config.md).
 
 The configuration API is tailored towards failing fast in case of missing or bad
 config. That's because configuration errors can always be considered programming
@@ -110,9 +110,18 @@ example `getString`. These will throw an error if there is no value available.
 
 ## Accessing ConfigApi in Frontend Plugins
 
-The [ConfigApi](../reference/utility-apis/Config.md) in the frontend is a
+The [ConfigApi](../reference/core-plugin-api.configapi.md) in the frontend is a
 [UtilityApi](../api/utility-apis.md). It's accessible as usual via the
-`configApiRef` exported from `@backstage/core-plugin-api`.
+`configApiRef` exported from `@backstage/core-plugin-api`:
+
+```
+import { useApi, configApiRef } from '@backstage/core-plugin-api';
+...
+const MyReactComponent = (...) => {
+  const config = useApi(configApiRef);
+  ...
+}
+```
 
 Depending on the config api in another API is slightly different though, as the
 `ConfigApi` implementation is supplied via the App itself and not instantiated

docs/deployment/docker.md

@@ -113,6 +113,9 @@ browser at `http://localhost:7000`
 
 ## Multi-stage Build
 
+> NOTE: The `.dockerignore` is different in this setup, read on for more
+> details.
+
 This section describes how to set up a multi-stage Docker build that builds the
 entire project within Docker. This is typically slower than a host build, but is
 sometimes desired because Docker in Docker is not available in the build
@@ -137,6 +140,7 @@ WORKDIR /app
 COPY package.json yarn.lock ./
 
 COPY packages packages
+# Comment this out if you don't have any internal plugins
 COPY plugins plugins
 
 RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -exec rm -rf {} \+
@@ -182,8 +186,9 @@ end up being properly installed.
 
 To speed up the build when not running in a fresh clone of the repo you should
 set up a `.dockerignore`. This one is different than the host build one, because
-we want to have access to the source code of all packages for the build, but can
-ignore any existing build output or dependencies:
+we want to have access to the source code of all packages for the build. We can
+however ignore any existing build output or dependencies on the host. For our
+new `.dockerignore`, replace the contents of your existing one with this:
 
 ```text
 node_modules

docs/features/search/architecture.md

@@ -4,8 +4,6 @@ title: Search Architecture
 description: Documentation on Search Architecture
 ---
 
-# Search Architecture
-
 > _This architecture has not been fully implemented yet. Find our milestones to
 > follow our progress and help contribute on the
 > [Search Roadmap](./README.md#project-roadmap)._

docs/features/search/concepts.md

@@ -4,8 +4,6 @@ title: Search Concepts
 description: Documentation on Backstage Search Concepts
 ---
 
-# Search Concepts
-
 Backstage Search lets you find the right information you are looking for in the
 Backstage ecosystem.
 

docs/features/search/getting-started.md

@@ -4,8 +4,6 @@ title: Getting Started with Search
 description: How to set up and install Backstage Search
 ---
 
-# Getting Started
-
 Search functions as a plugin to Backstage, so you will need to use Backstage to
 use Search.
 
@@ -252,13 +250,9 @@ an example:
 Backstage Search isn't a search engine itself, rather, it provides an interface
 between your Backstage instance and a
 [Search Engine](./concepts.md#search-engines) of your choice. Currently, we only
-support one, an in-memory search Engine called Lunr. It can be instantiated like
-this:
-
-```typescript
-const searchEngine = new LunrSearchEngine({ logger });
-const indexBuilder = new IndexBuilder({ logger, searchEngine });
-```
+support two engines, an in-memory search Engine called Lunr and ElasticSearch.
+See [Search Engines](./search-engines.md) documentation for more information how
+to configure these in your Backstage instance.
 
 Backstage Search can be used to power search of anything! Plugins like the
 Catalog offer default [collators](./concepts.md#collators) (e.g.

docs/features/search/search-engines.md

@@ -0,0 +1,158 @@
+---
+id: search-engines
+title: Search Engines
+description: Choosing and configuring your search engine for Backstage
+---
+
+Backstage supports 2 search engines by default, an in-memory engine called Lunr
+and ElasticSearch. You can configure your own search engines by implementing the
+provided interface as mentioned in the
+[search backend documentation.](./getting-started.md#Backend)
+
+Provided search engine implementations have their own way of constructing
+queries, which may be something you want to modify. Alterations to the querying
+logic of a search engine can be made by providing your own implementation of a
+QueryTranslator interface. This modification can be done without touching
+provided search engines by using the exposed setter to set the modified query
+translator into the instance.
+
+```typescript
+const searchEngine = new LunrSearchEngine({ logger });
+searchEngine.setTranslator(new MyNewAndBetterQueryTranslator());
+```
+
+## Lunr
+
+Lunr search engine is enabled by default for your backstage instance if you have
+not done additional changes to the scaffolded app.
+
+Lunr can be instantiated like this:
+
+```typescript
+// app/backend/src/plugins/search.ts
+const searchEngine = new LunrSearchEngine({ logger });
+const indexBuilder = new IndexBuilder({ logger, searchEngine });
+```
+
+## Postgres
+
+The Postgres based search engine only requires that postgres being configured as
+the database engine for Backstage. Therefore it targets setups that want to
+avoid maintaining another external service like elastic search. The search
+provides decent results and performs well with ten thousands of indexed
+documents. The connection to postgres is established via the database manager
+also used by other plugins.
+
+> **Important**: The search plugin requires at least Postgres 12!
+
+To use the `PgSearchEngine`, make sure that you have a Postgres database
+configured and make the following changes to your backend:
+
+1. Add a dependency on `@backstage/plugin-search-backend-module-pg` to your
+   backend's `package.json`.
+2. Initialize the search engine. It is recommended to initialize it with a
+   fallback to the lunr search engine if you are running Backstage for
+   development locally with SQLite:
+
+```typescript
+// In packages/backend/src/plugins/search.ts
+
+// Initialize a connection to a search engine.
+const searchEngine = (await PgSearchEngine.supported(database))
+  ? await PgSearchEngine.from({ database })
+  : new LunrSearchEngine({ logger });
+```
+
+## ElasticSearch
+
+Backstage supports ElasticSearch search engine connections, indexing and
+querying out of the box. Available configuration options enable usage of either
+AWS or Elastic.co hosted solutions, or a custom self-hosted solution.
+
+Similarly to Lunr above, ElasticSearch can be set up like this:
+
+```typescript
+// app/backend/src/plugins/search.ts
+const searchEngine = await ElasticSearchSearchEngine.initialize({
+  logger,
+  config,
+});
+const indexBuilder = new IndexBuilder({ logger, searchEngine });
+```
+
+For the engine to be available, your backend package needs a dependency into
+package `@backstage/plugin-search-backend-module-elasticsearch`.
+
+ElasticSearch needs some additional configuration before it is ready to use
+within your instance. The configuration options are documented in the
+[configuration schema definition file.](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-elasticsearch/config.d.ts)
+
+The underlying functionality is using official ElasticSearch client version 7.x,
+meaning that ElasticSearch version 7 is the only one confirmed to be supported.
+
+## Example configurations
+
+### AWS
+
+Using AWS hosted ElasticSearch the only configuration option needed is the URL
+to the ElasticSearch service. The implementation assumes that environment
+variables for AWS access key id and secret access key are defined in accordance
+to the
+[default AWS credential chain.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html).
+
+```yaml
+search:
+  elasticsearch:
+    provider: aws
+    node: https://my-backstage-search-asdfqwerty.eu-west-1.es.amazonaws.com
+```
+
+### Elastic.co
+
+Elastic Cloud hosted ElasticSearch uses a Cloud ID to determine the instance of
+hosted ElasticSearch to connect to. Additionally, username and password needs to
+be provided either directly or using environment variables like defined in
+[Backstage documentation.](https://backstage.io/docs/conf/writing#includes-and-dynamic-data)
+
+```yaml
+search:
+  elasticsearch:
+    provider: elastic
+    cloudId: backstage-elastic:asdfqwertyasdfqwertyasdfqwertyasdfqwerty==
+    auth:
+      username: elastic
+      password: changeme
+```
+
+### Others
+
+Other ElasticSearch instances can be connected to by using standard
+ElasticSearch authentication methods and exposed URL, provided that the cluster
+supports that. The configuration options needed are the URL to the node and
+authentication information. Authentication can be handled by either providing
+username/password or an API key. For more information how to create an API key,
+see
+[Elastic documentation on API keys](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html).
+
+#### Configuration examples
+
+##### With username and password
+
+```yaml
+search:
+  elasticsearch:
+    node: http://localhost:9200
+    auth:
+      username: elastic
+      password: changeme
+```
+
+##### With API key
+
+```yaml
+search:
+  elasticsearch:
+    node: http://localhost:9200
+    auth:
+      apiKey: base64EncodedKey
+```

docs/features/software-catalog/catalog-customization.md

@@ -27,27 +27,32 @@ default catalog page and create a component in a
 ```tsx
 // imports, etc omitted for brevity. for full source see:
 // https://github.com/backstage/backstage/blob/master/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx
-export const CustomCatalogPage = () => {
+export const CustomCatalogPage = ({
+  columns,
+  actions,
+  initiallySelectedFilter = 'owned',
+}: CatalogPageProps) => {
   return (
-    <CatalogLayout>
-      <Content>
-        <ContentHeader title="Components">
-          <CreateComponentButton />
-          <SupportButton>All your software catalog entities</SupportButton>
-        </ContentHeader>
-        <div className={styles.contentWrapper}>
-          <EntityListProvider>
-            <div>
-              <EntityKindPicker initialFilter="component" hidden />
+    <PageWithHeader title={`${orgName} Catalog`} themeId="home">
+      <EntityListProvider>
+        <Content>
+          <ContentHeader titleComponent={<CatalogKindHeader />}>
+            <CreateButton title="Create Component" to={link} />
+            <SupportButton>All your software catalog entities</SupportButton>
+          </ContentHeader>
+          <FilteredEntityLayout>
+            <FilterContainer>
               <EntityTypePicker />
-              <UserListPicker />
+              <UserListPicker initialFilter={initiallySelectedFilter} />
               <EntityTagPicker />
-            </div>
-            <CatalogTable />
-          </EntityListProvider>
-        </div>
-      </Content>
-    </CatalogLayout>
+            </FilterContainer>
+            <EntityListContainer>
+              <CatalogTable columns={columns} actions={actions} />
+            </EntityListContainer>
+          </FilteredEntityLayout>
+        </Content>
+      </EntityListProvider>
+    </PageWithHeader>
   );
 };
 ```
@@ -137,19 +142,27 @@ export const EntitySecurityTierPicker = () => {
 Now we can add the component to `CustomCatalogPage`:
 
 ```diff
-export const CustomCatalogPage = () => {
+export const CustomCatalogPage = ({
+  columns,
+  actions,
+  initiallySelectedFilter = 'owned',
+}: CatalogPageProps) => {
   return (
     ...
-          <EntityListProvider>
-            <div>
+        <EntityListProvider>
+          <FilteredEntityLayout>
+            <FilterContainer>
               <EntityKindPicker initialFilter="component" hidden />
               <EntityTypePicker />
-              <UserListPicker />
-+              <EntitySecurityTierPicker />
+              <UserListPicker initialFilter={initiallySelectedFilter} />
++             <EntitySecurityTierPicker />
               <EntityTagPicker />
-            </div>
-            <CatalogTable />
-          </EntityListProvider>
+            <FilterContainer>
+            <EntityListContainer>
+              <CatalogTable columns={columns} actions={actions} />
+            </EntityListContainer>
+          </FilteredEntityLayout>
+        </EntityListProvider>
     ...
 };
 ```
@@ -173,7 +186,7 @@ new `CustomCatalogIndexPage`.
 # packages/app/src/App.tsx
 const routes = (
   <FlatRoutes>
-    <Navigate key="/" to="/catalog" />
+    <Navigate key="/" to="catalog" />
 -    <Route path="/catalog" element={<CatalogIndexPage />} />
 +    <Route path="/catalog" element={<CustomCatalogIndexPage />} />
 ```

docs/features/software-catalog/descriptor-format.md

@@ -245,6 +245,22 @@ the entity belongs to the `"default"` namespace.
 Namespaces may also be part of the catalog, and are `v1` / `Namespace` entities,
 i.e. not Backstage specific but the same as in Kubernetes.
 
+### `title` [optional]
+
+A display name of the entity, to be presented in user interfaces instead of the
+`name` property above, when available.
+
+This field is sometimes useful when the `name` is cumbersome or ends up being
+perceived as overly technical. The title generally does not have as stringent
+format requirements on it, so it may contain special characters and be more
+explanatory. Do keep it very short though, and avoid situations where a title
+can be confused with the name of another entity, or where two entities share a
+title.
+
+Note that this is only for display purposes, and may be ignored by some parts of
+the code. [Entity references](references.md) still always make use of the `name`
+property for example, not the title.
+
 ### `description` [optional]
 
 A human readable description of the entity, to be shown in Backstage. Should be
@@ -531,8 +547,8 @@ The current set of well-known and common values for this field is:
 
 ### `spec.owner` [required]
 
-An [entity reference](#string-references) to the owner of the component, e.g.
-`artist-relations-team`. This field is required.
+An [entity reference](references.md#string-references) to the owner of the
+component, e.g. `artist-relations-team`. This field is required.
 
 In Backstage, the owner of a component is the singular entity (commonly a team)
 that bears ultimate responsibility for the component, and has the authority and
@@ -550,8 +566,8 @@ component, but there will always be one ultimate owner.
 
 ### `spec.system` [optional]
 
-An [entity reference](#string-references) to the system that the component
-belongs to, e.g. `artist-engagement-portal`. This field is optional.
+An [entity reference](references.md#string-references) to the system that the
+component belongs to, e.g. `artist-engagement-portal`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                            |
 | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- |
@@ -559,8 +575,8 @@ belongs to, e.g. `artist-engagement-portal`. This field is optional.
 
 ### `spec.subcomponentOf` [optional]
 
-An [entity reference](#string-references) to another component of which the
-component is a part, e.g. `spotify-ios-app`. This field is optional.
+An [entity reference](references.md#string-references) to another component of
+which the component is a part, e.g. `spotify-ios-app`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required)  | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                            |
 | ---------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- |
@@ -568,8 +584,8 @@ component is a part, e.g. `spotify-ios-app`. This field is optional.
 
 ### `spec.providesApis` [optional]
 
-An array of [entity references](#string-references) to the APIs that are
-provided by the component, e.g. `artist-api`. This field is optional.
+An array of [entity references](references.md#string-references) to the APIs
+that are provided by the component, e.g. `artist-api`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                                                  |
 | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------- |
@@ -577,8 +593,8 @@ provided by the component, e.g. `artist-api`. This field is optional.
 
 ### `spec.consumesApis` [optional]
 
-An array of [entity references](#string-references) to the APIs that are
-consumed by the component, e.g. `artist-api`. This field is optional.
+An array of [entity references](references.md#string-references) to the APIs
+that are consumed by the component, e.g. `artist-api`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                                                  |
 | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------- |
@@ -586,9 +602,9 @@ consumed by the component, e.g. `artist-api`. This field is optional.
 
 ### `spec.dependsOn` [optional]
 
-An array of [entity references](#string-references) to the components and
-resources that the component depends on, e.g. `artists-db`. This field is
-optional.
+An array of [entity references](references.md#string-references) to the
+components and resources that the component depends on, e.g. `artists-db`. This
+field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                                            |
 | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------- |
@@ -690,12 +706,6 @@ shape, this kind has the following structure.
 
 Exactly equal to `backstage.io/v1beta2` and `Template`, respectively.
 
-### `metadata.title` [required]
-
-The nice display name for the template as a string, e.g. `React SSR Template`.
-This field is required as is used to reference the template to the user instead
-of the `metadata.name` field.
-
 ### `metadata.tags` [optional]
 
 A list of strings that can be associated with the template, e.g.
@@ -722,8 +732,8 @@ You can find out more about the `steps` key
 
 ### `spec.owner` [optional]
 
-An [entity reference](#string-references) to the owner of the template, e.g.
-`artist-relations-team`. This field is required.
+An [entity reference](references.md#string-references) to the owner of the
+template, e.g. `artist-relations-team`. This field is required.
 
 In Backstage, the owner of a Template is the singular entity (commonly a team)
 that bears ultimate responsibility for the Template, and has the authority and
@@ -832,8 +842,8 @@ The current set of well-known and common values for this field is:
 
 ### `spec.owner` [required]
 
-An [entity reference](#string-references) to the owner of the component, e.g.
-`artist-relations-team`. This field is required.
+An [entity reference](references.md#string-references) to the owner of the
+component, e.g. `artist-relations-team`. This field is required.
 
 In Backstage, the owner of an API is the singular entity (commonly a team) that
 bears ultimate responsibility for the API, and has the authority and capability
@@ -851,8 +861,8 @@ one ultimate owner.
 
 ### `spec.system` [optional]
 
-An [entity reference](#string-references) to the system that the API belongs to,
-e.g. `artist-engagement-portal`. This field is optional.
+An [entity reference](references.md#string-references) to the system that the
+API belongs to, e.g. `artist-engagement-portal`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                            |
 | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- |
@@ -1059,8 +1069,8 @@ Exactly equal to `backstage.io/v1alpha1` and `Resource`, respectively.
 
 ### `spec.owner` [required]
 
-An [entity reference](#string-references) to the owner of the resource, e.g.
-`artist-relations-team`. This field is required.
+An [entity reference](references.md#string-references) to the owner of the
+resource, e.g. `artist-relations-team`. This field is required.
 
 In Backstage, the owner of a resource is the singular entity (commonly a team)
 that bears ultimate responsibility for the resource, and has the authority and
@@ -1091,8 +1101,8 @@ Some common values for this field could be:
 
 ### `spec.system` [optional]
 
-An [entity reference](#string-references) to the system that the resource
-belongs to, e.g. `artist-engagement-portal`. This field is optional.
+An [entity reference](references.md#string-references) to the system that the
+resource belongs to, e.g. `artist-engagement-portal`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                            |
 | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- |
@@ -1100,9 +1110,9 @@ belongs to, e.g. `artist-engagement-portal`. This field is optional.
 
 ### `spec.dependsOn` [optional]
 
-An array of [entity references](#string-references) to the components and
-resources that the resource depends on, e.g. `artist-lookup`. This field is
-optional.
+An array of [entity references](references.md#string-references) to the
+components and resources that the resource depends on, e.g. `artist-lookup`.
+This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                                            |
 | --------------------------------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------- |
@@ -1146,8 +1156,8 @@ Exactly equal to `backstage.io/v1alpha1` and `System`, respectively.
 
 ### `spec.owner` [required]
 
-An [entity reference](#string-references) to the owner of the system, e.g.
-`artist-relations-team`. This field is required.
+An [entity reference](references.md#string-references) to the owner of the
+system, e.g. `artist-relations-team`. This field is required.
 
 In Backstage, the owner of a system is the singular entity (commonly a team)
 that bears ultimate responsibility for the system, and has the authority and
@@ -1165,8 +1175,8 @@ but there will always be one ultimate owner.
 
 ### `spec.domain` [optional]
 
-An [entity reference](#string-references) to the domain that the system belongs
-to, e.g. `artists`. This field is optional.
+An [entity reference](references.md#string-references) to the domain that the
+system belongs to, e.g. `artists`. This field is optional.
 
 | [`kind`](#apiversion-and-kind-required) | Default [`namespace`](#namespace-optional) | Generated [relation](well-known-relations.md) type                            |
 | --------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------- |
@@ -1205,8 +1215,8 @@ Exactly equal to `backstage.io/v1alpha1` and `Domain`, respectively.
 
 ### `spec.owner` [required]
 
-An [entity reference](#string-references) to the owner of the domain, e.g.
-`artist-relations-team`. This field is required.
+An [entity reference](references.md#string-references) to the owner of the
+domain, e.g. `artist-relations-team`. This field is required.
 
 In Backstage, the owner of a domain is the singular entity (commonly a team)
 that bears ultimate responsibility for the domain, and has the authority and

docs/features/software-catalog/extending-the-model.md

@@ -393,3 +393,39 @@ of interest to you, you might consider contacting the maintainers on Discord or
 my making a GitHub issue describing your use case.
 [This issue](https://github.com/backstage/backstage/issues/2292) also contains
 more context.
+
+## Referencing different environments with the model
+
+Example intent:
+
+> "I have multiple versions of my API deployed in different environments so I
+> want to have `mytool-dev` and `mytool-prod` as different entities."
+
+While it's possible to have different versions of the same thing represented as
+separate entities, it's something we generally recommend against. We believe
+that a developer should be able to just find for example one `Component`
+representing a service, and to be able to see the different code versions that
+are deployed throughout your stack within its view. This reasoning works
+similarly for other kinds as well, such as `API`.
+
+That being said - sometimes the differences between versions are so large, that
+they represent what is for all intents and purposes an entirely new entity as
+seen from the consumer's point of view. This can happen for example for
+different _significant_ major versions of an API, and in particular if the two
+major versions coexist in the ecosystem for some time. In those cases, it can be
+motivated to have one `my-api-v2` and one `my-api-v3` named entity. This matches
+the end user's expectations when searching for the API, and matches the desire
+to maybe have separate documentation for the two and similar. But use this
+sparingly - only do it if the extra modelling burden is outweighed by any
+potential better clarity for users.
+
+When writing your custom plugins, we encourage designing them such that they can
+show all the different variations through environments etc under one canonical
+reference to your software in the catalog. For example for a continuous
+deployment plugin, a user is likely to be greatly helped by being able to see
+the entity's versions deployed in all different environments next to each other
+in one view. That is also where they might be offered the ability to promote
+from one environment to the other, do rollbacks, see their relative performance
+metrics, and similar. This coherency and collection of tooling in one place is
+where something like Backstage can offer the most value and effectiveness of
+use. Splitting your entities apart into small islands makes this harder.

docs/features/software-catalog/index.md

@@ -8,12 +8,11 @@ description: The Backstage Software Catalog
 
 ## What is a Software Catalog?
 
-The Backstage Software Catalog — actually, a software catalog, since it includes
-more than just services — is a centralized system that keeps track of ownership
-and metadata for all the software in your ecosystem (services, websites,
-libraries, data pipelines, etc). The catalog is built around the concept of
-[metadata YAML files](descriptor-format.md) stored together with the code, which
-are then harvested and visualized in Backstage.
+The Backstage Software Catalog is a centralized system that keeps track of
+ownership and metadata for all the software in your ecosystem (services,
+websites, libraries, data pipelines, etc). The catalog is built around the
+concept of [metadata YAML files](descriptor-format.md) stored together with the
+code, which are then harvested and visualized in Backstage.
 
 ![software-catalog](https://backstage.io/blog/assets/6/header.png)
 

docs/features/software-catalog/life-of-an-entity.md

@@ -0,0 +1,246 @@
+---
+id: life-of-an-entity
+title: The Life of an Entity
+sidebar_label: The Life of an Entity
+# prettier-ignore
+description: The life cycle of entities, from being introduced into the catalog, through processing, to being removed again
+---
+
+This document gives a high level overview of the catalog backend, and the
+technical processes involved in making entities flow through it. It is mainly
+aimed at developers who want to understand the internals while installing or
+extending the catalog. However, it can be informative for other personas too.
+
+## Key Concepts
+
+The catalog forms a hub of sorts, where entities are ingested from various
+authoritative sources and held in a database, subject to automated processing,
+and then presented through an API for quick and easy access by Backstage and
+others. The most common source is [YAML files](descriptor-format.md) on a
+standard format, living in version control systems near the source code of
+systems that they describe. Those files are registered with the catalog and
+maintained by the respective owners. The catalog makes sure to keep itself up to
+date with changes to those files.
+
+The main extension points where developers can customize the catalog are:
+
+- _Entity providers_, that feed initial raw entity data into the catalog,
+- _Policies_, that establish baseline rules about the shape of entities,
+- _Processors_, that validate, analyze, and mutate the raw entity data into its
+  final form.
+
+The high level processes involved are:
+
+- _Ingestion_, where entity providers fetch raw entity data from external
+  sources and seed it into the database,
+- _Processing_, where the policies and processors continually treat the ingested
+  data and may emit both other raw entities (that are also subject to
+  processing), errors, relations to other entities, etc.,
+- _Stitching_, where all of the data emitted by various processors are assembled
+  together into the final output entity.
+
+An entity is not visible to the outside world (through the catalog API), until
+it has passed through the last process and landed among the final entities.
+
+![General overview](../../assets/features/catalog/life-of-an-entity_overview.svg)
+
+The details of these processes are described below.
+
+## Ingestion
+
+Each catalog deployment has a number of entity providers installed. They are
+responsible for fetching data from external authoritative sources in any way
+that they see fit, to translate those into entity objects, and to notify the
+database when those entities are added or removed. These are the _unprocessed
+entities_ that will be subject to later processing (see below), and they form
+the very basis of existence for entities. If there were no entity providers, no
+entities would ever enter the system.
+
+The database always keeps track of the set of entities that belong to each
+provider; no two providers can try to output the same entity. And when a
+provider signals the removal of an entity, then that leads to an _eager
+deletion_: the entity and all auxiliary data that it has led to in the database
+is immediately purged.
+
+![Ingestion overview](../../assets/features/catalog/life-of-an-entity_ingestion.svg)
+
+There are two providers installed by default: the one that deals with user
+registered locations (e.g. URLs to YAML files), and the one that deals with
+static locations in the app-config. You can add more third party providers by
+passing them to the catalog builder in your backend initialization code, and you
+can easily write your own.
+
+An entity provider is a class that implements the `EntityProvider` interface. It
+has three main parts:
+
+- The identity: Each provider instance has a unique, stable identifier that the
+  database can use to keep track of the originator of each unprocessed entity.
+- The connection: During backend startup, each provider is attached to the
+  catalog runtime.
+- The stream of events: During its lifetime, the provider can issue change
+  events to the runtime at any point in time, to modify its set of unprocessed
+  entities.
+
+It is entirely up to the provider to choose how and when it produces these
+change events. For example, the app-config provider only fires off an update at
+startup and then lies dormant. The location database provider does an initial
+update at startup, and then small delta updates every time a location database
+change is detected. The LDAP provider is driven externally by a timer loop that
+occasionally triggers a full update. Some future provider may be entirely event
+driven, feeding off an event bus or web hook. There is no magic coordination
+among providers; if they need to arrange synchronization or locking among
+themselves for example to avoid duplicate work across multiple catalog service
+machines, they need to handle that out-of-band.
+
+The entities that are emitted get some coarse validation applied to them, to
+ensure that they at least adhere to the most basic schema rules about how an
+entity should be shaped. For example, they need to have a `kind`, a
+`metadata.name`, and optionally a `metadata.namespace`, among others. Apart from
+that, the ingestion stage considers its work done, and stores the unprocessed
+entities to be picked up at a later time by the processing system. This means
+that the more precise validation rules that you put in place on entities are
+_not_ yet applied at this stage.
+
+## Processing
+
+Every unprocessed entity comes with a timestamp, which tells at what time that
+the processing loop should next try to process it. When the entity first
+appears, this timestamp is set to "now" - asking for it to be picked up as soon
+as possible.
+
+Each catalog deployment has a number of processors installed. They are
+responsible for receiving unprocessed entities that the catalog decided are due
+for processing, and then running that data through a number of processing
+stages, mutating the entity and emitting auxiliary data about it. When all of
+that is done, the catalog takes all of that information and stores it as the
+processed entity, and errors and relations to other entities separately. Then,
+the catalog checks to see what entities are touched by that output, and triggers
+the final assembly of those (see Stitching below).
+
+![Processing overview](../../assets/features/catalog/life-of-an-entity_processing.svg)
+
+Entities are always processed one by one, but all of your catalog service hosts
+collaborate in doing so to distribute the load. Note how each processor can
+contribute to one or more of the fixed steps in the processing pipeline. First
+all of the processors' contributions to one step are run in the order that the
+processors were registered, then all of their contributions to the next step in
+the same order, and so on.
+
+Each step has the opportunity to optionally modify the entity, and to optionally
+emit other information. For example, the processor might look at information in
+the `spec` field of the entity, and emit relations that correspond to those
+declarations. If the processor emits an entity, then that entity gets stored
+verbatim with a timestamp saying that it, too, should be processed as soon as
+possible. If errors are emitted, then that signals that something is wrong with
+the entity and that it should not replace whatever previously error-free version
+we had among the final entities. If relations are emitted, then they are put in
+a dedicated relations table to be picked up by the stitching process below.
+
+> Optional low level detail note: When entities are emitted, the catalog keeps
+> track of the edges between the emitting entity and the ones emitted. This
+> happens behind the scenes, hidden from the outside, and is used to form a
+> graph. This is _not_ the same thing as relations! The purpose of these edges,
+> is to be able to detect when an entity becomes orphaned (see below), and to be
+> able to perform eager deletions throughout the graph when a root is explicitly
+> unregistered and nothing else is keeping lower nodes alive. We will talk more
+> about orphaning and deletions later on in this article.
+
+When the final step has completed, and no errors were encountered, the processed
+entity and all of the relations are finally persisted in the database. Then the
+catalog considers this entity, and all of the entities it had relations to,
+subject for stitching.
+
+It is worth noting here that the processing does not lead to deletion or
+unregistration of entities; it can only call new entities into existence or
+update entities that it has previously called into existence. More about that
+later.
+
+## Stitching
+
+Stitching finalizes the entity, by gathering all of the output from the previous
+steps and merging them into the final object which is what is visible from the
+catalog API. As the final entity itself gets updated, the stitcher makes sure
+that the search table gets refreshed accordingly as well.
+
+> Note: The search table mentioned here is not related to the core Search
+> feature of Backstage. It's rather the table that backs the ability to filter
+> catalog API query results.
+
+![Stitching overview](../../assets/features/catalog/life-of-an-entity_stitching.svg)
+
+The diagram shows how the stitcher reads from several sources:
+
+- The processed entity, as returned from the processing step
+- The errors, if any, that were emitted by the processing step
+- All relations that were emitted by the processing step, as well as any
+  relations emitted by _other_ entity processing steps that happen to point at
+  the current entity
+
+The last part is noteworthy: This is how the stitcher is able to collect all of
+the relation edges, both incoming and outgoing, no matter who produced them.
+
+The stitching is currently a fixed process, that cannot be modified or extended.
+This means that any modifications you want to make on the final result, has to
+happen during ingestion or processing.
+
+## Errors
+
+> TODO: Describe how errors are exposed through entities
+
+## Orphaning
+
+As mentioned earlier, entities internally form a graph. The edges go from
+processed parent entities, to child entities emitted while processing the
+parent.
+
+The processing loop runs continuously, so these edges are reconsidered over
+time. If processing a parent entity no longer emits a given child entity, then
+that former edge is severed. If that child has no other edges pointing at it
+either, it becomes _orphaned_. The end result is as follows:
+
+- The stitching process injects a `backstage.io/orphan: 'true'` annotation on
+  the child entity.
+- The child entity is _not_ removed from the catalog, but stays around until
+  explicitly deleted via the catalog API, or "reclaimed" by the original parent
+  or another parent starting to reference it.
+- The catalog page in Backstage for the child entity detects the new annotation
+  and informs users about the orphan status.
+
+Orphaning can occur in several different scenarios. One common cause is that the
+end user edited a corresponding catalog catalog-info YAML file removing the
+entity's entry. In the case of a `Location` parent entity, orphaning can happen
+if removing the target line pointing to the file containing the child entity.
+Another common cause is large batch processors such as the ones that crawl
+through remote systems looking for entities, no longer finding something that it
+used to find before. Maybe the data was moved, or deleted, in the remote system.
+So for example when a person leaves the company an LDAP org discovery processor
+might leave an orphaned `User` entity behind. Note that this only applies to
+processors - ingestion that happens using entity providers work differently,
+described below.
+
+> Note that removing a file, or accidentally corrupting a file so that it cannot
+> be read successfully, does _not_ lead to orphaning. Hard errors, including the
+> inability to find or read a distinct remote, are marked as such on the entity
+> to inform the owner that something is wrong. But processing and other
+> behaviors continue as usual.
+
+The reason that the orphaning mechanism exists instead of having an eager
+deletion triggered, is safety. Scenarios like these can happen purely by
+accident, due to the asynchronous nature of the system and the fallible nature
+of humans. In particular when external systems start consuming and relying on
+the catalog, there could be substantial consequences to suddenly dropping
+entities without explicit owner consent. The catalog therefore takes the stance
+that entities that often were added by direct user action should also be deleted
+only by direct user action.
+
+It is possible to use the catalog API to build automated "reaper" systems that
+finally delete entities that are orphaned. This is however not something that's
+provided out of the box.
+
+## Implicit Deletion
+
+> TODO: Describe the process of entity providers eagerly deleting entities
+
+## Explicit Deletion
+
+> TODO: Describe direct deletion via the catalog API

docs/features/software-catalog/well-known-annotations.md

@@ -57,18 +57,48 @@ if the original location delegates to another location. A common case is, that a
 location is registered as `bootstrap:bootstrap` which means that it is part of
 the `app-config.yaml` of a Backstage installation.
 
+### backstage.io/orphan
+
+This annotation is either absent, or present with the exact _string_ value
+`"true"`. It should never be added manually. Instead, the catalog itself injects
+the annotation as part of its processing loops, on entities that are found to
+have no registered locations or config locations that keep them "active" /
+"alive".
+
+For example, suppose that the user first registers a location URL pointing to a
+`Location` kind entity, which in turn refers to two `Component` kind entities in
+two other files nearby. The end result is that the catalog contains those three
+entities. Now suppose that the user edits the original `Location` entity to only
+refer to the first of the `Component` kind entities. This will intentionally
+_not_ lead to the other `Component` entity to be removed from the catalog (for
+safety reasons). Instead, it gains this orphan marker annotation, to make it
+clear that user action is required to completely remove it, if desired.
+
+```yaml
+# Example:
+metadata:
+  annotations:
+    backstage.io/orphan: 'true'
+```
+
 ### backstage.io/techdocs-ref
 
 ```yaml
 # Example:
 metadata:
   annotations:
-    backstage.io/techdocs-ref: url:https://github.com/backstage/backstage/tree/master
+    backstage.io/techdocs-ref: dir:.
 ```
 
-The value of this annotation is a location reference string (see above). If this
-annotation is specified, it is expected to point to a repository that the
-TechDocs system can read and generate docs from.
+The value of this annotation informs _where_ TechDocs source content is stored
+so that it can be read and docs can be generated from it. Most commonly, it's
+written as a path, relative to the location of the `catalog-info.yaml` itself,
+where the associated `mkdocs.yml` file can be found.
+
+In unusual situations where the documentation for a catalog entity does not live
+alongside the entity's source code, the value of this annotation can point to an
+absolute URL, matching the location reference string format outlined above, for
+example: `url:https://github.com/backstage/backstage/tree/master`
 
 ### backstage.io/view-url, backstage.io/edit-url
 
@@ -101,18 +131,22 @@ repository itself. If the URL points to a folder, it is important that it is
 suffixed with a `'/'` in order for relative path resolution to work
 consistently.
 
-### jenkins.io/github-folder
+### jenkins.io/job-full-name
 
 ```yaml
 # Example:
 metadata:
   annotations:
-    jenkins.io/github-folder: folder-name/job-name
+    jenkins.io/job-full-name: folder-name/job-name
 ```
 
 The value of this annotation is the path to a job on Jenkins, that builds this
 entity.
 
+The value can be the format of just `[folder-path]` or
+`[instanceName]:[folder-path]`, if multiple instances are configured in
+`app-config.yaml`
+
 Specifying this annotation may enable Jenkins related features in Backstage for
 that entity.
 
@@ -304,6 +338,10 @@ This annotation allowed to load the API definition from another location. Use
 [substitution](./descriptor-format.md#substitutions-in-the-descriptor-format)
 instead.
 
+### jenkins.io/github-folder
+
+Use the `jenkins.io/job-full-name` instead.
+
 ## Links
 
 - [Descriptor Format: annotations](descriptor-format.md#annotations-optional)

docs/features/software-templates/configuration.md

@@ -33,9 +33,12 @@ scaffolder:
 
 ### Disabling Docker in Docker situation (Optional)
 
-Software Templates use
-[Cookiecutter](https://github.com/cookiecutter/cookiecutter) as a templating
-library. By default it will use the
+Software templates use the `fetch:template` action by default, which requires no
+external dependencies and offers a
+[Cookiecutter-compatible mode](https://backstage.io/docs/features/software-templates/builtin-actions#using-cookiecuttercompat-mode).
+There is also a `fetch:cookiecutter` action, which uses
+[Cookiecutter](https://github.com/cookiecutter/cookiecutter) directly for
+templating. By default, the `fetch:cookiecutter` action will use the
 [scaffolder-backend/Cookiecutter](https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/scripts/Cookiecutter.dockerfile)
 docker image.
 

docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md

@@ -325,7 +325,7 @@ spec:
   output:
     links:
       - url: '{{steps.publish.output.remoteUrl}}'
-        text: 'Go to Repo'
+        title: 'Go to Repo'
 ```
 
 ## Questions?

docs/features/software-templates/writing-custom-actions.md

@@ -131,6 +131,9 @@ want to have those as well as your new one, you'll need to do the following:
 
 ```ts
 import { createBuiltinActions } from '@backstage/plugin-scaffolder-backend';
+import { ScmIntegrations } from '@backstage/integration';
+
+const integrations = ScmIntegrations.fromConfig(config);
 
 const builtInActions = createBuiltinActions({
   containerRunner,

docs/features/software-templates/writing-templates.md

@@ -227,6 +227,33 @@ spec:
             inputType: tel
 ```
 
+#### Hide or mask sensitive data on Review step
+
+Sometimes, specially in custom fields, you collect some data on Create form that
+must not be shown to the user on Review step. To hide or mask this data, you can
+use `ui:widget: password` or set some properties of `ui:backstage`:
+
+```yaml
+- title: Hide or mask values
+  properties:
+    password:
+      title: Password
+      type: string
+      ui:widget: password # will print '******' as value for property 'password' on Review Step
+    masked:
+      title: Masked
+      type: string
+      ui:backstage:
+        review:
+          mask: '<some-value-to-show>' # will print '<some-value-to-show>' as value for property 'Masked' on Review Step
+    hidden:
+      title: Hidden
+      type: string
+      ui:backstage:
+        review:
+          show: false # wont print any info about 'hidden' property on Review Step
+```
+
 #### The Repository Picker
 
 So in order to make working with repository providers easier, we've built a

docs/features/techdocs/README.md

@@ -29,7 +29,7 @@ about TechDocs and the philosophy in its
 - Explore and take advantage of the large ecosystem of
   [MkDocs plugins](https://www.mkdocs.org/user-guide/plugins/) to create a rich
   reading experience.
-- Search for and find docs (coming soon).
+- Search for and find docs.
 - Highlight text and raise an Issue to create feedback loop to drive quality
   documentation (future).
 - Contribute to and deploy from a marketplace of TechDocs widgets (future).
@@ -54,23 +54,12 @@ providers are used.
 | Google Cloud Storage (GCS)        | Yes ✅         |
 | Amazon Web Services (AWS) S3      | Yes ✅         |
 | Azure Blob Storage                | Yes ✅         |
-| OpenStack Swift                   | Yes ✅         |
+| OpenStack Swift                   | Community ✅   |
 
 [Reach out to us](#feedback) if you want to request more platforms.
 
 ## Project roadmap
 
-### **Ongoing work 🚧**
-
-**Beta release** -
-[Milestone](https://github.com/backstage/backstage/milestone/29)
-
-- It should be possible and easy to use TechDocs in most environments across
-  organizations.
-- Minimal bugs, better error handling and scalable backend and frontend.
-- Documentation Search
-- TechDocs Homepage with basic features
-
 ### **Future work 🔮**
 
 **General Availability (GA) release** -

docs/features/techdocs/configuration.md

@@ -13,14 +13,29 @@ configuration options for TechDocs.
 # File: app-config.yaml
 
 techdocs:
-  # generators.techdocs can have two values: 'docker' or 'local'. This is to determine how to run the generator - whether to
-  # spin up the techdocs-container docker image or to run mkdocs locally (assuming all the dependencies are taken care of).
-  # You want to change this to 'local' if you are running Backstage using your own custom Docker setup and want to avoid running
-  # into Docker in Docker situation. Read more here
-  # https://backstage.io/docs/features/techdocs/getting-started#disable-docker-in-docker-situation-optional
+  # techdocs.generator is used to configure how documentation sites are generated using MkDocs.
 
-  generators:
-    techdocs: 'docker'
+  generator:
+    # techdocs.generator.runIn can be either 'docker' or 'local'. This is to determine how to run the generator - whether to
+    # spin up the techdocs-container docker image or to run mkdocs locally (assuming all the dependencies are taken care of).
+    # You want to change this to 'local' if you are running Backstage using your own custom Docker setup and want to avoid running
+    # into Docker in Docker situation. Read more here
+    # https://backstage.io/docs/features/techdocs/getting-started#disable-docker-in-docker-situation-optional
+
+    runIn: 'docker'
+
+    # (Optional) techdocs.generator.dockerImage can be used to control the docker image used during documentation generation. This can be useful
+    # if you want to use MkDocs plugins or other packages that are not included in the default techdocs-container (spotify/techdocs).
+    # NOTE: This setting is only used when techdocs.generator.runIn is set to 'docker'.
+
+    dockerImage: 'spotify/techdocs'
+
+    # (Optional) techdocs.generator.pullImage can be used to disable pulling the latest docker image by default. This can be useful when you are
+    # using a custom techdocs.generator.dockerImage and you have a custom docker login requirement. For example, you need to login to
+    # AWS ECR to pull the docker image.
+    # NOTE: Disabling this requires the docker image was pulled by other means before running the techdocs generator.
+
+    pullImage: true
 
   # techdocs.builder can be either 'local' or 'external.
   # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage
@@ -98,6 +113,14 @@ techdocs:
         # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
         accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY}
 
+  # (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs
+  # sites could only be accessed over paths with case-sensitive entity triplets
+  # e.g. (namespace/Kind/name). If you are upgrading from an older version of
+  # TechDocs and are unable to perform the necessary migration of files in your
+  # external storage, you can set this value to `true` to temporarily revert to
+  # the old, case-sensitive entity triplet behavior.
+  legacyUseCaseSensitiveTripletPaths: false
+
   # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc.
   # You don't have to specify this anymore.
 

docs/features/techdocs/creating-and-publishing.md

@@ -66,9 +66,7 @@ Update your component's entity description by adding the following lines to its
 ```yaml
 metadata:
   annotations:
-    backstage.io/techdocs-ref: url:https://github.com/org/repo
-    # Or
-    # backstage.io/techdocs-ref: url:https://github.com/org/repo/tree/branchName/subFolder
+    backstage.io/techdocs-ref: dir:.
 ```
 
 The

docs/features/techdocs/getting-started.md

@@ -33,14 +33,24 @@ In `packages/app/src/App.tsx`, import `TechDocsPage` and add the following to
 `FlatRoutes`:
 
 ```tsx
-import { TechDocsPage } from '@backstage/plugin-techdocs';
+import {
+  DefaultTechDocsHome,
+  TechDocsIndexPage,
+  TechDocsReaderPage,
+} from '@backstage/plugin-techdocs';
 
 // ...
 
 const AppRoutes = () => {
   <FlatRoutes>
     // ... other plugin routes
-    <Route path="/docs" element={<TechdocsPage />} />
+    <Route path="/docs" element={<TechDocsIndexPage />}>
+      <DefaultTechDocsHome />
+    </Route>
+    <Route
+      path="/docs/:namespace/:kind/:name/*"
+      element={<TechDocsReaderPage />}
+    />
   </FlatRoutes>;
 };
 ```
@@ -204,7 +214,7 @@ techdocs:
   builder: 'local'
   publisher:
     type: 'local'
-  generators:
+  generator:
     techdocs: local
 ```
 

docs/features/techdocs/how-to-guides.md

@@ -35,38 +35,64 @@ In your Backstage instance's `app-config.yaml`, set `techdocs.builder` from
 `'local'` to `'external'`. By doing this, TechDocs will not try to generate
 docs. Look at [TechDocs configuration](configuration.md) for reference.
 
-## How to use URL Reader in TechDocs Prepare step?
+## How to understand techdocs-ref annotation values
 
-If TechDocs is configured to generate docs, it will first download the
-repository associated with the `backstage.io/techdocs-ref` annotation defined in
-the Entity's `catalog-info.yaml` file. This is also called the
+If TechDocs is configured to generate docs, it will first download source files
+based on the value of the `backstage.io/techdocs-ref` annotation defined in the
+Entity's `catalog-info.yaml` file. This is also called the
 [Prepare](./concepts.md#techdocs-preparer) step.
 
-There are two kinds of preparers or two ways of downloading these source files
+We strongly recommend that the `backstage.io/techdocs-ref` annotation in each
+documented catalog entity's `catalog-info.yaml` be set to `dir:.` in almost all
+situations. This is because TechDocs is aligned with the "docs like code"
+philosophy, whereby documentation should be authored and managed alongside the
+source code of the underlying software itself.
 
-- Preparer 1: Doing a `git clone` of the repository (also known as Common Git
-  Preparer)
-- Preparer 2: Downloading an archive.zip or equivalent of the repository (also
-  known as URL Reader)
+When you see `dir:.`, you can translate it to mean:
 
-If `backstage.io/techdocs-ref` is equal to any of these -
+- That the documentation source code lives in the same location as the
+  `catalog-info.yaml` file.
+- That, in particular, the `mkdocs.yml` file is a sibling of `catalog-info.yaml`
+  (meaning, it is in the same directory)
+- And that all of the source content of the documentation would be available if
+  one were to download the directory containing those two files (as well as all
+  sub-directories).
 
-1. `github:https://githubhost.com/org/repo`
-2. `gitlab:https://gitlabhost.com/org/repo`
-3. `bitbucket:https://bitbuckethost.com/project/repo`
-4. `azure/api:https://azurehost.com/org/project`
+The directory tree of the entity would look something like this:
 
-Then Common Git Preparer will be used i.e. a `git clone`. But the URL Reader is
-a much faster way to do this step. Convert the `backstage.io/techdocs-ref`
-values to the following -
+```
+├── catalog-info.yaml
+├── mkdocs.yml
+└── docs
+    └── index.md
+```
 
-1. `url:https://githubhost.com/org/repo/tree/<branch_name>`
-2. `url:https://gitlabhost.com/org/repo/tree/<branch_name>`
-3. `url:https://bitbuckethost.com/project/repo/src/<branch_name>`
-4. `url:https://azurehost.com/organization/project/_git/repository`
+If, for example, you wanted to keep a lean root directory, you could place your
+`mkdocs.yml` file in a subdirectory and update the `backstage.io/techdocs-ref`
+annotation value accordingly, e.g. to `dir:./sub-folder`:
 
-Note that you can also provide a path to a non-root directory inside the
-repository which contains the `docs/` directory.
+```
+├── catalog-info.yaml
+└── sub-folder
+    ├── mkdocs.yml
+    └── docs
+        └── index.md
+```
+
+In rare situations where your TechDocs source content is managed and stored in a
+location completely separate from your `catalog-info.yaml`, you can instead
+specify a URL location reference, the exact value of which will vary based on
+the source code hosting provider. Notice that instead of the `dir:` prefix, the
+`url:` prefix is used instead. For example:
+
+- **GitHub**: `url:https://githubhost.com/org/repo/tree/<branch_name>`
+- **GitLab**: `url:https://gitlabhost.com/org/repo/tree/<branch_name>`
+- **Bitbucket**: `url:https://bitbuckethost.com/project/repo/src/<branch_name>`
+- **Azure**: `url:https://azurehost.com/organization/project/_git/repository`
+
+Note, just as it's possible to specify a subdirectory with the `dir:` prefix,
+you can also provide a path to a non-root directory inside the repository which
+contains the `mkdocs.yml` file and `docs/` directory.
 
 e.g.
 `url:https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend/examples/documented-component`
@@ -78,83 +104,116 @@ the repository. The archive does not have any git history attached to it. Also
 it is a compressed file. Hence the file size is significantly smaller than how
 much data git clone has to transfer.
 
-## How to use a custom TechDocs home page?
+## How to customize the TechDocs home page?
 
-### 1st way: TechDocsCustomHome with a custom configuration
+TechDocs uses a composability pattern similar to the Search and Catalog plugins
+in Backstage. While a default table experience, similar to the one provided by
+the Catalog plugin, is made available for ease-of-use, it's possible for you to
+provide a completely custom experience, tailored to the needs of your
+organization.
 
-As an example, in your main App.tsx:
+This is done in your `app` package. By default, you might see something like
+this in your `App.tsx`:
 
 ```tsx
-import {
-  TechDocsCustomHome,
-  PanelType,
-  TechDocsReaderPage,
-} from '@backstage/plugin-techdocs';
-import { Entity } from '@backstage/catalog-model';
-
-const tabsConfig = [
-  {
-    label: 'Custom Tab',
-    panels: [
-      {
-        title: 'Custom Documents Cards 1',
-        description:
-          'Explore your internal technical ecosystem through documentation.',
-        panelType: 'DocsCardGrid' as PanelType,
-        // optional, is applied to a container of the panel (excludes header of panel)
-        panelCSS: { maxHeight: '400px', overflow:'auto' },
-        filterPredicate: (entity: Entity) => !!entity.metadata.annotations?.['customCardAnnotationOne'];
-    },
-      {
-        title: 'Custom Documents Cards 2',
-        description:
-          'Explore your internal technical ecosystem through documentation.',
-        panelType: 'DocsCardGrid' as PanelType,
-        panelCSS: { maxHeight: '400px', overflow:'auto' },
-        filterPredicate: (entity: Entity) => !!entity.metadata.annotations?.['customCardAnnotationTwo'];
-      },
-    ],
-  },
-  {
-    label: 'Overview',
-    panels: [
-      {
-        title: 'Overview',
-        description:
-          'Explore your internal technical ecosystem through documentation.',
-        panelType: 'DocsTable' as PanelType,
-        filterPredicate: () => true,
-      },
-    ],
-  },
-];
-
-const routes = (
+const AppRoutes = () => {
   <FlatRoutes>
-    <Route
-      path="/docs"
-      element={<TechDocsCustomHome tabsConfig={tabsConfig} />}
-    />
-    <Route
-      path="/docs/:namespace/:kind/:name/*"
-      element={<TechDocsReaderPage />}
-    />
-  </FlatRoutes>
+    <Route path="/docs" element={<TechDocsIndexPage />}>
+      <DefaultTechDocsHome />
+    </Route>
+  </FlatRoutes>;
+};
 ```
 
-An example of tabsConfig that corresponds to the default documentation home page
-can be found at `plugins/techdocs/src/home/components/TechDocsHome.tsx`.
+But you can replace `<DefaultTechDocsHome />` with any React component, which
+will be rendered in its place. Most likely, you would want to create and
+maintain such a component in a new directory at
+`packages/app/src/components/techdocs`, and import and use it in `App.tsx`:
 
-Currently `panelType` has DocsCardGrid and DocsTable available. We currently
-recommend that DocsCardGrid can be optionally vertically stacked by setting a
-maxHeight using `panelCSS`, and DocsTable to be in a tab by itself.
+```tsx
+import { CustomTechDocsHome } from './components/techdocs/CustomTechDocsHome';
+// ...
+const AppRoutes = () => {
+  <FlatRoutes>
+    <Route path="/docs" element={<TechDocsIndexPage />}>
+      <CustomTechDocsHome />
+    </Route>
+  </FlatRoutes>;
+};
+```
 
-### 2nd way: Custom home page plugin
+## How to migrate from TechDocs Alpha to Beta
 
-A custom home page plugin can be built that uses the components extensions
-DocsCardGrid and DocsTable, exported from @backstage/techdocs. They both take a
-array of documentation entities ( i.e.have a 'backstage.io/techdocs-ref'
-annotation ) as an 'entities' attribute.
+> This guide only applies to the "recommended" TechDocs deployment method (where
+> an external storage provider and external CI/CD is used). If you use the
+> "basic" or "out-of-the-box" setup, you can stop here! No action needed.
 
-For a reference to the React structure of the default home page, please refer to
-`plugins/techdocs/src/home/components/TechDocsCustomHome.tsx`.
+For the purposes of this guide, TechDocs Beta version is defined as:
+
+- **TechDocs Plugin**: At least `v0.11.0`
+- **TechDocs Backend Plugin**: At least `v0.10.0`
+- **TechDocs CLI**: At least `v0.7.0`
+
+The beta version of TechDocs made a breaking change to the way TechDocs content
+was accessed and stored, allowing pages to be accessed with case-insensitive
+entity triplet paths (e.g. `/docs/namespace/kind/name` whereas in prior
+versions, they could only be accessed at `/docs/namespace/Kind/name`). In order
+to enable this change, documentation has to be stored in an external storage
+provider using an object key whose entity triplet is lower-cased.
+
+New installations of TechDocs since the beta version will work fine with no
+action, but for those who were running TechDocs prior to this version, a
+migration will need to be performed so that all existing content in your storage
+bucket matches this lower-case entity triplet expectation.
+
+1. **Ensure you have the right permissions on your storage provider**: In order
+   to migrate files in your storage provider, the `techdocs-cli` needs to be
+   able to read/copy/rename/move/delete files. The exact instructions vary by
+   storage provider, but check the [using cloud storage][using-cloud-storage]
+   page for details.
+
+2. **Run a non-destructive migration of files**: Ensure you have the latest
+   version of `techdocs-cli` installed. Then run the following command, using
+   the details relevant for your provider / configuration. This will copy all
+   files from, e.g. `namespace/Kind/name/index.html` to
+   `namespace/kind/name/index.html`, without removing the original files.
+
+```sh
+techdocs-cli migrate --publisher-type <awsS3|googleGcs|azureBlobStorage> --storage-name <bucket/container name> --verbose
+```
+
+3. **Deploy the updated versions of the TechDocs plugins**: Once the migration
+   above has been run, you can deploy the beta versions of the TechDocs backend
+   and frontend plugins to your Backstage instance.
+
+4. **Verify that your TechDocs sites are still loading/accessible**: Try
+   accessing a TechDocs site using different entity-triplet case variants, e.g.
+   `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`. Your TechDocs
+   site should load regardless of the URL path casing you use.
+
+5. **Clean up the old objects from storage**: Once you've verified that your
+   TechDocs site is accessible, you can clean up your storage bucket by
+   re-running the `migrate` command on the TechDocs CLI, but with an additional
+   `removeOriginal` flag passed:
+
+```sh
+techdocs-cli migrate --publisher-type <awsS3|googleGcs|azureBlobStorage> --storage-name <bucket/container name> --removeOriginal --verbose
+```
+
+6. **Update your CI/CD pipelines to use the beta version of the TechDocs CLI**:
+   Finally, you can update all of your CI/CD pipelines to use at least v0.x.y of
+   the TechDocs CLI, ensuring that all sites are published to the new,
+   lower-cased entity triplet paths going forward.
+
+If you encounter problems running this migration, please [report the
+issue][beta-migrate-bug]. You can temporarily revert to pre-beta storage
+expectations with a configuration change:
+
+```yaml
+techdocs:
+  legacyUseCaseSensitiveTripletPaths: true
+```
+
+[beta-migrate-bug]:
+https://github.com/backstage/backstage/issues/new?assignees=&labels=bug&template=bug_template.md&title=[TechDocs]%20Unable%20to%20run%20beta%20migration
+[using-cloud-storage]: ./using-cloud-storage.md

docs/features/techdocs/using-cloud-storage.md

@@ -61,7 +61,7 @@ If you do not prefer (3a) and optionally like to use a service account, you can
 follow these steps.
 
 Create a new Service Account and a key associated with it. In roles of the
-service account, use "Storage Admin".
+service account, use "Storage Object Admin".
 
 If you want to create a custom role, make sure to include both `get` and
 `create` permissions for both "Objects" and "Buckets". See
@@ -143,6 +143,8 @@ permissions to:
 
 - `s3:ListBucket` to retrieve bucket metadata
 - `s3:PutObject` to upload files to the bucket
+- `s3:DeleteObject` and `s3:DeleteObjectVersion` to delete stale content during
+  re-publishing
 
 To _read_ TechDocs from the S3 bucket the IAM policy needs to have at a minimum
 permissions to:
@@ -150,6 +152,39 @@ permissions to:
 - `s3:ListBucket` - To retrieve bucket metadata
 - `s3:GetObject` - To retrieve files from the bucket
 
+> Note: If you need to migrate documentation objects from an older-style path
+> format including case-sensitive entity metadata, you will need to add some
+> additional permissions to be able to perform the migration, including:
+>
+> - `s3:PutBucketAcl` (for copying files,
+>   [more info here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html))
+> - `s3:DeleteObject` and `s3:DeleteObjectVersion` (for deleting migrated files,
+>   [more info here](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html))
+>
+> ...And you will need to ensure the permissions apply to the bucket itself, as
+> well as all resources under the bucket. See the example policy below.
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "TechDocsWithMigration",
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:DeleteObjectVersion",
+        "s3:ListBucket",
+        "s3:DeleteObject",
+        "s3:PutObjectAcl"
+      ],
+      "Resource": ["arn:aws:s3:::your-bucket", "arn:aws:s3:::your-bucket/*"]
+    }
+  ]
+}
+```
+
 **4a. (Recommended) Setup authentication the AWS way, using environment
 variables**
 
@@ -312,6 +347,10 @@ techdocs:
         accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY}
 ```
 
+In either case, the account or credentials used to access your container and all
+TechDocs objects underneath it should have the `Storage Blog Data Owner` role
+applied, in order to read, write, and delete objects as needed.
+
 **4. That's it!**
 
 Your Backstage app is now ready to use Azure Blob Storage for TechDocs, to store
@@ -360,9 +399,34 @@ techdocs:
 
 Set the configs in your `app-config.yaml` to point to your container name.
 
-https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail#password-authentication-with-unscoped-authorization
+https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential
 for more details.
 
+```yaml
+techdocs:
+  publisher:
+    type: 'openStackSwift'
+    openStackSwift:
+      containerName: 'name-of-techdocs-storage-bucket'
+      credentials:
+        id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID}
+        secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET}
+      authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
+      swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL}
+```
+
+**4. That's it!**
+
+Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to
+store and read the static generated documentation files. When you start the
+backend of the app, you should be able to see
+`techdocs info Successfully connected to the OpenStack Swift Storage container`
+in the logs.
+
+## Bonus: Migration from old OpenStack Swift Configuration
+
+Let's assume we have the old OpenStack Swift configuration here.
+
 ```yaml
 techdocs:
   publisher:
@@ -379,10 +443,40 @@ techdocs:
       region: ${OPENSTACK_SWIFT_STORAGE_REGION}
 ```
 
-**4. That's it!**
+##### Step 1: Change the credential keys
 
-Your Backstage app is now ready to use OpenStack Swift Storage for TechDocs, to
-store and read the static generated documentation files. When you start the
-backend of the app, you should be able to see
-`techdocs info Successfully connected to the OpenStack Swift Storage container`
-in the logs.
+Since the new SDK uses _Application Credentials_ to authenticate OpenStack, we
+need to change the keys `credentials.username` to `credentials.id`,
+`credentials.password` to `credentials.secret` and use Application Credential ID
+and secret here. For more detail about credentials look
+[here](https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-unscoped-authorization-detail,authenticating-with-an-application-credential-detail#authenticating-with-an-application-credential).
+
+##### Step 2: Remove the unused keys
+
+Since the new SDK doesn't use the old way authentication, we don't need the keys
+`openStackSwift.keystoneAuthVersion`, `openStackSwift.domainId`,
+`openStackSwift.domainName` and `openStackSwift.region`. So you can remove them.
+
+##### Step 3: Add Swift URL
+
+The new SDK needs the OpenStack Swift connection URL for connecting the Swift.
+So you need to add a new key called `openStackSwift.swiftUrl` and give the
+OpenStack Swift url here. Example url should look like that:
+`https://example.com:6780/swift/v1`
+
+##### That's it!
+
+Your new configuration should look like that!
+
+```yaml
+techdocs:
+  publisher:
+    type: 'openStackSwift'
+    openStackSwift:
+      containerName: 'name-of-techdocs-storage-bucket'
+      credentials:
+        id: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_ID}
+        secret: ${OPENSTACK_SWIFT_STORAGE_APPLICATION_CREDENTIALS_SECRET}
+      authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
+      swiftUrl: ${OPENSTACK_SWIFT_STORAGE_SWIFT_URL}
+```

docs/getting-started/app-custom-theme.md

@@ -35,7 +35,7 @@ If you want more control over the theme, and for example customize font sizes
 and margins, you can use the lower-level `createThemeOverrides` function
 exported by [@backstage/theme](https://www.npmjs.com/package/@backstage/theme)
 in combination with
-[createMuiTheme](https://material-ui.com/customization/theming/#createmuitheme-options-args-theme)
+[createTheme](https://material-ui.com/customization/theming/#createmuitheme-options-args-theme)
 from [@material-ui/core](https://www.npmjs.com/package/@material-ui/core). See
 the
 [@backstage/theme source](https://github.com/backstage/backstage/tree/master/packages/theme/src)
@@ -128,7 +128,7 @@ const themeOptions = createThemeOptions({
     tool: genPageTheme(['#123456','#123456'], shapes.round),
     service: genPageTheme(['#123456','#123456'], shapes.wave),
     website: genPageTheme(['#123456','#123456'], shapes.wave),
-    library: genPageTheme(['#123456','#123456'] shapes.wave),
+    library: genPageTheme(['#123456','#123456'], shapes.wave),
     other: genPageTheme(['#123456','#123456'], shapes.wave),
     app: genPageTheme(['#123456','#123456'], shapes.wave),
     apis: genPageTheme(['#123456','#123456'], shapes.wave),

docs/getting-started/index.md

@@ -24,11 +24,9 @@ an easier path to make Pull Requests.
 Backstage provides the `@backstage/create-app` package to scaffold standalone
 instances of Backstage. You will need to have
 [Node.js](https://nodejs.org/en/download/) Active LTS Release installed
-(currently v14), [Yarn](https://classic.yarnpkg.com/en/docs/install) and
-[Python](https://www.python.org/downloads/) (although you likely have it
-already). You will also need to have
-[Docker](https://docs.docker.com/engine/install/) installed to use some features
-like Software Templates and TechDocs.
+(currently v14) and [Yarn](https://classic.yarnpkg.com/en/docs/install). You
+will also need to have [Docker](https://docs.docker.com/engine/install/)
+installed to use some features like Software Templates and TechDocs.
 
 Using `npx` you can then run the following to create an app in a chosen
 subdirectory of your current working directory:

docs/getting-started/project-structure.md

@@ -147,11 +147,6 @@ are separated out into their own folder, see further down.
   Helps you setup a plugin for isolated development so that it can be served
   separately.
 
-- [`docgen/`](https://github.com/backstage/backstage/tree/master/packages/docgen) -
-  Uses the
-  [TypeScript Compiler API](https://github.com/microsoft/TypeScript/wiki/Using-the-Compiler-API)
-  to read out definitions and generate documentation for it.
-
 - [`e2e-test/`](https://github.com/backstage/backstage/tree/master/packages/e2e-test) -
   Another CLI that can be run to try out what would happen if you build all the
   packages, publish them, create a new app, and then run them. CI uses this for

docs/integrations/aws-s3/locations.md

@@ -0,0 +1,42 @@
+---
+id: locations
+sidebar_label: Locations
+title: Amazon Web Services S3 Locations
+# prettier-ignore
+description: Setting up an integration with Amazon Web Services S3
+---
+
+The AWS S3 integration supports loading catalog entities from an S3 Bucket.
+Entities can be added to
+[static catalog configuration](../../features/software-catalog/configuration.md),
+or registered with the
+[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import)
+plugin.
+
+## Configuration
+
+To use this integration, add configuration to your `app-config.yaml`:
+
+```yaml
+integrations:
+  awsS3:
+    - host: amazonaws.com
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
+```
+
+Then make sure the environment variables `AWS_ACCESS_KEY_ID` and
+`AWS_SECRET_ACCESS_KEY` are set when you run Backstage.
+
+Users with multiple AWS accounts may want to use a role for S3 storage that is
+in a different AWS account. Using the `roleArn` parameter as seen below, you can
+instruct the AWS S3 reader to assume a role before accessing S3:
+
+```yaml
+integrations:
+  awsS3:
+    - host: amazonaws.com
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
+      roleArn: 'arn:aws:iam::xxxxxxxxxxxx:role/example-role'
+```

docs/integrations/bitbucket/discovery.md

@@ -38,7 +38,11 @@ The target is composed of four parts:
   repositories prefixed with `service-`.
 - The path within each repository to find the catalog YAML file. This will
   usually be `/catalog-info.yaml` or a similar variation for catalog files
-  stored in the root directory of each repository.
+  stored in the root directory of each repository. If omitted, the default value
+  `catalog-info.yaml` will be used. E.g. given that `my-project`and `service-a`
+  exists, `https://bitbucket.mycompany.com/projects/my-project/repos/service-*/`
+  will result in:
+  `https://bitbucket.mycompany.com/projects/my-project/repos/service-a/catalog-info.yaml`.
 
 ## Custom repository processing
 
@@ -52,13 +56,11 @@ matching repository is processed.
 repository.
 
 ```typescript
-const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({
-  client,
-  repository,
-}) {
-  // Custom logic for interpret the matching repository.
-  // See defaultRepositoryParser for an example
-};
+const customRepositoryParser: BitbucketRepositoryParser =
+  async function* customRepositoryParser({ client, repository }) {
+    // Custom logic for interpret the matching repository.
+    // See defaultRepositoryParser for an example
+  };
 
 const processor = BitbucketDiscoveryProcessor.fromConfig(env.config, {
   parser: customRepositoryParser,

docs/integrations/gitlab/discovery.md

@@ -0,0 +1,36 @@
+---
+id: discovery
+title: GitLab Discovery
+sidebar_label: Discovery
+# prettier-ignore
+description: Automatically discovering catalog entities from repositories in GitLab
+---
+
+The GitLab integration has a special discovery processor for discovering catalog
+entities from GitLab. The processor will crawl the GitLab instance and register
+entities matching the configured path. This can be useful as an alternative to
+static locations or manually adding things to the catalog.
+
+To use the discovery processor, you'll need a GitLab integration
+[set up](locations.md) with a `token`. Then you can add a location target to the
+catalog configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: gitlab-discovery
+      target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml
+```
+
+Note the `gitlab-discovery` type, as this is not a regular `url` processor.
+
+The target is composed of three parts:
+
+- The base URL, `https://gitlab.com` in this case
+- The group path, `group/subgroup` in this case. This is optional: If you omit
+  this path the processor will scan the entire GitLab instance instead.
+- The path within each repository to find the catalog YAML file. This will
+  usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or
+  a similar variation for catalog files stored in the root directory of each
+  repository. If you want to use the repository's default branch use the `*`
+  wildcard, e.g.: `/blob/*/catalog-info.yaml`