diff --git a/.changeset/breezy-cobras-deny.md b/.changeset/breezy-cobras-deny.md deleted file mode 100644 index 5f035c05ce..0000000000 --- a/.changeset/breezy-cobras-deny.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/core': patch ---- - -Clear sidebar search field once a search is executed diff --git a/.changeset/clever-moons-shake.md b/.changeset/clever-moons-shake.md deleted file mode 100644 index 0fb6c690c2..0000000000 --- a/.changeset/clever-moons-shake.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-common': patch ---- - -Added readTree support to AzureUrlReader diff --git a/.changeset/cli-links.md b/.changeset/cli-links.md deleted file mode 100644 index 0a9779de93..0000000000 --- a/.changeset/cli-links.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/cli': patch ---- - -The CLI now detects and transforms linked packages. You can link in external packages by adding them to both the `lerna.json` and `package.json` workspace paths. diff --git a/.changeset/cost-insights-tiny-llamas-perform.md b/.changeset/cost-insights-cyan-nails-film.md similarity index 52% rename from .changeset/cost-insights-tiny-llamas-perform.md rename to .changeset/cost-insights-cyan-nails-film.md index c81c28f8f8..69e21d87fe 100644 --- a/.changeset/cost-insights-tiny-llamas-perform.md +++ b/.changeset/cost-insights-cyan-nails-film.md @@ -2,4 +2,4 @@ '@backstage/plugin-cost-insights': patch --- -truncate large percentages > 1000% +Add breakdown view to the Cost Overview panel diff --git a/.changeset/cost-insights-quiet-bikes-occur.md b/.changeset/cost-insights-quiet-bikes-occur.md deleted file mode 100644 index aac283bbd4..0000000000 --- a/.changeset/cost-insights-quiet-bikes-occur.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-cost-insights': patch ---- - -fix product icon configuration diff --git a/.changeset/curly-yaks-invite.md b/.changeset/curly-yaks-invite.md deleted file mode 100644 index b9191a3895..0000000000 --- a/.changeset/curly-yaks-invite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-catalog-backend': patch ---- - -Add support for reading groups and users from the Microsoft Graph API. diff --git a/.changeset/dull-pans-sip.md b/.changeset/dull-pans-sip.md deleted file mode 100644 index 13616c1f8d..0000000000 --- a/.changeset/dull-pans-sip.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@backstage/plugin-scaffolder': patch -'@backstage/plugin-scaffolder-backend': patch ---- - -Move constructing the catalog-info.yaml URL for scaffolded components to the publishers diff --git a/.changeset/empty-kids-look.md b/.changeset/empty-kids-look.md deleted file mode 100644 index 680cd7f25c..0000000000 --- a/.changeset/empty-kids-look.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-register-component': patch ---- - -Remove catalog link on validate popup diff --git a/.changeset/four-plants-happen.md b/.changeset/four-plants-happen.md new file mode 100644 index 0000000000..b273d8ec41 --- /dev/null +++ b/.changeset/four-plants-happen.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-sentry': patch +'@backstage/plugin-welcome': patch +--- + +Refactor route registration to remove deprecating code diff --git a/.changeset/grumpy-crews-build.md b/.changeset/grumpy-crews-build.md new file mode 100644 index 0000000000..502210c257 --- /dev/null +++ b/.changeset/grumpy-crews-build.md @@ -0,0 +1,9 @@ +--- +'@backstage/config-loader': minor +--- + +Fix typo of "visibility" in config schema reference + +If you have defined a config element named `visiblity`, you +will need to fix the spelling to `visibility`. For more info, +see https://backstage.io/docs/conf/defining#visibility. diff --git a/.changeset/light-nails-crash.md b/.changeset/light-nails-crash.md new file mode 100644 index 0000000000..c7791d1027 --- /dev/null +++ b/.changeset/light-nails-crash.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Bump versions of `esbuild` and `rollup-plugin-esbuild` diff --git a/.changeset/perfect-dryers-sell.md b/.changeset/perfect-dryers-sell.md new file mode 100644 index 0000000000..e1c5592162 --- /dev/null +++ b/.changeset/perfect-dryers-sell.md @@ -0,0 +1,13 @@ +--- +'@backstage/create-app': patch +--- + +Removed `"resolutions"` entry for `esbuild` in the root `package.json` in order to use the version specified by `@backstage/cli`. + +To apply this change to an existing app, remove the following from your root `package.json`: + +```json +"resolutions": { + "esbuild": "0.6.3" +}, +``` diff --git a/.changeset/slow-insects-fail.md b/.changeset/slow-insects-fail.md new file mode 100644 index 0000000000..a07e025f34 --- /dev/null +++ b/.changeset/slow-insects-fail.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Add [API docs plugin](https://github.com/backstage/backstage/tree/master/plugins/api-docs) to new apps being created through the CLI. diff --git a/.changeset/small-worms-check.md b/.changeset/small-worms-check.md deleted file mode 100644 index e39265f8f9..0000000000 --- a/.changeset/small-worms-check.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'example-app': patch -'@backstage/plugin-search': patch ---- - -Using the search field in the sidebar now navigates to the search result page. diff --git a/.changeset/smart-turkeys-bathe.md b/.changeset/smart-turkeys-bathe.md new file mode 100644 index 0000000000..5f429b5b37 --- /dev/null +++ b/.changeset/smart-turkeys-bathe.md @@ -0,0 +1,28 @@ +--- +'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-import': minor +'@backstage/catalog-model': patch +'@backstage/plugin-scaffolder': patch +--- + +Add Analyze location endpoint to catalog backend. Add catalog-import plugin and replace import-component with it. To start using Analyze location endpoint, you have add it to the `createRouter` function options in the `\backstage\packages\backend\src\plugins\catalog.ts` file: + +```ts +export default async function createPlugin(env: PluginEnvironment) { + const builder = new CatalogBuilder(env); + const { + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + locationAnalyzer, //<-- + } = await builder.build(); + + return await createRouter({ + entitiesCatalog, + locationsCatalog, + higherOrderOperation, + locationAnalyzer, //<-- + logger: env.logger, + }); +} +``` diff --git a/.changeset/tidy-actors-repair.md b/.changeset/tidy-actors-repair.md new file mode 100644 index 0000000000..4869400556 --- /dev/null +++ b/.changeset/tidy-actors-repair.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Use type EntityName from catalog-model for entities diff --git a/.changeset/weak-roses-search.md b/.changeset/weak-roses-search.md new file mode 100644 index 0000000000..9f5074ce48 --- /dev/null +++ b/.changeset/weak-roses-search.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-proxy-backend': patch +--- + +Add configuration schema for the commonly used properties diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index c31e19e35c..7f889392ac 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -131,6 +131,7 @@ npm nvm oauth Oauth +oidc Okta Oldsberg onboarding @@ -229,3 +230,4 @@ yaml Zalando Zhou Zolotusky +zoomable diff --git a/.github/workflows/changeset.yml b/.github/workflows/changeset.yml index a557b4921b..4bd03b4d96 100644 --- a/.github/workflows/changeset.yml +++ b/.github/workflows/changeset.yml @@ -19,4 +19,4 @@ jobs: # Calls out to `changeset version`, but also runs prettier version: yarn release env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} diff --git a/.gitignore b/.gitignore index 5c27601791..3334bf956d 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ typings/ .nuxt dist dist-types +dist-workspace # Gatsby files .cache/ diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 65841d1c6a..0ab8d8390d 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -1,6 +1,6 @@ # Process for becoming a maintainer -## Your organization is not yet a maintainer +## a) Your organization is not yet a maintainer - Express interest to the sponsors that your organization is interested in becoming a maintainer. Becoming a maintainer generally means that you are going to be spending substantial time on Backstage for the foreseeable future. You should have domain expertise and be extremely proficient in TypeScript. - We will expect you to start contributing increasingly complicated PRs, under the guidance of the existing maintainers. @@ -8,7 +8,7 @@ - As you gain experience with the code base and our standards, we will ask you to do code reviews for incoming PRs. - After a period of approximately 2-3 months of working together and making sure we see eye to eye, the existing sponsors and maintainers will confer and decide whether to grant maintainer status or not. We make no guarantees on the length of time this will take, but 2-3 months is the approximate goal. -## Your organization is currently a maintainer +## b) Your organization is currently a maintainer To become a maintainer you need to demonstrate the following: diff --git a/app-config.yaml b/app-config.yaml index 32d3470f1b..3b476981e4 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -38,7 +38,7 @@ proxy: headers: Authorization: $env: TRAVISCI_AUTH_TOKEN - travis-api-version: 3 + travis-api-version: '3' '/newrelic/apm/api': target: https://api.newrelic.com/v2 @@ -66,8 +66,8 @@ sentry: rollbar: organization: my-company - accountToken: - $env: ROLLBAR_ACCOUNT_TOKEN + # NOTE: The rollbar-backend & accountToken key may be deprecated in the future (replaced by a proxy config) + accountToken: my-rollbar-account-token lighthouse: baseUrl: http://localhost:3003 @@ -189,8 +189,10 @@ scaffolder: api: token: $env: AZURE_TOKEN - auth: + ### Providing an auth.session.secret will enable session support in the auth-backend + # session: + # secret: custom session secret providers: google: development: @@ -235,6 +237,20 @@ auth: $env: AUTH_OAUTH2_AUTH_URL tokenUrl: $env: AUTH_OAUTH2_TOKEN_URL + oidc: + development: + metadataUrl: + $env: AUTH_OIDC_METADATA_URL + clientId: + $env: AUTH_OIDC_CLIENT_ID + clientSecret: + $env: AUTH_OIDC_CLIENT_SECRET + authorizationUrl: + $env: AUTH_OIDC_AUTH_URL + tokenUrl: + $env: AUTH_OIDC_TOKEN_URL + tokenSignedResponseAlg: + $env: AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG auth0: development: clientId: diff --git a/docs/assets/techdocs/architecture-basic.drawio.svg b/docs/assets/techdocs/architecture-basic.drawio.svg new file mode 100644 index 0000000000..8e3be76f8b --- /dev/null +++ b/docs/assets/techdocs/architecture-basic.drawio.svg @@ -0,0 +1,462 @@ + + + + + + + + + + + + + + +
+
+
+ Repo 1 +
+ (with docs) +
+
+
+
+ + Repo 1... + +
+
+ + + + +
+
+
+ Repo 2 +
+ (with docs) +
+
+
+
+ + Repo 2... + +
+
+ + + + +
+
+
+ TechDocs plugin +
+
+
+
+ + TechDocs plugin + +
+
+ + + + + +
+
+
+ Request TechDocs site +
+
+
+
+ + Request TechDocs site + +
+
+ + + + + + + + + + + + + + + + + + +
+
+
+ Source code hosting +
+
+
+
+ + Source code hosting + +
+
+ + + + +
+
+
+ TechDocs Backend Plugin +
+
+
+
+ + TechDocs Backend Plugin + +
+
+ + + + +
+
+
+ (Stages) +
+
+
+
+ + (Stages) + +
+
+ + + + + + +
+
+
+ Prepare +
+
+
+
+ + Prepare + +
+
+ + + + + + +
+
+
+ Generate +
+
+
+
+ + Generate + +
+
+ + + + +
+
+
+ Publish +
+
+
+
+ + Publish + +
+
+ + + + + +
+
+
+ Local Filesystem +
+
+
+
+ + Local File... + +
+
+ + + + +
+
+
+ Fetch markdown files +
+
+
+
+ + Fetch mark... + +
+
+ + + + +
+
+
+ Build doc files using mkdocs +
+
+
+
+ + Build doc... + +
+
+ + + + +
+
+
+ Store generated static content +
+
+
+
+ + Store gene... + +
+
+ + + + +
+
+
+ TechDocs Basic (out-of-the-box) architecture +
+
+
+
+ + TechDocs Basic (out-of-the-box) architect... + +
+
+ + + + + + + + +
+
+
+ + Route handler + +
+
+
+
+ + Route hand... + +
+
+ + + + +
+
+
+ or +
+
+
+
+ + or + +
+
+ + + + +
+
+
+ Trigger new build if docs D.N.E. +
+ or are outdated +
+
+
+
+ + Trigger new build... + +
+
+ + + + +
+
+
+ Fetch files to render +
+
+
+
+ + Fetch files... + +
+
+ + + + + + + + + + + +
+
+
+ + GCP +
+ Bucket +
+
+
+
+
+ + GCP... + +
+
+ + + + +
+
+
+ Azure storage +
+
+
+
+ + Azure st... + +
+
+ + + + + +
+
+
+ Storage solutions +
+
+
+
+ + Storage solutions + +
+
+ + + + + + + + +
+
+
+ + Amazon S3 + +
+
+
+
+ + Amazon S3 + +
+
+ +
+ + + + + Viewer does not support full SVG 1.1 + + + +
diff --git a/docs/assets/techdocs/architecture-recommended.drawio.svg b/docs/assets/techdocs/architecture-recommended.drawio.svg new file mode 100644 index 0000000000..e3af4b6b5f --- /dev/null +++ b/docs/assets/techdocs/architecture-recommended.drawio.svg @@ -0,0 +1,391 @@ + + + + + + + + + + + + + + +
+
+
+ Repo 1 +
+ (with docs) +
+
+
+
+ + Repo 1... + +
+
+ + + + +
+
+
+ Repo 2 +
+ (with docs) +
+
+
+
+ + Repo 2... + +
+
+ + + + + + + + +
+
+
+ CI/CD System +
+
+
+
+ + CI/CD System + +
+
+ + + + + + +
+
+
+ techdocs-cli +
+
+
+
+ + techdocs-cli + +
+
+ + + + +
+
+
+ Cloud Storage +
+
+
+
+ + Cloud Storage + +
+
+ + + + + + +
+
+
+ + Amazon S3 + +
+
+
+
+ + Amazon S3 + +
+
+ + + + + + + + + +
+
+
+ + GCP +
+ Bucket +
+
+
+
+
+ + GCP... + +
+
+ + + + +
+
+
+ Azure storage +
+
+
+
+ + Azure st... + +
+
+ + + + +
+
+
+ New commit +
+
+
+
+ + New commit + +
+
+ + + + +
+
+
+ New commit +
+
+
+
+ + New commit + +
+
+ + + + +
+
+
+ Build docs +
+ & +
+ Store generated static content +
+
+
+
+ + Build docs... + +
+
+ + + + + + + +
+
+
+ TechDocs plugin +
+
+
+
+ + TechDocs plugin + +
+
+ + + + +
+
+
+ TechDocs Backend plugin +
+
+
+
+ + TechDocs Backend plu... + +
+
+ + + + + + +
+
+
+ Request TechDocs site +
+
+
+
+ + Request TechDocs site + +
+
+ + + + +
+
+
+ Fetch files to render +
+
+
+
+ + Fetch files to render + +
+
+ + + + + + + + + + + + + + + +
+
+
+ Source code hosting +
+
+
+
+ + Source code hosting + +
+
+ + + + + +
+
+
+ Caching +
+ (Optional) +
+
+
+
+ + Caching... + +
+
+ + + + + + +
+
+
+ Storage solutions +
+
+
+
+ + Storage solutions + +
+
+ + + + +
+
+
+ TechDocs Recommended deployment architecture +
+
+
+
+ + TechDocs Recommended deployment architecture + +
+
+
+ + + + + Viewer does not support full SVG 1.1 + + + +
diff --git a/docs/assets/techdocs/techdocs_big_picture.png b/docs/assets/techdocs/techdocs_big_picture.png deleted file mode 100644 index 8b8f7a2338..0000000000 Binary files a/docs/assets/techdocs/techdocs_big_picture.png and /dev/null differ diff --git a/docs/auth/auth-backend-classes.md b/docs/auth/auth-backend-classes.md index 442553d4b8..8dbae0daa7 100644 --- a/docs/auth/auth-backend-classes.md +++ b/docs/auth/auth-backend-classes.md @@ -6,57 +6,71 @@ description: Documentation on Auth backend classes ## How Does Authentication Work? -The Backstage application can use various authentication providers for -authentication. A provider has to implement an `AuthProviderRouteHandlers` -interface for handling authentication. This interface consists of four methods. -Each of these methods is hosted at an endpoint `/auth/[provider]/method`, where -`method` performs a certain operation as follows: +The Backstage application can use various external authentication providers for +authentication. An external provider is wrapped using an +`AuthProviderRouteHandlers` interface for handling authentication. This +interface consists of four methods. Each of these methods is hosted at an +endpoint (by default) `/api/auth/[provider]/method`, where `method` performs a +certain operation as follows: ``` - /auth/[provider]/start -> start - /auth/[provider]/handler/frame -> frameHandler - /auth/[provider]/refresh -> refresh - /auth/[provider]/logout -> logout + /auth/[provider]/start -> Initiate a login from the web page + /auth/[provider]/handler/frame -> Handle a finished authentication operation + /auth/[provider]/refresh -> Refresh the validity of a login + /auth/[provider]/logout -> Log out a logged-in user ``` -For more information on how these methods are used and for which purpose, refer -to the [OAuth documentation](oauth.md). +The flow is as follows: -For details on the parameters, input and output conditions for each method, -refer to the type documentation under -`plugins/auth-backend/src/providers/types.ts`. +1. A user attempts to sign in. +2. A popup window is opened, pointing to the `auth` endpoint. That endpoint does + initial preparations and then re-directs the user to an external + authenticator, still inside the popup. +3. The authenticator validates the user and returns the result of the validation + (success OR failure), to the wrapper's endpoint (`handler/frame`). +4. The `handler/frame` rendered b´webpage will issue the appropriate response to + the webpage that opened the popup window, and the popup is closed. +5. The user signs out by clicking on a UI interface and the webpage makes a + request to logout the user. There are currently two different classes for two authentication mechanisms that implement this interface: an `OAuthAdapter` for [OAuth](https://oauth.net/2/) based mechanisms and a `SAMLAuthProvider` for -[SAML](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html) -based mechanisms. +[SAML](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html). -### OAuth mechanisms +If you do not have an `OAuth2` or `SAML` based authentication provider, look in +the section [below](#implementing-your-own-auth-wrapper). + +### OAuth Mechanisms + +For more information on how these methods are used and for which purpose, refer +to the [OAuth documentation](oauth.md). Currently OAuth is assumed to be the de facto authentication mechanism for Backstage based applications. Backstage comes with a "batteries-included" set of supported commonly used OAuth -providers: Okta, GitHub, Google, GitLab, and a generic OAuth2 provider. +providers: Okta, GitHub, Google, GitLab, and a generic OAuth2 provider. For a +list of available providers, look at the available wrappers in +`backstage/plugins/auth-backend/src/providers/`. -All of these use the authorization flow of OAuth2 to implement authentication. +All of these use the **authorization flow** of OAuth2 to implement +authentication. -If your authentication provider is any of the above mentioned (except generic -OAuth2) providers, you can configure them by setting the right variables in -`app-config.yaml` under the `auth` section. +If your authentication provider is any of the above mentioned providers, you can +configure them by setting the right variables in `app-config.yaml` under the +`auth` section. ### Configuration Each authentication provider (except SAML) needs five parameters: an OAuth -client ID, a client secret, an authorization endpoint and a token endpoint, and -an app origin. The app origin is the URL at which the frontend of the -application is hosted, and it is read from the `app.baseUrl` config. This is -required because the application opens a popup window to perform the -authentication, and once the flow is completed, the popup window sends a -`postMessage` to the frontend application to indicate the result of the -operation. Also this URL is used to verify that authentication requests are -coming from only this endpoint. +client ID, a client secret, an authorization endpoint, a token endpoint, and an +app origin. The app origin is the URL at which the frontend of the application +is hosted, and it is read from the `app.baseUrl` config. This is required +because the application opens a popup window to perform the authentication, and +once the flow is completed, the popup window sends a `postMessage` to the +frontend application to indicate the result of the operation. Also this URL is +used to verify that authentication requests are coming from only this endpoint. These values are configured via the `app-config.yaml` present in the root of your app folder. @@ -85,20 +99,60 @@ auth: ... ``` -## Technical Notes +## Implementing Your Own Auth Wrapper -### OAuthEnvironmentHandler +The core interface of any auth wrapper is the `AuthProviderRouteHandlers` +interface. This interface has four methods corresponding to the API described in +the initial section. Any auth wrapper will have to implement this interface. -The concept of an "env" is core to the way the auth backend works. It uses an +When initiating a login, a pop-up window is created by the frontend, to allow +the user to initiate a login. This login request is done to the `/start` +endpoint which is handled by the `start` method. + +The `start` method re-directs to the external auth provider who authenticates +the request and re-directs the request to the `/frame/handler` endpoint, which +is handled by the `frameHandler` method. + +The `frameHandler` returns an HTML response, containing a script that does a +`postMessage` to the frontend's window, containing the result of the request. +The `WebMessageResponse` type is the message sent by the `postMessage` to the +frontend. + +A `postMessageResponse` utility function wraps the logic of generating a +`postMessage` response that ensures that CORS is successfully handled. This +function takes an `express.Response`, a `WebMessageResponse` and the URL of the +frontend (`appOrigin`) as parameters and return an HTML page with the script and +the message. + +### OAuth Wrapping Interfaces. + +Each OAuth external provider is supported by a corresponding +[Passport](https://github.com/jaredhanson/passport) strategy. For a generic +OAuth2 provider, passport has a `passport-oauth2` strategy. The strategy class +handles the implementation details of working with each provider. + +Each strategy is wrapped by an `OAuthHandlers` interface. + +This interface cannot be directly used as an Express HTTP request handler. To do +so, `OAuthHandlers` are wrapped in an `OAuthAdapter`, which implements the +`AuthProviderRouterHandlers` interface. + +#### Env + +The concept of an `env` is core to the way the auth backend works. It uses an `env` query parameter to identify the environment in which the application is -running (`development`, `staging`, `production`, etc). Each runtime can support -multiple environments at the same time and the right handler for each request is -identified and dispatched to based on the `env` parameter. All -`AuthProviderRouteHandlers` are wrapped within an `OAuthEnvironmentHandler`. +running (`development`, `staging`, `production`, etc). Each runtime can +simultaneously support multiple environments at the same time and the right +handler for each request is identified and dispatched to, based on the `env` +parameter. -To instantiate multiple OAuth providers for different environments, use +`OAuthEnvironmentHandler` is a utility wrapper for an `OAuthHandlers` that +implements the `AuthProviderRouteHandlers` interface while supporting multiple +`env`s. + +To instantiate OAuth providers (the same but for different environments), use `OAuthEnvironmentHandler.mapConfig`. It's a helper to iterate over a -configuration object that is a map of environment to configurations. See one of +configuration object that is a map of environments to configurations. See one of the existing OAuth providers for an example of how it is used. Given the following configuration: @@ -113,13 +167,18 @@ production: ``` The `OAuthEnvironmentHandler.mapConfig(config, envConfig => ...)` call will -split the `config` by the top level `development` and `production` keys, and -pass on each block as `envConfig`. +split the config by the top level `development` and `production` keys, and pass +on each block as `envConfig`. -For a list of currently available providers, look in the `factories` module -located in `plugins/auth-backend/src/providers/factories.ts` +For convenience, the `AuthProviderFactory` is a factory function that has to be +implemented which can then generate a `AuthProviderRouteHandlers` for a given +provider. -### OAuth2 provider +All of the supported providers provide an `AuthProviderFactory` that returns an +`OAuthEnvironmentHandler`, capable of handling authentication for multiple +environments. + +### OAuth2 Provider The `oauth2` provider abstracts a generic **OAuth2 + OIDC** based authentication provider. What this means is that after the application has been given diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 5d7a5b9af0..438f874990 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -43,7 +43,7 @@ software catalog API. "kind": "Component", "metadata": { "annotations": { - "backstage.io/managed-by-location": "file:/tmp/component-info.yaml", + "backstage.io/managed-by-location": "file:/tmp/catalog-info.yaml", "example.com/service-discovery": "artistweb", "circleci.com/project-slug": "github/example-org/artist-website" }, @@ -93,6 +93,43 @@ significance and have reserved purposes and distinct shapes. See below for details about these fields. +## Substitutions In The Descriptor Format + +The descriptor format supports substitutions using `$text`, `$json`, and +`$yaml`. + +Placeholders like `$json: https://example.com/entity.json` are substituted by +the content of the referenced file. Files can be referenced from any configured +integration similar to locations by passing an absolute URL. It's also possible +to reference relative files like `./referenced.yaml` from the same location. +Relative references are handled relative to the folder of the +`catalog-info.yaml` that contains the placeholder. There are three different +types of placeholders: + +- `$text`: Interprets the contents of the referenced file as plain text and + embeds it as a string. +- `$json`: Interprets the contents of the referenced file as JSON and embeds the + parsed structure. +- `$yaml`: Interprets the contents of the referenced file as YAML and embeds the + parsed structure. + +For example, this can be used to load the definition of an API entity from a web +server and embed it as a string in the field `spec.definition`: + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: API +metadata: + name: petstore + description: The Petstore API +spec: + type: openapi + lifecycle: production + owner: petstore@example.com + definition: + $text: https://petstore.swagger.io/v2/swagger.json +``` + ## Common to All Kinds: The Envelope The root envelope object has the following structure. @@ -344,7 +381,7 @@ spec: type: website lifecycle: production owner: artist-relations@example.com - implementsApis: + providesApis: - artist-api ``` @@ -408,12 +445,35 @@ group of people in an organizational structure. ### `spec.implementsApis` [optional] +**NOTE**: This field was marked for deprecation on Nov 25nd, 2020. It will be +removed entirely from the model on Dec 14th, 2020 in the repository and will not +be present in released packages following the next release after that. Please +update your code to not consume this field before the removal date. + Links APIs that are implemented by the component, e.g. `artist-api`. This field is optional. The software catalog expects a list of one or more strings that references the names of other entities of the `kind` `API`. +This field has the same behavior as `spec.providesApis`. + +### `spec.providesApis` [optional] + +Links APIs that are provided by the component, e.g. `artist-api`. This field is +optional. + +The software catalog expects a list of one or more strings that references the +names of other entities of the `kind` `API`. + +### `spec.consumesApis` [optional] + +Links APIs that are consumed by the component, e.g. `artist-api`. This field is +optional. + +The software catalog expects a list of one or more strings that references the +names of other entities of the `kind` `API`. + ## Kind: Template Describes the following entity kind: @@ -592,6 +652,9 @@ The current set of well-known and common values for this field is: [OpenAPI](https://swagger.io/specification/) version 2 or version 3 spec. - `asyncapi` - An API definition based on the [AsyncAPI](https://www.asyncapi.com/docs/specifications/latest/) spec. +- `graphql` - An API definition based on + [GraphQL schemas](https://spec.graphql.org/) for consuming + [GraphQL](https://graphql.org/) based APIs. - `grpc` - An API definition based on [Protocol Buffers](https://developers.google.com/protocol-buffers) to use with [gRPC](https://grpc.io/). @@ -700,6 +763,11 @@ sufficient to enter only the `metadata.name` field of that group. ### `spec.ancestors` [required] +**NOTE**: This field was marked for deprecation on Nov 22nd, 2020. It will be +removed entirely from the model on Dec 6th, 2020 in the repository and will not +be present in released packages following the next release after that. Please +update your code to not consume this field before the removal date. + The recursive list of parents up the hierarchy, by stepping through parents one by one. The list must be present, but may be empty if `parent` is not present. The first entry in the list is equal to `parent`, and then the following ones @@ -728,6 +796,11 @@ sufficient to enter only the `metadata.name` field of those groups. ### `spec.descendants` [required] +**NOTE**: This field was marked for deprecation on Nov 22nd, 2020. It will be +removed entirely from the model on Dec 6th, 2020 in the repository and will not +be present in released packages following the next release after that. Please +update your code to not consume this field before the removal date. + The immediate and recursive child groups of this group in the hierarchy (children, and children's children, etc.). The list must be present, but may be empty if there are no child groups. The items are not guaranteed to be ordered diff --git a/docs/features/software-catalog/references.md b/docs/features/software-catalog/references.md index e438ddc977..d347035950 100644 --- a/docs/features/software-catalog/references.md +++ b/docs/features/software-catalog/references.md @@ -51,7 +51,7 @@ spec: type: service lifecycle: experimental owner: group:pet-managers - implementsApis: + providesApis: - petstore - internal/streetlights - hello-world @@ -66,7 +66,7 @@ catalog that is of kind `Group`, namespace `default` (which, actually, also can be left out in its own yaml file because that's the default value there too), and name `pet-managers`. -The entries in `implementsApis` are also references. In this case, none of them +The entries in `providesApis` are also references. In this case, none of them needs to specify a kind since we know from the context that that's the only kind that's supported here. The second entry specifies a namespace but the other ones don't, and in this context, the default is to refer to the same namespace as the diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 7d16427a3c..3627c74b9a 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -238,22 +238,9 @@ annotation, with the same value format. ### backstage.io/definition-at-location -This annotation allowed to load the API definition from another location. Now -placeholders can be used instead: - -``` -apiVersion: backstage.io/v1alpha1 -kind: API -metadata: - name: petstore - description: The Petstore API -spec: - type: openapi - lifecycle: production - owner: petstore@example.com - definition: - $text: https://petstore.swagger.io/v2/swagger.json -``` +This annotation allowed to load the API definition from another location. Use +[substitution](./descriptor-format.md#substitutions-in-the-descriptor-format) +instead. ## Links diff --git a/docs/features/software-catalog/well-known-relations.md b/docs/features/software-catalog/well-known-relations.md index 07a4fc0a5c..54f7833d1b 100644 --- a/docs/features/software-catalog/well-known-relations.md +++ b/docs/features/software-catalog/well-known-relations.md @@ -45,17 +45,28 @@ entity, but there will always be one ultimate owner. This relation is commonly generated based on `spec.owner` of the owned entity, where present. -### `consumesApi` and `providesApi` +### `providesApi` and `apiProvidedBy` A relation with an [API](descriptor-format.md#kind-api) entity, typically from a [Component](descriptor-format.md#kind-component) or [System](descriptor-format.md#kind-system). -These relations express that a component or system either exposes an API - -meaning that it hosts callable endpoints from which you can consume that API - -or that they are dependent on being able to consume that API. +These relations express that a component or system exposes an API - meaning that +it hosts callable endpoints from which you can consume that API. -This relation is commonly generated based on `spec.implementsApis` of the +This relation is commonly generated based on `spec.providesApis` of the +component or system in question. + +### `consumesApi` and `apiConsumedBy` + +A relation with an [API](descriptor-format.md#kind-api) entity, typically from a +[Component](descriptor-format.md#kind-component) or +[System](descriptor-format.md#kind-system). + +These relations express that a component or system consumes an API - meaning +that it depends on endpoints of the API. + +This relation is commonly generated based on `spec.consumesApis` of the component or system in question. ### `dependsOn` and `dependencyOf` diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 75a75003f7..e2b69d6767 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -55,6 +55,10 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. +_NOTE_: When the `publish` step is completed, it is currently assumed by the +scaffolder that the final repository should contain a `catalog-info.yaml` in +order to register this with the Catalog in Backstage. + Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md index 926ec238e1..f502b3836e 100644 --- a/docs/features/techdocs/architecture.md +++ b/docs/features/techdocs/architecture.md @@ -1,7 +1,153 @@ --- id: architecture -title: Architecture -description: Documentation on Architecture +title: TechDocs Architecture +description: Documentation on TechDocs Architecture --- -![TechDocs Big Picture](../../assets/techdocs/techdocs_big_picture.png) +## Basic (out-of-the-box) + +When you deploy Backstage (with TechDocs enabled by default), you get a basic +out-of-the box experience. + +TechDocs Architecture diagram + +> Note: See below for our recommended deployment architecture which takes care +> of stability, scalability and speed. + +When you open a TechDocs site in Backstage, the +[TechDocs Reader](./concepts.md#techdocs-reader) makes a request to +`techdocs-backend` with the entity ID and the path of the current page you are +looking at. In response, it receives the static files (HTML, CSS, JSON, etc.) to +render on the page in TechDocs/Backstage. + +The static files consist of HTML, CSS and Images generated by MkDocs. We remove +all the Javascript before adding them to Backstage for security reasons. And +there are some additional techdocs metadata JSON files that TechDocs needs to +render a site. + +The TechDocs Reader then applies a list of "Transformers" (see +[Concepts](./concepts.md)) which modify the generated static HTML files for a +number of use cases e.g. Remove certain headers, filter out some HTML tags, etc. + +Currently, we use the Backstage server's (or techdocs-backend's) local file +system to store the generated files. Publishing to an external storage system +(AWS S3, GCS, etc.) is also possible, but has not been implemented yet. + +A word about `UrlReader` vs Git preparer - Right now, we have two ways to fetch +files from its source repository for docs site generation. 1. By using Git +and 2. By directly using Source control (GitHub, Azure, etc.) APIs. This work is +heavily in progress. Please reach out to us on Discord in the #docs-like-code +channel to talk about it. + +## Recommended deployment + +This is how we recommend deploying TechDocs in production environment. + +TechDocs Architecture diagram + +The key difference in the recommended deployment approach is where the docs are +built. + +We assume each entity lives in a repository somewhere (GitHub, GitLab, etc.). We +recommend using a CI/CD pipeline with the repository that has a dedicated +step/job to build docs for TechDocs. The generated static files are then stored +in a cloud storage solution of your choice. +[Track progress here](https://github.com/backstage/backstage/issues/3096). + +Similar to how it is done in the Basic setup, the TechDocs Reader requests +`techdocs-backend` plugin for the docs site. `techdocs-backend` then requests +your configured storage solution for the necessary files and returns them to +TechDocs Reader. + +We will provide instructions, scripts and/or templates (e.g. GitHub actions) to +build docs in your CI/CD system. +[Track progress here.](https://github.com/backstage/backstage/issues/3400) You +will be able to use `techdocs-cli` to build docs and publish the generated docs +site files to your cloud storage system. + +Note about caching: We have noticed internally that some storage providers can +be quite slow, which is why we are recommending a cache that sits between the +TechDocs Reader and the Storage. + +_Feel free to suggest better ideas to us in #docs-like-code channel in Discord +or via a GitHub issue._ + +### Security consideration + +Our biggest security concern is managing the access to the docs in the cloud +storage. We also want to have only one security solution for all different types +of storage (GCS, AWS, custom SFTP server, etc.) Restricting access to the +storage and only allowing `techdocs-backend` to fetch files is a good way to +achieve this. + +This would also allow us to use the access control management Backstage when +that is ready. +[Track progress here.](https://github.com/backstage/backstage/issues/3218) + +In theory, you can directly enable TechDocs Reader to read from your storage. +But, you will have to think about how to do it without the docs being public and +how access to user groups is managed. + +For cloud storage access tokens, `techdocs-backend` only needs a token with Read +permissions. But in your CI/CD system, there needs to be a token with Write +permissions to publish the generated docs site files. + +## FAQs + +**Q: Why do you have separate "basic" and "recommended" deployment approaches?** + +A: The basic or out-of-the-box setup is what you get when you create a new app +or do a git clone of the Backstage repository. We want the first experience to +_just work magically_ so that you can have your first experience with TechDocs +which is smooth. However, if you decide to deploy Backstage/TechDocs for +production use, the basic setup would work but there are going to be downsides +as you scale with the number of documentation sites and sizes of them. So you +would want to make sure the deployment is as stable as possible. Hence there is +a recommended approach. There can be even more deployment approaches to TechDocs +and we welcome such "Alternative" ideas from the community. + +**Q: Why don't you recommend techdocs-backend local filesystem to serve static +files?** + +A: It would make scaling a Backstage instance harder. Think about the case where +we have distributed Backstage deployments. Using a separate file storage system +for TechDocs makes it easier to do some operations like delete a docs site and +wipe its contents. + +**Q: Why aren't docs built on the fly i.e. when users visits a page, generate +docs site in real-time?** + +A: Generating the content from Markdown on the fly is not optimal (although that +is how the basic out-of-the-box setup is implemented). Storage solutions act as +a cache for the generated static content. TechDocs is also currently built on +MkDocs which does not allow us to build docs per-page, so we would have to build +all docs for a entity on every request. + +# Future work + +_Ideas here are far fetched and not in the project's milestone for near future +(~6 months)._ + +We currently depend on MkDocs to parse doc sites written in Markdown. And we +store the generated static assets and re-use it later to render in Backstage. A +better (futuristic) approach will be to directly parse whatever type of source +files you have in your docs repository and directly render in Backstage in +real-time. + +# Features status + +Status of all the features mentioned above. + +**In place ✅** + +- Basic setup with techdocs-backend file server as storage. + +**Work in progress 🚧** + +- Basic setup with cloud storage solution. + +**Not implemented yet ❌** + +- `techdocs-cli` is able to generate docs in CI/CD environment. +- `techdocs-cli` is able to publish docs site to any storage. +- `techdocs-backend` integration with Backstage access control management. diff --git a/docs/getting-started/configure-app-with-plugins.md b/docs/getting-started/configure-app-with-plugins.md index 1d02f147c1..042d8964ca 100644 --- a/docs/getting-started/configure-app-with-plugins.md +++ b/docs/getting-started/configure-app-with-plugins.md @@ -6,7 +6,53 @@ description: Documentation on How Configuring App with plugins ## Adding existing plugins to your app -Coming soon! +The following steps assume that you have created a new Backstage app and want to +add an existing plugin to it. We are using the +[CircleCI](https://github.com/backstage/backstage/blob/master/plugins/circleci/README.md) +plugin in this example. + +1. Add the plugin's NPM package to the repo: + +```bash +yarn add @backstage/plugin-circleci +``` + +2. Add the plugin itself: + +```js +// packages/app/src/plugins.ts +export { plugin as Circleci } from '@backstage/plugin-circleci'; +``` + +3. Register the plugin router: + +```jsx +// packages/app/src/components/catalog/EntityPage.tsx + +import { Router as CircleCIRouter } from '@backstage/plugin-circleci'; + +// Then somewhere inside +} +/>; +``` + +Note that stand-alone plugins that are not "attached" to the Software Catalog +would be added outside the `EntityPage`. + +4. [Optional] Add proxy config: + +```yaml +// app-config.yaml +proxy: + '/circleci/api': + target: https://circleci.com/api/v1.1 + headers: + Circle-Token: + $env: CIRCLECI_AUTH_TOKEN +``` ### Adding a plugin page to the Sidebar diff --git a/docs/getting-started/create-an-app.md b/docs/getting-started/create-an-app.md index 552b6c3ec2..6aef4c48c1 100644 --- a/docs/getting-started/create-an-app.md +++ b/docs/getting-started/create-an-app.md @@ -14,7 +14,7 @@ need to run Backstage in your own environment. To create a Backstage app, you will need to have [Node.js](https://nodejs.org/en/download/) Active LTS Release installed -(currently v12). +(currently v14). Backstage provides a utility for creating new apps. It guides you through the initial setup of selecting the name of the app and a database for the backend. diff --git a/docs/getting-started/running-backstage-locally.md b/docs/getting-started/running-backstage-locally.md index a30a918fd0..6de9d234e3 100644 --- a/docs/getting-started/running-backstage-locally.md +++ b/docs/getting-started/running-backstage-locally.md @@ -8,19 +8,19 @@ description: Documentation on How to run Backstage Locally - Node.js -First make sure you are using Node.js with an Active LTS Release, currently v12. +First make sure you are using Node.js with an Active LTS Release, currently v14. This is made easy with a version manager such as [nvm](https://github.com/nvm-sh/nvm) which allows for version switching. ```bash # Installing a new version -nvm install 12 -> Downloading and installing node v12.18.3... -> Now using node v12.18.3 (npm v6.14.6) +nvm install 14 +> Downloading and installing node v14.15.1... +> Now using node v14.15.1 (npm v6.14.8) # Checking your version node --version -> v12.18.3 +> v14.15.1 ``` - Yarn diff --git a/docs/overview/stability-index.md b/docs/overview/stability-index.md index 416cdcdd4a..567e9dcbff 100644 --- a/docs/overview/stability-index.md +++ b/docs/overview/stability-index.md @@ -79,11 +79,11 @@ want to ensure some stability. ### [`cli`](https://github.com/backstage/backstage/tree/master/packages/cli/) -The main toolchain used for Backstage development. The interface that is -considered for stability are the various commands and options passed to those -commands, as well as the environment variables read by the CLI. The build output -may change over time and is not considered a breaking change unless it is likely -to affect external tooling. +The main toolchain used for Backstage development. The various CLI commands and +options passed to those commands, as well as the environment variables read by +the CLI, are considered to be the interface that the stability index refers to. +The build output may change over time and is not considered a breaking change +unless it is likely to affect external tooling. Stability: `2` diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md index f55b4d9229..ac0d110010 100644 --- a/docs/plugins/proxying.md +++ b/docs/plugins/proxying.md @@ -52,19 +52,7 @@ configuration will lead to the proxy acting on backend requests to The value inside each route is either a simple URL string, or an object on the format accepted by -[http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware). It -is also possible to limit the forwarded HTTP methods with the configuration -`allowedMethods`, for example `allowedMethods: ['GET']` to enforce read-only -access. - -By default, the proxy will only forward safe HTTP request headers to the target. -Those are based on the headers that are considered safe for CORS and includes -headers like `content-type` or `last-modified`, as well as all headers that are -set by the proxy. If the proxy should forward other headers like -`authorization`, this must be enabled by the `allowedHeaders` config, for -example `allowedHeaders: ['Authorization']`. This should help to not -accidentally forward confidential headers (`cookie`, `X-Auth-Request-User`) to -third-parties. +[http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware). If the value is a string, it is assumed to correspond to: @@ -85,3 +73,18 @@ except with the following caveats for convenience: `'^/api/proxy/larger-example/v1/': '/'` is added. That means that a request to `/api/proxy/larger-example/v1/some/path` will be translated to a request to `http://larger.example.com:8080/svc.v1/some/path`. + +There are also additional settings: + +- `allowedMethods`: Limit the forwarded HTTP methods. For example + `allowedMethods: ['GET']` enforces read-only access. +- `allowedHeaders`: A list of headers that should be forwarded to the target. + +By default, the proxy will only forward safe HTTP request headers to the target. +Those are based on the headers that are considered safe for CORS and includes +headers like `content-type` or `last-modified`, as well as all headers that are +set by the proxy. If the proxy should forward other headers like +`authorization`, this must be enabled by the `allowedHeaders` config, for +example `allowedHeaders: ['Authorization']`. This should help to not +accidentally forward confidential headers (`cookie`, `X-Auth-Request-User`) to +third-parties. diff --git a/docs/reference/createPlugin.md b/docs/reference/createPlugin.md index 45e3303124..4a58a5ecdc 100644 --- a/docs/reference/createPlugin.md +++ b/docs/reference/createPlugin.md @@ -4,7 +4,7 @@ title: createPlugin description: Documentation on createPlugin --- -Taking a plugin config as argument and returns a new plugin. +Takes a plugin config as an argument and returns a new plugin. ## Plugin Config @@ -28,18 +28,22 @@ type PluginHooks = { ### Creating a basic plugin -Showcasing adding multiple routes, a feature flag and a redirect. +Showcasing adding a route and a feature flag. ```jsx -import { createPlugin } from '@backstage/core'; +import { createPlugin, createRouteRef } from '@backstage/core'; import ExampleComponent from './components/ExampleComponent'; +export const rootRouteRef = createRouteRef({ + path: '/new-plugin', + title: 'New Plugin', +}); + export default createPlugin({ id: 'new-plugin', register({ router, featureFlags }) { + router.addRoute(rootRouteRef, ExampleComponent); featureFlags.register('enable-example-component'); - - router.registerRoute('/new-plugin', ExampleComponent); }, }); ``` diff --git a/microsite/README.md b/microsite/README.md index d244caa22a..9589def3e3 100644 --- a/microsite/README.md +++ b/microsite/README.md @@ -204,3 +204,8 @@ For more information about custom pages, click [here](https://docusaurus.io/docs # Full Documentation Full documentation can be found on the [website](https://docusaurus.io/). + +## Additional notes + +- If you want to make images zoomable on click, add the `data-zoomable` attribute to your `img` element. + - In a docs or blog `.md` file, convert `![This is image](/microsite/static/img/code.png)` syntax to `This is image` diff --git a/microsite/data/plugins/jira.yaml b/microsite/data/plugins/jira.yaml new file mode 100644 index 0000000000..579243d11c --- /dev/null +++ b/microsite/data/plugins/jira.yaml @@ -0,0 +1,9 @@ +--- +title: Jira +author: roadie.io +authorUrl: https://roadie.io +category: Project Management +description: View Jira summary for your projects in Backstage. +documentation: https://roadie.io/backstage/plugins/jira +iconUrl: https://roadie.io/images/logos/jira.png +npmPackageName: '@roadiehq/backstage-plugin-jira' diff --git a/microsite/package.json b/microsite/package.json index 3b5ab2eeb4..6be0579475 100644 --- a/microsite/package.json +++ b/microsite/package.json @@ -17,7 +17,7 @@ "@spotify/prettier-config": "^9.0.0", "docusaurus": "^2.0.0-alpha.66", "js-yaml": "^3.14.0", - "prettier": "^2.2.0" + "prettier": "^2.2.1" }, "prettier": "@spotify/prettier-config" } diff --git a/microsite/siteConfig.js b/microsite/siteConfig.js index 6e8f8edddf..517cac2496 100644 --- a/microsite/siteConfig.js +++ b/microsite/siteConfig.js @@ -86,7 +86,11 @@ const siteConfig = { }, // Add custom scripts here that would be placed in