diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index cc20afaf57..fb69937613 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -143,6 +143,7 @@ css dariddler dataflow deadnaming +declaratively destructured dev devops diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index 3ffd01a38a..fb5d664f16 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -11,79 +11,13 @@ such as reading raw entity data from a remote source, parsing it, transforming it, and validating it. These processors are configured under the `catalog.processors` configuration key. -### Processor: url +### Static Location Configuration -The `url` processor is responsible for fetching entity data from files in any -external provider like GitHub, GitLab, Bitbucket, etc. Unlike other processors, -the configuration of this processor lives under the top-level `integrations` -key, as it is used by other parts of Backstage too. +The simplest configuration for the catalog, as shown in the default +`@backstage/create-app` template, is to declaratively add locations pointing to +YAML files with [static configuration](../../conf/index.md). -```yaml -integrations: - github: - - host: github.com - token: - $env: GITHUB_TOKEN - - host: ghe.example.net - apiBaseUrl: https://ghe.example.net/api/v3 - rawBaseUrl: https://ghe.example.net/raw - token: - $env: GHE_TOKEN - gitlab: - - host: gitlab.com - token: - $env: GITLAB_TOKEN - bitbucket: - - host: bitbucket.org - username: - $env: BITBUCKET_USERNAME - appPassword: - $env: BITBUCKET_APP_PASSWORD - azure: - - host: dev.azure.com - token: - $env: AZURE_TOKEN -``` - -Each key under `integrations` is a separate configuration for each external -provider. The providers each have their own configuration, so let's look at the -GitHub section as an example. - -Directly under the `github` key is a list of provider configurations, where you -can list the various GitHub compatible providers you want to be able to fetch -data from. Each entry is a structure with up to four elements: - -- `host` (optional): The host of the location target that you want to match on. - The default host is `github.com`. -- `token` (optional): An authentication token as expected by GitHub. If - supplied, it will be passed along with all calls to this provider, both API - and raw. If it is not supplied, anonymous access will be used. -- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method - with this provider, specify the base URL for its endpoint here, with no - trailing slash. Specifically when the target is GitHub, you can leave it out - to be inferred automatically. For a GitHub Enterprise installation, it is - commonly at `https://api.` or `https:///api/v3`. -- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method - with this provider, specify the base URL for its endpoint here, with no - trailing slash. Specifically when the target is public GitHub, you can leave - it out to be inferred automatically. For a GitHub Enterprise installation, it - is commonly at `https://api.` or `https:///api/v3`. - -You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for -public GitHub, for which we can infer them). The `apiBaseUrl` will always be -preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be -preferred. - -If you do not supply a public GitHub provider, one will be added automatically, -silently at startup for convenience. So you only have to list it if you want to -supply a token for it - and if you do, you can also leave out the `apiBaseUrl` -and `rawBaseUrl` fields. - -## Static Location Configuration - -To enable declarative catalog setups, it is possible to add locations to the -catalog via [static configuration](../../conf/index.md). Locations are added to -the catalog under the `catalog.locations` key, for example: +Locations are added to the catalog under the `catalog.locations` key: ```yaml catalog: @@ -92,10 +26,35 @@ catalog: target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` -The locations added through static configuration can not be removed through the -catalog locations API. To remove the locations, you have to remove them from the +The `url` type locations are handled by a standard processor included with the +catalog (`UrlReaderProcessor`), so no processor configuration is needed. This +processor _does however_ need an [integration](../../integrations/index.md) to +understand how to retrieve a given URL. For the example above, you would need to +configure the [GitHub integration](../../integrations/github/locations.md) to +read files from github.com. + +The locations added through static configuration cannot be removed through the +catalog locations API. To remove these locations, you must remove them from the configuration. +### Integration Processors + +Integrations may simply provide a mechanism to handle `url` location type for an +external provider, or they may also include additional processors out of the +box, such as the GitHub [discovery](../../integrations/github/discovery.md) +processor that scans a GitHub organization for +[entity descriptor files](descriptor-format.md). + +Check the [integrations](../../integrations/index.md) documentation to see what +is offered by each integration. + +### Custom Processors + +To ingest entities from an existing system already tracking software, you can +also write a _custom processor_ to convert between the existing system and +Backstage's descriptor format. This is documented in +[External Integrations](external-integrations.md). + ## Catalog Rules By default the catalog will only allow ingestion of entities with the kind diff --git a/docs/integrations/azure/locations.md b/docs/integrations/azure/locations.md new file mode 100644 index 0000000000..4ce19bad4e --- /dev/null +++ b/docs/integrations/azure/locations.md @@ -0,0 +1,30 @@ +--- +id: locations +title: Azure DevOps Locations +sidebar_label: Locations +description: + Integrating source code stored in Azure DevOps into the Backstage catalog +--- + +The Azure integration supports loading catalog entities from Azure DevOps. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +```yaml +integrations: + azure: + - host: dev.azure.com + token: ${AZURE_TOKEN} +``` + +> Note: An Azure DevOps provider is added automatically at startup for +> convenience, so you only need to list it if you want to supply a +> [token](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate). + +The configuration is a structure with two elements: + +- `host`: The DevOps host; only `dev.azure.com` is supported. +- `token` (optional): A personal access token as expected by Azure DevOps. diff --git a/docs/integrations/bitbucket/locations.md b/docs/integrations/bitbucket/locations.md new file mode 100644 index 0000000000..bf7a6af57c --- /dev/null +++ b/docs/integrations/bitbucket/locations.md @@ -0,0 +1,41 @@ +--- +id: locations +title: BitBucket Locations +sidebar_label: Locations +description: + Integrating source code stored in BitBucket into the Backstage catalog +--- + +The BitBucket integration supports loading catalog entities from bitbucket.com +or a self-hosted BitBucket. Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +```yaml +integrations: + bitbucket: + - host: bitbucket.org + username: ${BITBUCKET_USERNAME} + token: ${BITBUCKET_TOKEN} +``` + +> Note: A public BitBucket provider is added automatically at startup for +> convenience, so you only need to list it if you want to supply a +> [token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html). + +Directly under the `bitbucket` key is a list of provider configurations, where +you can list the BitBucket providers you want to fetch data from. Each entry is +a structure with up to four elements: + +- `host`: The host of the BitBucket instance, e.g. `bitbucket.company.com`. +- `token` (optional): An personal access token as expected by BitBucket. Either + an access token **or** a username + appPassword may be supplied. +- `username`: The BitBucket username to use in API requests. If neither a + username nor token are supplied, anonymous access will be used. +- `appPassword` (optional): The password for the BitBucket user. Only needed + when using `username` instead of `token`. +- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted + installations, it is commonly at `https:///api/v4`. For gitlab.com, this + configuration is not needed as it can be inferred. diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md new file mode 100644 index 0000000000..fe52feab6e --- /dev/null +++ b/docs/integrations/github/discovery.md @@ -0,0 +1,55 @@ +--- +id: discovery +title: GitHub Discovery +sidebar_label: Discovery +description: + Automatically discovering catalog entities from repositories in a GitHub + organization +--- + +The GitHub integration has a special discovery processor for discovering catalog +entities within a GitHub organization. The processor will crawl the GitHub +organization and register entities matching the configured path. This can be +useful as an alternative to static locations or manually adding things to the +catalog. + +To use the discovery processor, you'll need a GitHub integration +[set up](locations.md) with a `GITHUB_TOKEN`. Then you can add a location target +to the catalog configuration: + +```yaml +catalog: + locations: + - type: github-discovery + target: https://github.com/myorg/service-*/blob/main/catalog-info.yaml +``` + +Note the `github-discovery` type, as this is not a regular `url` processor. + +The target is composed of three parts: + +- The base organization URL, `https://github.com/myorg` in this case +- The repository blob to scan, which accepts \* wildcard tokens. This can simply + be `*` to scan all repositories in the organization. This example only looks + for repositories prefixed with `service-`. +- The path within each repository to find the catalog YAML file. This will + usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or + a similar variation for catalog files stored in the root directory of each + repository. + +## GitHub API Rate Limits + +GitHub +[rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) +API requests to 5,000 per hour (or more for Enterprise accounts). The default +Backstage catalog backend refreshes data every 100 seconds, which issues an API +request for each discovered location. + +This means if you have more than ~140 catalog entities, you may get throttled by +rate limiting. This will soon be resolved once catalog refreshes make use of +ETags; to work around this in the meantime, you can change the refresh rate of +the catalog in your `packages/backend/src/plugins/catalog.ts` file, or configure +Backstage to use the [github-apps plugin](../../plugins/github-apps.md). + +This is true for any method of adding GitHub entities to the catalog, but +especially easy to hit with automatic discovery. diff --git a/docs/integrations/github/locations.md b/docs/integrations/github/locations.md new file mode 100644 index 0000000000..5b972f332c --- /dev/null +++ b/docs/integrations/github/locations.md @@ -0,0 +1,64 @@ +--- +id: locations +title: GitHub Locations +sidebar_label: Locations +description: Integrating source code stored in GitHub into the Backstage catalog +--- + +The GitHub integration supports loading catalog entities from github.com or +GitHub Enterprise. Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin, or [discovered](discovery.md) from a GitHub organization. Users and +Groups can also be [loaded from an organization](org.md). + +## Configuration + +To use this integration, add configuration to your root `app-config.yaml`: + +```yaml +integrations: + github: + - host: github.com + token: ${GITHUB_TOKEN} + - host: ghe.example.net + apiBaseUrl: https://ghe.example.net/api/v3 + rawBaseUrl: https://ghe.example.net/raw + token: ${GHE_TOKEN} +``` + +> Note: A public GitHub provider is added automatically at startup for +> convenience, so you only need to list it if you want to supply a +> [token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token). + +Directly under the `github` key is a list of provider configurations, where you +can list the various GitHub-compatible providers you want to be able to fetch +data from. Each entry is a structure with up to four elements: + +- `host` (optional): The host of the location target that you want to match on. + The default host is `github.com`. +- `token` (optional): An authentication token as expected by GitHub. If + supplied, it will be passed along with all calls to this provider, both API + and raw. If it is not supplied, anonymous access will be used. +- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is GitHub, you can leave it out + to be inferred automatically. For a GitHub Enterprise installation, it is + commonly at `https://api.` or `https:///api/v3`. +- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method + with this provider, specify the base URL for its endpoint here, with no + trailing slash. Specifically when the target is public GitHub, you can leave + it out to be inferred automatically. For a GitHub Enterprise installation, it + is commonly at `https:///raw`. + +You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for +public GitHub, for which we can infer them). The `apiBaseUrl` will always be +preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be +preferred. + +## Authentication with GitHub Apps + +Alternatively, Backstage can use GitHub Apps for backend authentication. This +has higher rate limits, and a clearer authorization model. See the +[github-apps plugin](../../plugins/github-apps.md) for how to set this up. diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 04a0620233..b52e811002 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -3,7 +3,7 @@ id: org title: GitHub Organizational Data sidebar_label: Org Data # prettier-ignore -description: Setting up ingestion of organizational data from GitHub +description: Importing users and groups from a GitHub organization into Backstage --- The Backstage catalog can be set up to ingest organizational data - users and @@ -46,8 +46,7 @@ catalog: providers: - target: https://github.com apiBaseUrl: https://api.github.com - token: - $env: GITHUB_TOKEN + token: ${GITHUB_TOKEN} ``` Locations point out the specific org(s) you want to import. The `type` of these diff --git a/docs/integrations/gitlab/locations.md b/docs/integrations/gitlab/locations.md new file mode 100644 index 0000000000..965d183a9d --- /dev/null +++ b/docs/integrations/gitlab/locations.md @@ -0,0 +1,38 @@ +--- +id: locations +title: GitLab Locations +sidebar_label: Locations +description: Integrating source code stored in GitLab into the Backstage catalog +--- + +The GitLab integration supports loading catalog entities from gitlab.com or a +self-hosted GitLab. Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +```yaml +integrations: + gitlab: + - host: gitlab.com + token: ${GITLAB_TOKEN} +``` + +> Note: A public GitLab provider is added automatically at startup for +> convenience, so you only need to list it if you want to supply a +> [token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html). + +Directly under the `gitlab` key is a list of provider configurations, where you +can list the GitLab providers you want to fetch data from. Each entry is a +structure with up to four elements: + +- `host`: The host of the GitLab instance, e.g. `gitlab.company.com`. +- `token` (optional): An authentication token as expected by GitLab. If this is + not supplied, anonymous access will be used. +- `apiBaseUrl` (optional): The URL of the GitLab API. For self-hosted + installations, it is commonly at `https:///api/v4`. For gitlab.com, this + configuration is not needed as it can be inferred. +- `baseUrl` (optional): The base URL for this provider, e.g. + `https://gitlab.com`. If this is not provided, it is assumed to be + `https://{host}`. diff --git a/docs/integrations/index.md b/docs/integrations/index.md new file mode 100644 index 0000000000..544ce08ac4 --- /dev/null +++ b/docs/integrations/index.md @@ -0,0 +1,34 @@ +--- +id: index +title: Integrations +sidebar_label: Overview +description: + Configuring Backstage to read or publish data with external providers using + integrations +--- + +Integrations allow Backstage to read or publish data using external providers +such as GitHub, GitLab, BitBucket, LDAP, or cloud providers. + +## Configuration + +Integrations are configured at the root level of `app-config.yaml` since +integrations are used by many Backstage core features and other plugins. + +Each key under `integrations` is a separate configuration for a single external +provider. Providers each have different configuration; here's an example of +configuration to use both GitHub and BitBucket: + +```yaml +integrations: + github: + - host: github.com + token: ${GITHUB_TOKEN} + bitbucket: + - host: bitbucket.org + username: ${BITBUCKET_USERNAME} + appPassword: ${BITBUCKET_APP_PASSWORD} +``` + +See documentation for each type of integration for full details on +configuration. diff --git a/docs/integrations/ldap/org.md b/docs/integrations/ldap/org.md index 5d25afceca..24a4335b8b 100644 --- a/docs/integrations/ldap/org.md +++ b/docs/integrations/ldap/org.md @@ -46,8 +46,7 @@ catalog: - target: ldaps://ds.example.net bind: dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net - secret: - $env: LDAP_SECRET + secret: ${LDAP_SECRET} users: dn: ou=people,ou=example,dc=example,dc=net options: @@ -93,8 +92,7 @@ authenticate) towards the server. It has the following fields. ```yaml dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net -secret: - $env: LDAP_SECRET +secret: ${LDAP_SECRET} ``` The `dn` is the full LDAP Distinguished Name for the user that the plugin diff --git a/microsite/sidebars.json b/microsite/sidebars.json index a41a77115f..322ffc2e03 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -105,20 +105,40 @@ } ], "Integrations": [ + "integrations/index", { "type": "subcategory", - "label": "GitHub", - "ids": ["integrations/github/org"] + "label": "Azure DevOps", + "ids": ["integrations/azure/locations"] }, { "type": "subcategory", - "label": "LDAP", - "ids": ["integrations/ldap/org"] + "label": "BitBucket", + "ids": ["integrations/bitbucket/locations"] + }, + { + "type": "subcategory", + "label": "GitHub", + "ids": [ + "integrations/github/locations", + "integrations/github/discovery", + "integrations/github/org" + ] + }, + { + "type": "subcategory", + "label": "GitLab", + "ids": ["integrations/gitlab/locations"] }, { "type": "subcategory", "label": "Google Analytics", "ids": ["integrations/google-analytics/installation"] + }, + { + "type": "subcategory", + "label": "LDAP", + "ids": ["integrations/ldap/org"] } ], "Plugins": [