diff --git a/.github/ISSUE_TEMPLATE/feature_template.md b/.github/ISSUE_TEMPLATE/feature_template.md index 012b8d7a06..d70622bf52 100644 --- a/.github/ISSUE_TEMPLATE/feature_template.md +++ b/.github/ISSUE_TEMPLATE/feature_template.md @@ -1,7 +1,7 @@ --- name: 'Feature Request' about: 'Suggest new features and changes' -labels: help wanted +labels: enhancement --- diff --git a/.github/ISSUE_TEMPLATE/rfc_template.md b/.github/ISSUE_TEMPLATE/rfc_template.md new file mode 100644 index 0000000000..c4990ee5d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc_template.md @@ -0,0 +1,26 @@ +--- +name: 'RFC' +about: 'Request For Comments (RFC) from the community' +labels: rfc +title: '[RFC] ' +--- + +**Status:** Open for comments + + + +## Need + + + +## Proposal + + + +## Alternatives + + + +## Risks + + diff --git a/.github/workflows/e2e-win.yml b/.github/workflows/e2e-win.yml index 5ebf67bc9a..b4ea1e350f 100644 --- a/.github/workflows/e2e-win.yml +++ b/.github/workflows/e2e-win.yml @@ -49,6 +49,6 @@ jobs: - run: yarn tsc - name: yarn build - run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli --ignore backstage-microsite + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli - name: run E2E test run: yarn workspace e2e-test start diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 401fe7297a..d53ca1b607 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -64,7 +64,7 @@ jobs: - run: yarn tsc - name: yarn build - run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli --ignore backstage-microsite + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli - name: run E2E test run: | sudo sysctl fs.inotify.max_user_watches=524288 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 04b50c3e60..354838037e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -74,3 +74,11 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" package_root: "packages/core" tag_prefix: "v" + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + with: + args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' diff --git a/.github/workflows/microsite-build-check.yml b/.github/workflows/microsite-build-check.yml index dc86ca2630..ff290b71e4 100644 --- a/.github/workflows/microsite-build-check.yml +++ b/.github/workflows/microsite-build-check.yml @@ -22,33 +22,17 @@ jobs: steps: - uses: actions/checkout@v2 - # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - registry-url: https://registry.npmjs.org/ # Needed for auth - - name: cache all node_modules - id: cache-modules - uses: actions/cache@v2 - with: - path: '**/node_modules' - key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} - - name: find location of global yarn cache - id: yarn-cache - if: steps.cache-modules.outputs.cache-hit != 'true' - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - if: steps.cache-modules.outputs.cache-hit != 'true' - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + + # Skip caching of microsite dependencies, it keeps the global cache size + # smaller, which make Windows builds a lot faster for the rest of the project. - name: yarn install run: yarn install --frozen-lockfile - # End of yarn setup + working-directory: microsite - name: build microsite - run: yarn workspace backstage-microsite build + run: yarn build + working-directory: microsite diff --git a/.github/workflows/microsite-with-storybook-deploy.yml b/.github/workflows/microsite-with-storybook-deploy.yml index 2c8b8af40c..a1e9723be0 100644 --- a/.github/workflows/microsite-with-storybook-deploy.yml +++ b/.github/workflows/microsite-with-storybook-deploy.yml @@ -54,8 +54,14 @@ jobs: run: yarn install --frozen-lockfile # End of yarn setup + # Previous install was to be able to build storybook, this is for the microsite, and we skip caching + - name: yarn install + run: yarn install --frozen-lockfile + working-directory: microsite + - name: build microsite - run: yarn workspace backstage-microsite build + run: yarn build + working-directory: microsite - name: build storybook run: yarn workspace storybook build-storybook diff --git a/.gitignore b/.gitignore index 743561c3b6..9884edf96e 100644 --- a/.gitignore +++ b/.gitignore @@ -90,9 +90,6 @@ typings/ .nuxt dist -# Microsite build output -microsite/build - # Gatsby files .cache/ # Comment in the public line in if your project uses Gatsby and not Next.js diff --git a/ADOPTERS.md b/ADOPTERS.md index dc02ad3b85..0ba3e3196b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -12,3 +12,4 @@ | [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | | [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | | [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | diff --git a/CHANGELOG.md b/CHANGELOG.md index ce39a9f8d2..20ae9a414d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re > Collect changes for the next release below +- The backend plugin + [service builder](https://github.com/spotify/backstage/blob/master/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts) + no longer adds `express.json()` automatically to all routes. While convenient + in a lot of cases, it also led to problems where for example the proxy + middleware could hang because the body had already been altered and could not + be streamed. Also, plugins that rather wanted to handle e.g. form encoded data + still had to cater to that manually. We therefore decided to let plugins add + `express.json()` themselves if they happen to deal with JSON data. + ## v0.1.1-alpha.20 - Includes https://github.com/spotify/backstage/pull/2097 to resolve issues with create-plugin command. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b8febd1d2..853332f74d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,4 @@ ---- -id: CONTRIBUTING -title: Contributing ---- +# Contributing to Backstage Our vision for Backstage is for it to become the trusted standard toolbox (read: UX layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We canโ€™t do it alone. @@ -31,7 +28,7 @@ What kind of plugins should/could be created? Some inspiration from the 120+ plu ## Suggesting a plugin -If you start developing a plugin that you aim to release as open source, we suggest that you create a new [new Issue](https://github.com/spotify/backstage/issues/new?template=plugin_template.md). This helps the community know what plugins are in development. +If you start developing a plugin that you aim to release as open source, we suggest that you create a new [new Issue](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME). This helps the community know what plugins are in development. You can also use this process if you have an idea for a good plugin but you hope that someone else will pick up the work. diff --git a/README.md b/README.md index b5e8704548..4009864edc 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Take a look at the [Getting Started](https://backstage.io/docs/getting-started/i ## Documentation -- [Main documentation](https://backstage.io/docs/overview/what-is-backstage) +- [Main documentation](https://backstage.io/docs) - [Service Catalog](https://backstage.io/docs/features/software-catalog/software-catalog-overview) - [Architecture](https://backstage.io/docs/overview/architecture-terminology) ([Decisions](https://backstage.io/docs/architecture-decisions/adrs-overview)) - [Designing for Backstage](https://backstage.io/docs/dls/design) diff --git a/app-config.yaml b/app-config.yaml index 5347fbe617..957d17990a 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -44,7 +44,12 @@ newrelic: baseUrl: 'https://api.newrelic.com/v2' key: NEW_RELIC_REST_API_KEY +lighthouse: + baseUrl: http://localhost:3003 + catalog: + rules: + - allow: [Component, API, Group, Template, Location] processors: githubApi: privateToken: @@ -80,6 +85,8 @@ catalog: - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/hello-world-api.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/streetlights-api.yaml auth: providers: @@ -113,10 +120,9 @@ auth: audience: $secret: env: GITLAB_BASE_URL - # saml: - # development: - # entryPoint: "http://localhost:7001/" - # issuer: "passport-saml" + saml: + entryPoint: "http://localhost:7001/" + issuer: "passport-saml" okta: development: clientId: diff --git a/catalog-info.yaml b/catalog-info.yaml index 4221ee6dc6..937a55bfca 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -3,10 +3,9 @@ kind: Component metadata: name: backstage description: | - Backstage is an open-source developer portal that puts the developer experience first. + Backstage is an open-source developer portal that puts the developer experience first. annotations: github.com/project-slug: spotify/backstage - backstage.io/github-actions-id: spotify/backstage backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git spec: type: library diff --git a/docs/auth/add-auth-provider.md b/docs/auth/add-auth-provider.md index a2830dcf4f..7e4f5c8e2f 100644 --- a/docs/auth/add-auth-provider.md +++ b/docs/auth/add-auth-provider.md @@ -48,22 +48,35 @@ provider class which implements a handler for the chosen framework. #### Adding an OAuth based provider If we're adding an `OAuth` based provider we would implement the -[OAuthProviderHandlers](#OAuthProviderHandlers) interface. +[OAuthProviderHandlers](#OAuthProviderHandlers) interface. By implementing this +interface we can use the `OAuthProvider` class provided by `lib/oauth`, meaning +we don't need to implement the full +[AuthProviderRouteHandlers](#AuthProviderRouteHandlers) interface that providers +otherwise need to implement. -The provider class takes the provider's configuration as a class parameter. It -also imports the `Strategy` from the passport package. +The provider class takes the provider's options as a class parameter. It also +imports the `Strategy` from the passport package. ```ts import { Strategy as ProviderAStrategy } from 'passport-provider-a'; +export type ProviderAProviderOptions = OAuthProviderOptions & { + // extra options here +} + export class ProviderAAuthProvider implements OAuthProviderHandlers { - private readonly providerConfig: AuthProviderConfig; private readonly _strategy: ProviderAStrategy; - constructor(providerConfig: AuthProviderConfig) { - this.providerConfig = providerConfig; + constructor(options: ProviderAProviderOptions) { this._strategy = new ProviderAStrategy( - { ...providerConfig.options }, + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + passReqToCallback: false as true, + response_type: 'code', + /// ... etc + } verifyFunction, // See the "Verify Callback" section ); } @@ -82,14 +95,18 @@ An non-`OAuth` based provider could implement [AuthProviderRouteHandlers](#AuthProviderRouteHandlers) instead. ```ts +type ProviderAOptions = { + // ... +}; + export class ProviderAAuthProvider implements AuthProviderRouteHandlers { - private readonly providerConfig: AuthProviderConfig; private readonly _strategy: ProviderAStrategy; - constructor(providerConfig: AuthProviderConfig) { - this.providerConfig = providerConfig; + constructor(options: ProviderAOptions) { this._strategy = new ProviderAStrategy( - { ...providerConfig.options }, + { + // ... + }, verifyFunction, // See the "Verify Callback" section ); } @@ -101,31 +118,61 @@ export class ProviderAAuthProvider implements AuthProviderRouteHandlers { } ``` -#### Create method +#### Factory function -Each provider exports a create method that creates the provider instance, -optionally extending a supported authorization framework. This method exists to -allow for flexibility if additional frameworks are supported in the future. +Each provider exports a factory function that instantiates the provider. The +factory should implement [AuthProviderFactory](#AuthProviderFactory), which +passes in a object with utilities for configuration, logging, token issuing, +etc. The factory should return an implementation of +[AuthProviderRouteHandlers](#AuthProviderRouteHandlers). -Implementing OAuth by returning an instance of `OAuthProvider` based of the -provider's class: +The factory is what decides the mapping from +[static configuration](../conf/index.md) to the creation of auth providers. For +example, OAuth providers use `OAuthEnvironmentHandler` to allow for multiple +different configurations, one for each environment, which looks like this; ```ts -export function createProviderAProvider(config: AuthProviderConfig) { - const provider = new ProviderAAuthProvider(config); - const oauthProvider = new OAuthProvider(provider, config.provider, true); - return oauthProvider; -} +export const createOktaProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + // read options from config + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + + // instantiate our OAuthProviderHandlers implementation + const provider = new OktaAuthProvider({ + audience, + clientId, + clientSecret, + callbackUrl, + }); + + // Wrap the OAuthProviderHandlers with OAuthProvider, which implements AuthProviderRouteHandlers + return OAuthProvider.fromConfig(globalConfig, provider, { + disableRefresh: false, + providerId, + tokenIssuer, + }); + }); ``` -Not extending with OAuth, the main difference here is that the create method is -returning a instance of the class without adding the OAuth authorization -framework to it. +The purpose of the different environments is to allow for a single auth-backend +to serve as the authentication service for multiple different frontend +environments, such as local development, staging, and production. + +The factory function for other providers can be a lot simpler, as they might not +have configuration for each environment. Looking something like this: ```ts -export function createProviderAProvider(config: AuthProviderConfig) { - return new ProviderAAuthProvider(config); -} +export const createProviderAProvider: AuthProviderFactory = ({ config }) => { + const a = config.getString('a'); + const b = config.getString('b'); + + return new ProviderAAuthProvider({ a, b }); +}; ``` #### Verify Callback @@ -144,7 +191,7 @@ export function createProviderAProvider(config: AuthProviderConfig) { > http://www.passportjs.org/docs/configure/ **`plugins/auth-backend/src/providers/providerA/index.ts`** is simply -re-exporting the create method to be used for hooking the provider up to the +re-exporting the factory function to be used for hooking the provider up to the backend. ```ts @@ -153,26 +200,14 @@ export { createProviderAProvider } from './provider'; ### Hook it up to the backend -**`plugins/auth-backend/src/providers/config.ts`** The provider needs to be -configured properly so you need to add it to the list of configured providers, -all of which implement [AuthProviderConfig](#AuthProviderConfig): - -```ts -export const providers = [ - { - provider: 'providerA', # used as an identifier - options: { ... }, # consult the provider documentation for which options you should provide - disableRefresh: true # if the provider lacks refresh tokens - }, -``` - **`plugins/auth-backend/src/providers/factories.ts`** When the `auth-backend` starts it sets up routing for all the available providers by calling -`createAuthProviderRouter` on each provider. You need to import the create -method from the provider and add it to the factory: +`createAuthProviderRouter` on each provider. You need to import the factory +function from the provider and add it to the factory: ```ts import { createProviderAProvider } from './providerA'; + const factories: { [providerId: string]: AuthProviderFactory } = { providerA: createProviderAProvider, }; @@ -203,10 +238,21 @@ web browser and you should be able to trigger the authorization flow. ```ts export interface OAuthProviderHandlers { - start(req: express.Request, options: any): Promise; - handler(req: express.Request): Promise; - refresh?(refreshToken: string, scope: string): Promise; - logout?(): Promise; + start( + req: express.Request, + options: Record, + ): Promise; + handler( + req: express.Request, + ): Promise<{ + response: AuthResponse; + refreshToken?: string; + }>; + refresh?( + refreshToken: string, + scope: string, + ): Promise>; + logout?(): Promise; } ``` @@ -221,12 +267,17 @@ export interface AuthProviderRouteHandlers { } ``` -##### AuthProviderConfig +##### AuthProviderFactory ```ts -export type AuthProviderConfig = { - provider: string; - options: any; - disableRefresh?: boolean; +export type AuthProviderFactoryOptions = { + globalConfig: AuthProviderConfig; + config: Config; + logger: Logger; + tokenIssuer: TokenIssuer; }; + +export type AuthProviderFactory = ( + options: AuthProviderFactoryOptions, +) => AuthProviderRouteHandlers; ``` diff --git a/docs/auth/auth-backend-classes.md b/docs/auth/auth-backend-classes.md index 9034a3d054..0147c66dd9 100644 --- a/docs/auth/auth-backend-classes.md +++ b/docs/auth/auth-backend-classes.md @@ -26,7 +26,7 @@ refer to the type documentation under `plugins/auth-backend/src/providers/types.ts`. There are currently two different classes for two authentication mechanisms that -implement this interface: an `OAuthProvider` for [OAuth](https://oauth.net/2/) +implement this interface: an `OAuthAdapter` for [OAuth](https://oauth.net/2/) based mechanisms and a `SAMLAuthProvider` for [SAML](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html) based mechanisms. @@ -50,11 +50,12 @@ OAuth2) providers, you can configure them by setting the right variables in Each authentication provider (except SAML) needs five parameters: an OAuth client ID, a client secret, an authorization endpoint and a token endpoint, and an app origin. The app origin is the URL at which the frontend of the -application is hosted. This is required because the application opens a popup -window to perform the authentication, and once the flow is completed, the popup -window sends a `postMessage` to the frontend application to indicate the result -of the operation. Also this URL is used to verify that authentication requests -are coming from only this endpoint. +application is hosted, and it is read from the `app.baseUrl` config. This is +required because the application opens a popup window to perform the +authentication, and once the flow is completed, the popup window sends a +`postMessage` to the frontend application to indicate the result of the +operation. Also this URL is used to verify that authentication requests are +coming from only this endpoint. These values are configured via the `app-config.yaml` present in the root of your app folder. @@ -64,8 +65,6 @@ auth: providers: google: development: - appOrigin: "http://localhost:3000/" - secure: false clientId: $secret: env: AUTH_GOOGLE_CLIENT_ID @@ -74,8 +73,6 @@ auth: env: AUTH_GOOGLE_CLIENT_SECRET github: development: - appOrigin: "http://localhost:3000/" - secure: false clientId: $secret: env: AUTH_GITHUB_CLIENT_ID @@ -87,8 +84,6 @@ auth: env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL gitlab: development: - appOrigin: "http://localhost:3000/" - secure: false clientId: $secret: ... @@ -96,24 +91,34 @@ auth: ## Technical Notes -### EnvironmentHandler +### OAuthEnvironmentHandler The concept of an "env" is core to the way the auth backend works. It uses an `env` query parameter to identify the environment in which the application is running (`development`, `staging`, `production`, etc). Each runtime can support multiple environments at the same time and the right handler for each request is identified and dispatched to based on the `env` parameter. All -`AuthProviderRouteHandlers` are wrapped within an `EnvironmentHandler`. +`AuthProviderRouteHandlers` are wrapped within an `OAuthEnvironmentHandler`. -An `EnvironmentHandler` takes an ID for each provider that it wraps, the -handlers for each env the provider is supported in, and a function that, given a -`Request` as argument, can extract the information about the env under which it -should be processed. +To instantiate multiple OAuth providers for different environments, use +`OAuthEnvironmentHandler.mapConfig`. It's a helper to iterate over a +configuration object that is a map of environment to configurations. See one of +the existing OAuth providers for an example of how it is used. -Each provider exposes a factory function `createXProvider` (where X is the name -of the provider) that takes the global config, env and other parameters and -returns an `AuthProviderRouteHandlers` for each env, and an `envIdentifier` -function to identify the env of a request. +Given the following configuration: + +```yaml +development: + clientId: abc + clientSecret: secret +production: + clientId: xyz + clientSecret: supersecret +``` + +The `OAuthEnvironmentHandler.mapConfig(config, envConfig => ...)` call will +split the `config` by the top level `development` and `production` keys, and +pass on each block as `envConfig`. For a list of currently available providers, look in the `factories` module located in `plugins/auth-backend/src/providers/factories.ts` diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md new file mode 100644 index 0000000000..be6314e494 --- /dev/null +++ b/docs/features/software-catalog/configuration.md @@ -0,0 +1,58 @@ +--- +id: software-catalog-configuration +title: Catalog Configuration +--- + +## Static Location Configuration + +To enable declarative catalog setups, it is possible to add locations to the +catalog via [static configuration](../../conf/index.md). Locations are added to +the catalog under the `catalog.locations` key, for example: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +The locations added through static configuration can not be removed through the +catalog locations API. To remove the locations, you have to remove them from the +configuration. + +## Catalog Rules + +By default the catalog will only allow ingestion of entities with the kind +`Component`, `API` and `Location`. In order to allow entities of other kinds to +be added, you need to add rules to the catalog. Rules are added either in a +separate `catalog.rules` key, or added to statically configured locations. + +For example, given the following configuration: + +```yaml +catalog: + rules: + - allow: [Component, API, Location, Template] + + locations: + - type: github + target: https://github.com/org/example/blob/master/org-data.yaml + rules: + - allow: [Group] +``` + +We are able to add entities of kind `Component`, `API`, `Location`, or +`Template` from any location, and `Group` entities from the `org-data.yaml`, +which will also be read as statically configured location. + +Note that if the `catalog.rules` key is present it will replace the default +value, meaning that you need to add rules for the default kinds if you want +those to still be allowed. + +The following configuration will reject any kind of entities from being added to +the catalog: + +```yaml +catalog: + rules: [] +``` diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 78dfcf365c..e90792c628 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -235,6 +235,9 @@ The `backstage.io/` prefix is reserved for use by Backstage core components. Values can be of any length, but are limited to being strings. +There is a list of [well-known annotations](well-known-annotations.md), but +anybody is free to add more annotations as they see fit. + ### `tags` [optional] A list of single-valued strings, for example to classify catalog entities in @@ -290,7 +293,7 @@ Exactly equal to `backstage.io/v1alpha1` and `Component`, respectively. The type of component as a string, e.g. `website`. This field is required. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface @@ -304,9 +307,9 @@ The current set of well-known and common values for this field is: ### `spec.lifecycle` [required] -The lifecyle state of the component, e.g. `production`. This field is required. +The lifecycle state of the component, e.g. `production`. This field is required. -The software catalog accepts any lifecycle value, but an organisation should +The software catalog accepts any lifecycle value, but an organization should take great care to establish a proper taxonomy for these. The current set of well-known and common values for this field is: @@ -418,7 +421,7 @@ potentially search and group templates by these tags. The type of component as a string, e.g. `website`. This field is optional but recommended. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface @@ -508,7 +511,7 @@ Exactly equal to `backstage.io/v1alpha1` and `API`, respectively. The type of the API definition as a string, e.g. `openapi`. This field is required. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, an OpenAPI type API may be displayed using an OpenAPI viewer tooling in diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index db8c183390..bfc3d53d4d 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -79,6 +79,22 @@ All software created through the [Backstage Software Templates](../software-templates/index.md) are automatically registered in the catalog. +### Static catalog configuration + +In addition to manually registering components, it is also possible to register +components though [static configuration](../../conf/index.md). For example, the +above example can be added using the following configuration: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +More information about catalog configuration can be found +[here](configuration.md). + ### Updating component metadata Teams owning the components are responsible for maintaining the metadata about diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md new file mode 100644 index 0000000000..159cfb4ca1 --- /dev/null +++ b/docs/features/software-catalog/well-known-annotations.md @@ -0,0 +1,116 @@ +--- +id: well-known-annotations +title: Well-known Annotations on Catalog Entities +sidebar_label: Well-known Annotations +--- + +This section lists a number of well known +[annotations](descriptor-format.md#annotations-optional), that have defined +semantics. They can be attached to catalog entities and consumed by plugins as +needed. + +## Annotations + +This is a (non-exhaustive) list of annotations that are known to be in active +use. + +### backstage.io/managed-by-location + +```yaml +# Example: +metadata: + annotations: + backstage.io/managed-by-location: github:http://github.com/spotify/backstage/catalog-info.yaml +``` + +The value of this annotation is a so called location reference string, that +points to the source from which the entity was originally fetched. This +annotation is added automatically by the catalog as it fetches the data from a +registered location, and is not meant to normally be written by humans. The +annotation may point to any type of generic location that the catalog supports, +so it cannot be relied on to always be specifically of type `github`, nor that +it even represents a single file. Note also that a single location can be the +source of many entities, so it represents a many-to-one relationship. + +The format of the value is `:`. Note that the target may also +contain colons, so it is not advisable to naively split the value on `:` and +expecting a two-item array out of it. The format of the target part is +type-dependent and could conceivably even be an empty string, but the separator +colon is always present. + +### backstage.io/techdocs-ref + +```yaml +# Example: +metadata: + annotations: + backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git +``` + +The value of this annotation is a location reference string (see above). If this +annotation is specified, it is expected to point to a repository that the +TechDocs system can read and generate docs from. + +### backstage.io/jenkins-github-folder + +```yaml +# Example: +metadata: + annotations: + backstage.io/jenkins-github-folder: folder-name/job-name +``` + +The value of this annotation is the path to a job on Jenkins, that builds this +entity. + +Specifying this annotation may enable Jenkins related features in Backstage for +that entity. + +### github.com/project-slug + +```yaml +# Example: +metadata: + annotations: + github.com/project-slug: spotify/backstage +``` + +The value of this annotation is the so-called slug that identifies a project on +[GitHub](https://github.com) that is related to this entity. It is on the format +`/`, and is the same as can be seen in the URL location +bar of the browser when viewing that project. + +Specifying this annotation will enable GitHub related features in Backstage for +that entity. + +### sentry.io/project-slug + +```yaml +# Example: +metadata: + annotations: + sentry.io/project-slug: pump-station +``` + +The value of this annotation is the so-called slug (or alternatively, the ID) of +a [Sentry](https://sentry.io) project within your organization. The organization +slug is currently not configurable on a per-entity basis, but is assumed to be +the same for all entities in the catalog. + +Specifying this annotation may enable Sentry related features in Backstage for +that entity. + +## Deprecated Annotations + +The following annotations are deprecated, and only listed here to aid in +migrating away from them. + +### backstage.io/github-actions-id + +This annotation was used for a while to enable the GitHub Actions feature. This +is now instead using the [github.com/project-slug](#github-com-project-slug) +annotation, with the same value format. + +## Links + +- [Descriptor Format: annotations](descriptor-format.md#annotations-optional) diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 67e9c23605..4f7af7c015 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -58,7 +58,30 @@ Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. -For loading from a file the following command should work when the backend is +You can add the template files to the catalog through +[static location configuration](../software-catalog/configuration.md#static-location-configuration), +for example + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + rules: + - allow: [Template] +``` + +Templates can also be added by posting the to the catalog directly. Note that if +you're doing this, you need to configure the catalog to allow template entities +to be ingested from any source, for example: + +```yaml +catalog: + rules: + - allow: [Component, API, Template] +``` + +For loading from a file, the following command should work when the backend is running: ```sh @@ -69,7 +92,7 @@ curl \ --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" ``` -If loading from a git location, you can run the following +If loading from a Git location, you can run the following ```sh curl \ @@ -83,7 +106,7 @@ This should then have added the catalog, and also should now be listed under the create page at http://localhost:3000/create. Alternatively, if you want to get setup with some mock templates that are -already provided for you, you can run the following to load those templates: +already provided, run the following to load those templates: ``` yarn lerna run mock-data diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index 51869dc3ad..391f6f0658 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -7,8 +7,8 @@ Publishers are responsible for pushing and storing the templated skeleton after the values have been templated by the `Templater`. See [Create your own templater](./create-your-own-templater.md) for more info. -They recieve a directory or location where the templater has sucessfully run on, -and is now ready to store somewhere. They also get given some other options +They receive a directory or location where the templater has sucessfully run +and is now ready to store somewhere. They also are given some other options which are sent from the frontend, such as the `storePath` which is a string of where the frontend thinks we should save this templated folder. @@ -17,7 +17,7 @@ Currently we provide the following `publishers`: - `github` This publisher is passed through to the `createRouter` function of the -`@spotify/plugin-scaffolder-backend`. Currently only one publisher is supported, +`@spotify/plugin-scaffolder-backend`. Currently, only one publisher is supported, but PR's are always welcome. An full example backend can be found diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md index a39f87924b..9dee66af06 100644 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ b/docs/features/software-templates/extending/create-your-own-templater.md @@ -8,14 +8,14 @@ returned by the preparers, and then executing the templating command on top of the file and returning the completed template path. This may or may not be the same directory as the input directory. -They also recieve additional values from the frontend, which can be used to +They also receive additional values from the frontend, which can be used to interpolate into the skeleton files. Currently we provide the following templaters: - `cookiecutter` -This templater is added the `TemplaterBuilder` and then passed into the +This templater is added to the `TemplaterBuilder` and then passed into the `createRouter` function of the `@spotify/plugin-scaffolder-backend` An full example backend can be found @@ -48,7 +48,7 @@ This `TemplaterKey` is used to select the correct templater from the `spec.templater` in the [Template Entity](../../software-catalog/descriptor-format.md#kind-template). -If you wish to add a new templater you'll need to register it with the +If you wish to add a new templater, you'll need to register it with the `TemplaterBuilder`. ### Creating your own Templater to add to the `TemplaterBuilder` @@ -83,10 +83,10 @@ follows: - `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to be able to run docker containers. -_note_ currently the templaters that we provide are basically docker action +_note_ Currently the templaters that we provide are basically Docker action containers that are run on top of the skeleton folder. This keeps dependencies to a minimal for running backstage scaffolder, but you don't /have/ to use -docker. You could create your own templater that spins up an EC2 instance and +Docker. You could create your own templater that spins up an EC2 instance and downloads the folder and does everything using an AMI if you want. It's entirely up to you! @@ -138,7 +138,7 @@ spec: description: Description of the component ``` -You see that the `spec.templater` is set as `handlebars`, you'll need to +You see that the `spec.templater` is set as `handlebars`, so you'll need to register this with the `TemplaterBuilder` like so: ```ts diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md index 21d8befda1..02d27a5725 100644 --- a/docs/features/software-templates/extending/index.md +++ b/docs/features/software-templates/extending/index.md @@ -5,11 +5,11 @@ title: Extending the Scaffolder Welcome. Take a seat. You're at the Scaffolder Documentation. -So - You wanna create stuff inside your company from some prebaked templates? +So, you want to create stuff inside your company from some prebaked templates? You're at the right place. -This guide is gonna take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through whats going on in the backend to be +This guide is going to take you through how the Scaffolder in Backstage works. +We'll dive into some jargon and run through what's going on in the backend to be able to create these templates. There's also more guides that you might find useful at the bottom of this document. At it's core, theres 3 simple stages. @@ -25,9 +25,9 @@ scaffolder that you will need to know: 3. Publish Each of these steps can be configured for your own use case, but we provide some -sensible defaults too. +sensible defaults, too. -Lets dive a little deeper into these phases. +Let's dive a little deeper into these phases. ### Glossary and Jargon @@ -38,8 +38,8 @@ the router to pick the correct `Preparer` to run for the `Template` entity. **Templater** - The templater is responsible for actually running the chosen templater on top of the previously returned temporary directory from the -**Preprarer**. We advise making these docker containers as it can keep all -dependencies, for example Cookiecutter, self contained and not a dependency on +**Preprarer**. We advise making these Docker containers as it can keep all +dependencies--for example Cookiecutter--self contained and not a dependency on the host machine. **Publisher** - The publisher is responsible for taking the finished directory, @@ -50,11 +50,11 @@ passed through to the scaffolder backend. ### How it works -The main of the heavy lifting is done in the +Most of the heavy lifting is done in the [router.ts](https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) file in the `scaffolder-backend` plugin. -There are 2 routes defined in the router. `POST /v1/jobs` and +There are two routes defined in the router: `POST /v1/jobs` and `GET /v1/job/:jobId` To create a scaffolding job, a JSON object containing the @@ -78,7 +78,7 @@ additional templating values must be posted as the post body. The values should represent something that is valid with the `schema` part of the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) -Once that has been posted, a job will be setup with different stages. And the +Once that has been posted, a job will be setup with different stages, and the job processor will complete each stage before moving onto the next stage, whilst collecting logs and mutating the running job. diff --git a/docs/features/software-templates/index.md b/docs/features/software-templates/index.md index 503fbb7a3d..1c67676b3e 100644 --- a/docs/features/software-templates/index.md +++ b/docs/features/software-templates/index.md @@ -4,8 +4,8 @@ title: Software Templates --- The Software Templates part of Backstage is a tool that can help you create -Components inside Backstage. It by default has the ability to load skeletons of -code, template in some variables and then publish the template to some location +Components inside Backstage. By default, it has the ability to load skeletons of +code, template in some variables, and then publish the template to some location like GitHub.