diff --git a/.eslintignore b/.eslintignore index 2f59b98bca..c73074efb3 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,5 +1,6 @@ **/node_modules/** **/dist/** +**/dist-types/** **/storybook-static/** **/coverage/** **/build/** diff --git a/.github/ISSUE_TEMPLATE/feature_template.md b/.github/ISSUE_TEMPLATE/feature_template.md index 012b8d7a06..d70622bf52 100644 --- a/.github/ISSUE_TEMPLATE/feature_template.md +++ b/.github/ISSUE_TEMPLATE/feature_template.md @@ -1,7 +1,7 @@ --- name: 'Feature Request' about: 'Suggest new features and changes' -labels: help wanted +labels: enhancement --- diff --git a/.github/ISSUE_TEMPLATE/rfc_template.md b/.github/ISSUE_TEMPLATE/rfc_template.md new file mode 100644 index 0000000000..c4990ee5d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc_template.md @@ -0,0 +1,26 @@ +--- +name: 'RFC' +about: 'Request For Comments (RFC) from the community' +labels: rfc +title: '[RFC] ' +--- + +**Status:** Open for comments + + + +## Need + + + +## Proposal + + + +## Alternatives + + + +## Risks + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 57746da2e8..0437698301 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,7 +4,9 @@ That makes it easier to understand the change so we can :shipit: faster. --> #### :heavy_check_mark: Checklist + + - [ ] All tests are passing `yarn test` - [ ] Screenshots attached (for UI changes) - [ ] Relevant documentation updated diff --git a/.github/workflows/chromatic-storybook-test.yml b/.github/workflows/chromatic-storybook-test.yml index e38b494a73..0bcaf400d0 100644 --- a/.github/workflows/chromatic-storybook-test.yml +++ b/.github/workflows/chromatic-storybook-test.yml @@ -14,7 +14,37 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # Required to retrieve git history - - run: yarn install && yarn build-storybook + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - run: yarn build-storybook + - uses: chromaui/action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/frontend.yml b/.github/workflows/ci.yml similarity index 55% rename from .github/workflows/frontend.yml rename to .github/workflows/ci.yml index dc65b987a8..0a094a10b3 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,11 @@ -name: Frontend CI +name: CI on: pull_request: paths-ignore: - 'microsite/**' jobs: - build: + verify: runs-on: ubuntu-latest strategy: @@ -20,47 +20,67 @@ jobs: - uses: actions/checkout@v2 - name: fetch branch master run: git fetch origin master - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + + # Beginning of yarn setup, keep in sync between all workflows. + # TODO(Rugvip): move this to composite action once all features we use are supported - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + + # Cache every node_modules folder inside the monorepo + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + # We use both yarn.lock and package.json as cache keys to ensure that + # changes to local monorepo packages bust the cache. + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + + # If we get a cache hit for node_modules, there's no need to bring in the global + # yarn cache or run yarn install, as all dependencies will be installed already. + + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: yarn install + if: steps.cache-modules.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + # End of yarn setup - name: check for yarn.lock changes id: yarn-lock run: git diff --quiet origin/master HEAD -- yarn.lock continue-on-error: true - - name: yarn install - run: yarn install --frozen-lockfile - - name: verify doc links run: node docs/verify-links.js + - name: prettier + run: yarn prettier:check + - name: lint run: yarn lerna -- run lint --since origin/master - name: type checking and declarations - run: yarn tsc --incremental false + run: yarn tsc:full - name: build changed packages if: ${{ steps.yarn-lock.outcome == 'success' }} - # Need to build all dependencies as well to be able to run tests later - run: yarn lerna -- run build --since origin/master --include-dependencies + run: yarn lerna -- run build --since origin/master - name: build all packages if: ${{ steps.yarn-lock.outcome == 'failure' }} @@ -80,5 +100,15 @@ jobs: - name: verify plugin template run: yarn lerna -- run diff -- --check - - name: verify storybook - run: yarn workspace storybook build-storybook + - name: ensure clean working directory + run: | + if files=$(git ls-files --exclude-standard --others --modified) && [[ -z "$files" ]]; then + exit 0 + else + echo "" + echo "Working directory has been modified:" + echo "" + git status --short + echo "" + exit 1 + fi diff --git a/.github/workflows/cli-win.yml b/.github/workflows/e2e-win.yml similarity index 67% rename from .github/workflows/cli-win.yml rename to .github/workflows/e2e-win.yml index 4e7a186282..b4ea1e350f 100644 --- a/.github/workflows/cli-win.yml +++ b/.github/workflows/e2e-win.yml @@ -1,12 +1,13 @@ -name: CLI Test Windows +name: E2E Test Windows -# Building on windows is really slow, so this workflow is separate from cli.yml and only builds on changes +# Building on windows is really slow, so this workflow is separate from e2e.yml and only builds on changes # to the cli itself. They're more likely to introduce issues on windows, compared to changes to core and yarn.lock. on: pull_request: paths: - - '.github/workflows/cli-win.yml' + - '.github/workflows/e2e-win.yml' - 'packages/cli/**' + - 'packages/e2e/**' - 'packages/create-app/**' jobs: @@ -25,6 +26,13 @@ jobs: name: Node ${{ matrix.node-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth - name: find location of global yarn cache id: yarn-cache run: echo "::set-output name=dir::$(yarn cache dir)" @@ -32,16 +40,15 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: use node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup + - run: yarn tsc - - run: yarn build - - name: verify app and plugin creation - run: node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js + - name: yarn build + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli + - name: run E2E test + run: yarn workspace e2e-test start diff --git a/.github/workflows/cli.yml b/.github/workflows/e2e.yml similarity index 66% rename from .github/workflows/cli.yml rename to .github/workflows/e2e.yml index 158310b5ea..d53ca1b607 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/e2e.yml @@ -1,11 +1,10 @@ -name: CLI Test +name: E2E Test Linux on: pull_request: paths-ignore: - 'microsite/**' - jobs: build: runs-on: ${{ matrix.os }} @@ -34,30 +33,44 @@ jobs: name: Node ${{ matrix.node-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup + - run: yarn tsc - - run: yarn build - - name: verify app and plugin creation + - name: yarn build + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli + - name: run E2E test + run: | + sudo sysctl fs.inotify.max_user_watches=524288 + yarn workspace e2e-test start env: POSTGRES_HOST: localhost POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - run: | - sudo sysctl fs.inotify.max_user_watches=524288 - node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js diff --git a/.github/workflows/master-win.yml b/.github/workflows/master-win.yml new file mode 100644 index 0000000000..3f65cf9aec --- /dev/null +++ b/.github/workflows/master-win.yml @@ -0,0 +1,60 @@ +name: Master Build Windows + +on: + push: + branches: [master] + +jobs: + build: + runs-on: windows-latest + + strategy: + matrix: + node-version: [12.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: find location of global yarn cache + id: yarn-cache + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - name: lint + run: yarn lerna -- run lint + + - name: type checking and declarations + run: yarn tsc:full + + - name: verify type dependencies + run: yarn lint:type-deps + + - name: test + run: yarn lerna -- run test + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + with: + args: 'Windows master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3ef0c1fee3..469ef20cda 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1,4 +1,4 @@ -name: Master Build +name: Main Master Build on: push: @@ -18,35 +18,40 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup - name: lint run: yarn lerna -- run lint - name: type checking and declarations - run: yarn tsc --incremental false + run: yarn tsc:full - name: build run: yarn build @@ -66,6 +71,14 @@ jobs: # Tags the commit with the version in the core package if the tag doesn't exist - uses: Klemensas/action-autotag@1.2.3 with: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - package_root: "packages/core" - tag_prefix: "v" + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + package_root: 'packages/core' + tag_prefix: 'v' + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + with: + args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' diff --git a/.github/workflows/microsite-build-check.yml b/.github/workflows/microsite-build-check.yml index 41f43399ca..ff290b71e4 100644 --- a/.github/workflows/microsite-build-check.yml +++ b/.github/workflows/microsite-build-check.yml @@ -21,32 +21,18 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} - registry-url: https://registry.npmjs.org/ # Needed for auth + # Skip caching of microsite dependencies, it keeps the global cache size + # smaller, which make Windows builds a lot faster for the rest of the project. - name: yarn install run: yarn install --frozen-lockfile + working-directory: microsite - name: build microsite - run: yarn workspace backstage-microsite build + run: yarn build + working-directory: microsite diff --git a/.github/workflows/microsite-with-storybook-deploy.yml b/.github/workflows/microsite-with-storybook-deploy.yml index 84ac6f9490..22fd48789a 100644 --- a/.github/workflows/microsite-with-storybook-deploy.yml +++ b/.github/workflows/microsite-with-storybook-deploy.yml @@ -25,23 +25,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -49,11 +32,18 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - - name: yarn install + # We avoid caching in this workflow, as we're running an install of both the top-level + # dependencies and the microsite. We leave it to the main master workflow to produce the + # cache, as that results in a smaller bundle. + - name: top-level yarn install run: yarn install --frozen-lockfile + - name: microsite yarn install + run: yarn install --frozen-lockfile + working-directory: microsite - name: build microsite - run: yarn workspace backstage-microsite build + run: yarn build + working-directory: microsite - name: build storybook run: yarn workspace storybook build-storybook diff --git a/.gitignore b/.gitignore index 743561c3b6..e7d294d55a 100644 --- a/.gitignore +++ b/.gitignore @@ -89,9 +89,7 @@ typings/ # Nuxt.js build / generate output .nuxt dist - -# Microsite build output -microsite/build +dist-types # Gatsby files .cache/ diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..9e75b74eee --- /dev/null +++ b/.prettierignore @@ -0,0 +1,8 @@ +.yarn +dist +microsite/build +coverage +*.hbs +templates +plugins/scaffolder-backend/sample-templates +.vscode diff --git a/ADOPTERS.md b/ADOPTERS.md index 7a96198529..0ba3e3196b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,13 +1,15 @@ -| Organization | Contact | Description of Use | -| --------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | -| [Spotify](https://www.spotify.com) | [@stefanalund](https://github.com/stefanalund) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@RoyJacobs](https://github.com/RoyJacobs) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| Organization | Contact | Description of Use | +| -------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| [Spotify](https://www.spotify.com) | [@stefanalund](https://github.com/stefanalund) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@RoyJacobs](https://github.com/RoyJacobs) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f03e3fa5..20ae9a414d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,21 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re > Collect changes for the next release below +- The backend plugin + [service builder](https://github.com/spotify/backstage/blob/master/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts) + no longer adds `express.json()` automatically to all routes. While convenient + in a lot of cases, it also led to problems where for example the proxy + middleware could hang because the body had already been altered and could not + be streamed. Also, plugins that rather wanted to handle e.g. form encoded data + still had to cater to that manually. We therefore decided to let plugins add + `express.json()` themselves if they happen to deal with JSON data. + +## v0.1.1-alpha.20 + +- Includes https://github.com/spotify/backstage/pull/2097 to resolve issues with create-plugin command. + +## v0.1.1-alpha.19 + ### @backstage/create-app - Many plugins have been added to the catalog and will for now be required to be added to separate apps as well. This will be solved as [#1536](https://github.com/spotify/backstage/issues/1536) gets sorted out, but for now you may need to install some plugins just to get pages to work. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 69f6457dd9..55269dd2a5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -4,12 +4,12 @@ This code of conduct outlines our expectations for participants within the **Spo Our open source community strives to: -* **Be friendly and patient.** -* **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability. -* **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language. -* **Be respectful**: Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one. -* **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable. -* **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes. +- **Be friendly and patient.** +- **Be welcoming**: We strive to be a community that welcomes and supports people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, colour, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability. +- **Be considerate**: Your work will be used by other people, and you in turn will depend on the work of others. Any decision you take will affect users and colleagues, and you should take those consequences into account when making decisions. Remember that we're a world-wide community, so you might not be communicating in someone else's primary language. +- **Be respectful**: Not all of us will agree all the time, but disagreement is no excuse for poor behavior and poor manners. We might all experience some frustration now and then, but we cannot allow that frustration to turn into a personal attack. It’s important to remember that a community where people feel uncomfortable or threatened is not a productive one. +- **Be careful in the words that we choose**: we are a community of professionals, and we conduct ourselves professionally. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior aren't acceptable. +- **Try to understand why we disagree**: Disagreements, both social and technical, happen all the time. It is important that we resolve disagreements and differing views constructively. Remember that we’re different. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Don’t forget that it is human to err and blaming each other doesn’t get us anywhere. Instead, focus on helping to resolve issues and learning from mistakes. ## Definitions @@ -18,7 +18,7 @@ Harassment includes, but is not limited to: - Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation - Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment - Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle -- Physical contact and simulated physical contact (eg, textual descriptions like “*hug*” or “*backrub*”) without consent or after a request to stop +- Physical contact and simulated physical contact (eg, textual descriptions like “_hug_” or “_backrub_”) without consent or after a request to stop - Threats of violence, both physical and psychological - Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm - Deliberate intimidation @@ -39,7 +39,6 @@ Our open source community prioritizes marginalized people’s safety over privil - Communicating in a ‘tone’ you don’t find congenial - Criticizing racist, sexist, cissexist, or otherwise oppressive behavior or assumptions - ### Diversity Statement We encourage everyone to participate and are committed to building a community for all. Although we will fail at times, we seek to treat everyone both as fairly and equally as possible. Whenever a participant has made a mistake, we expect them to take responsibility for it. If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong. @@ -53,18 +52,18 @@ If you experience or witness unacceptable behavior—or have any other concerns - Your contact information. - Names (real, nicknames, or pseudonyms) of any individuals involved. If there are additional witnesses, please -include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link. + include them as well. Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public IRC logger), please include a link. - Any additional information that may be helpful. After filing a report, a representative will contact you personally, review the incident, follow up with any additional questions, and make a decision as to how to respond. If the person who is harassing you is part of the response team, they will recuse themselves from handling your incident. If the complaint originates from a member of the response team, it will be handled by a different member of the response team. We will respect confidentiality requests for the purpose of protecting victims of abuse. ### Attribution & Acknowledgements -We all stand on the shoulders of giants across many open source communities. We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration: +We all stand on the shoulders of giants across many open source communities. We'd like to thank the communities and projects that established code of conducts and diversity statements as our inspiration: -* [Django](https://www.djangoproject.com/conduct/reporting/) -* [Python](https://www.python.org/community/diversity/) -* [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct) -* [Contributor Covenant](http://contributor-covenant.org/) -* [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/) -* [Citizen Code of Conduct](http://citizencodeofconduct.org/) +- [Django](https://www.djangoproject.com/conduct/reporting/) +- [Python](https://www.python.org/community/diversity/) +- [Ubuntu](http://www.ubuntu.com/about/about-ubuntu/conduct) +- [Contributor Covenant](http://contributor-covenant.org/) +- [Geek Feminism](http://geekfeminism.org/about/code-of-conduct/) +- [Citizen Code of Conduct](http://citizencodeofconduct.org/) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b8febd1d2..853332f74d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,4 @@ ---- -id: CONTRIBUTING -title: Contributing ---- +# Contributing to Backstage Our vision for Backstage is for it to become the trusted standard toolbox (read: UX layer) for the open source infrastructure landscape. Think of it like Kubernetes for developer experience. We realize this is an ambitious goal. We can’t do it alone. @@ -31,7 +28,7 @@ What kind of plugins should/could be created? Some inspiration from the 120+ plu ## Suggesting a plugin -If you start developing a plugin that you aim to release as open source, we suggest that you create a new [new Issue](https://github.com/spotify/backstage/issues/new?template=plugin_template.md). This helps the community know what plugins are in development. +If you start developing a plugin that you aim to release as open source, we suggest that you create a new [new Issue](https://github.com/spotify/backstage/issues/new?labels=plugin&template=plugin_template.md&title=%5BPlugin%5D+THE+PLUGIN+NAME). This helps the community know what plugins are in development. You can also use this process if you have an idea for a good plugin but you hope that someone else will pick up the work. diff --git a/Dockerfile b/Dockerfile index fa89debcee..174548a90c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,14 @@ FROM nginx:mainline +# The purpose of this image is to serve the frontend app content separately. +# By default the Backstage backend uses the app-backend plugin to serve the +# app from the backend itself, but it may be desirable to move the frontend +# content serving to a separate deployment, in which case this image can be used. + # This dockerfile requires the app to be built on the host first, as it # simply copies in the build output into the image. -# The safest way to build this image is to use `yarn docker-build` +# The safest way to build this image is to use `yarn docker-build:app` RUN apt-get update && apt-get -y install jq && rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index ee9e85fd7f..4009864edc 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,6 @@ For more information go to [backstage.io](https://backstage.io) or join our [Dis A detailed project roadmap, including already delivered milestones, is available [here](https://backstage.io/docs/overview/roadmap). -## Overview - -The Backstage platform consists of a number of different components: - -- **app** - Main web application that users interact with. It's built up by a number of different _Plugins_. This repo contains an example implementation of an app (located in `packages/app`) and you can easily get started with your own app by [creating one](docs/getting-started/create-an-app.md). -- [**plugins**](https://github.com/spotify/backstage/tree/master/plugins) - Each plugin is treated as a self-contained web app and can include almost any type of content. Plugins all use a common set of platform API's and reusable UI components. Plugins can fetch data either from the _backend_ or through any RESTful API exposed through the _proxy_. -- [**service catalog**](https://github.com/spotify/backstage/tree/master/packages/backend) - Service that holds the model of your software ecosystem, including organisational information and what team owns what software. The backend also has a Plugin model for extending its graph. -- [**proxy**](https://github.com/spotify/backstage/tree/master/plugins/proxy-backend) - Terminates HTTPS and exposes any RESTful API to Plugins. -- **identity** - A backend service that holds your organisation's metadata. - ## Getting Started There are two different ways to get started with Backstage, either by creating a standalone app, or by cloning this repo. Which method you use depends on what you're planning to do. @@ -71,7 +61,7 @@ Take a look at the [Getting Started](https://backstage.io/docs/getting-started/i ## Documentation -- [Main documentation](https://backstage.io/docs/overview/what-is-backstage) +- [Main documentation](https://backstage.io/docs) - [Service Catalog](https://backstage.io/docs/features/software-catalog/software-catalog-overview) - [Architecture](https://backstage.io/docs/overview/architecture-terminology) ([Decisions](https://backstage.io/docs/architecture-decisions/adrs-overview)) - [Designing for Backstage](https://backstage.io/docs/dls/design) diff --git a/app-config.development.yaml b/app-config.development.yaml new file mode 100644 index 0000000000..da274ba1a8 --- /dev/null +++ b/app-config.development.yaml @@ -0,0 +1,11 @@ +app: + baseUrl: http://localhost:3000 + +backend: + baseUrl: http://localhost:7000 + listen: + port: 7000 + cors: + origin: http://localhost:3000 + methods: [GET, POST, PUT, DELETE] + credentials: true diff --git a/app-config.yaml b/app-config.yaml index 01e2ff33fe..da2f5f2d11 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -1,34 +1,24 @@ app: title: Backstage Example App - baseUrl: http://localhost:3000 + baseUrl: http://localhost:7000 backend: baseUrl: http://localhost:7000 listen: port: 7000 - cors: - origin: http://localhost:3000 - methods: [GET, POST, PUT, DELETE] - credentials: true database: client: sqlite3 connection: ':memory:' +# See README.md in the proxy-backend plugin for information on the configuration format proxy: - '/circleci/api': - target: 'https://circleci.com/api/v1.1' - changeOrigin: true - pathRewrite: - '^/proxy/circleci/api/': '/' + '/circleci/api': https://circleci.com/api/v1.1 '/jenkins/api': - target: 'http://localhost:8080' - changeOrigin: true + target: http://localhost:8080 headers: Authorization: $secret: env: JENKINS_BASIC_AUTH_HEADER - pathRewrite: - '^/proxy/jenkins/api/': '/' organization: name: Spotify @@ -50,6 +40,50 @@ newrelic: baseUrl: 'https://api.newrelic.com/v2' key: NEW_RELIC_REST_API_KEY +lighthouse: + baseUrl: http://localhost:3003 + +catalog: + rules: + - allow: [Component, API, Group, Template, Location] + processors: + githubApi: + privateToken: + $secret: + env: GITHUB_PRIVATE_TOKEN + bitbucketApi: + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + gitlabApi: + privateToken: + $secret: + env: GITLAB_PRIVATE_TOKEN + azureApi: + privateToken: + $secret: + env: AZURE_PRIVATE_TOKEN + exampleEntityLocations: + github: + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/podcast-api-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/queue-proxy-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/searcher-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-lib-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/www-artist-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/shuffle-api-component.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml + - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/hello-world-api.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/streetlights-api.yaml + auth: providers: google: @@ -82,10 +116,9 @@ auth: audience: $secret: env: GITLAB_BASE_URL - # saml: - # development: - # entryPoint: "http://localhost:7001/" - # issuer: "passport-saml" + saml: + entryPoint: 'http://localhost:7001/' + issuer: 'passport-saml' okta: development: clientId: @@ -122,3 +155,14 @@ auth: domain: $secret: env: AUTH_AUTH0_DOMAIN + microsoft: + development: + clientId: + $secret: + env: AUTH_MICROSOFT_CLIENT_ID + clientSecret: + $secret: + env: AUTH_MICROSOFT_CLIENT_SECRET + tenantId: + $secret: + env: AUTH_MICROSOFT_TENANT_ID diff --git a/catalog-info.yaml b/catalog-info.yaml index 86b67f95cd..937a55bfca 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -3,10 +3,10 @@ kind: Component metadata: name: backstage description: | - Backstage is an open-source developer portal that puts the developer experience first. + Backstage is an open-source developer portal that puts the developer experience first. annotations: github.com/project-slug: spotify/backstage - backstage.io/github-actions-id: spotify/backstage + backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git spec: type: library owner: Spotify diff --git a/docker-compose.yaml b/docker-compose.yaml index d32928b4e3..a9922cfb6c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,10 @@ # Make sure that before you # run the docker-compose that you have run -# $ yarn docker-build:all +# $ yarn docker-build version: '3' services: - frontend: - image: 'spotify/backstage:latest' - ports: - - '3000:80' - backend: + backstage: image: 'example-backend:latest' ports: - '7000:7000' diff --git a/docs/FAQ.md b/docs/FAQ.md index d230e682a4..4b5df3ace0 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -3,7 +3,7 @@ id: FAQ title: FAQ --- -## Product FAQ: +## Product FAQ ### Can we call Backstage something different? So that it fits our company better? @@ -67,7 +67,7 @@ valuable as you grow. Yes! The Backstage UI is built using Material-UI. With the theming capabilities of Material-UI, you are able to adapt the interface to your brand guidelines. -## Technical FAQ: +## Technical FAQ ### Why Material-UI? diff --git a/docs/README.md b/docs/README.md index b9e79a9e0a..c63a12b589 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,104 +1,3 @@ -# Documentation structure +# Documentation -**Note!** This documentation structure is very much work in progress. If (when, -really 😆) you find broken links or missing content, please create an issue or, -better yet, a pull request. - -# Plugins - -- Overview - - [What is Backstage?](overview/what-is-backstage.md) - - [Backstage architecture](overview/architecture-overview.md) - - [Architecture and terminology](overview/architecture-terminology.md) - - [Roadmap](overview/roadmap.md) - - [Vision](overview/vision.md) -- Getting started - - [Running Backstage locally](getting-started/index.md) - - [Installation](getting-started/installation.md) - - [Local development](getting-started/development-environment.md) - - [Demo deployment](https://backstage-demo.roadie.io) - - Production deployments - - [Create an App](getting-started/create-an-app.md) - - App configuration - - [Configuring App with plugins](getting-started/configure-app-with-plugins.md) - - [Customize the look-and-feel of your App](getting-started/app-custom-theme.md) - - Deployment scenarios - - [Kubernetes](getting-started/deployment-k8s.md) - - [Other](getting-started/deployment-other.md) -- Features - - Software Catalog - - [Overview](features/software-catalog/index.md) - - [System model](features/software-catalog/system-model.md) - - [YAML File Format](features/software-catalog/descriptor-format.md) - - [Extending the model](features/software-catalog/extending-the-model.md) - - [External integrations](features/software-catalog/external-integrations.md) - - [API](features/software-catalog/api.md) - - Software creation templates - - [Overview](features/software-templates/index.md) - - [Adding templates](features/software-templates/adding-templates.md) - - Extending the Scaffolder: - - [Overview](features/software-templates/extending/index.md) - - [Create your own Templater](features/software-templates/extending/create-your-own-templater.md) - - [Create your own Publisher](features/software-templates/extending/create-your-own-publisher.md) - - [Create your own Preparer](features/software-templates/extending/create-your-own-preparer.md) - - Docs-like-code - - [Overview](features/techdocs/README.md) - - [Getting Started](features/techdocs/getting-started.md) - - [Concepts](features/techdocs/concepts.md) - - [Creating and Publishing Documentation](features/techdocs/creating-and-publishing.md) - - [FAQ](features/techdocs/FAQ.md) -- Plugins - - [Overview](plugins/index.md) - - [Existing plugins](plugins/existing-plugins.md) - - [Creating a new plugin](plugins/create-a-plugin.md) - - [Developing a plugin](plugins/plugin-development.md) - - [Structure of a plugin](plugins/structure-of-a-plugin.md) - - Backends and APIs - - [Proxying](plugins/proxying.md) - - [Backstage backend plugin](plugins/backend-plugin.md) - - [Call existing API](plugins/call-existing-api.md) - - Testing - - [Overview](plugins/testing.md) - - Publishing - - [Open source and NPM](plugins/publishing.md) - - [Private/internal (non-open source)](plugins/publish-private.md) -- Configuration - - [Overview](conf/index.md) - - [Reading Configuration](conf/reading.md) - - [Writing Configuration](conf/writing.md) - - [Defining Configuration](conf/defining.md) -- Authentication and identity - - [Overview](auth/index.md) - - [Add auth provider](auth/add-auth-provider.md) - - [Auth backend](auth/auth-backend.md) - - [OAuth](auth/oauth.md) - - [Glossary](auth/glossary.md) -- Designing for Backstage - - [Backstage Design Language System (DLS)](dls/design.md) - - [Storybook -- reusable UI components](http://backstage.io/storybook) - - [Contributing to Storybook](dls/contributing-to-storybook.md) - - [Figma resources](dls/figma.md) -- API references - - TypeScript API - - [Utility APIs](api/utility-apis.md) - - [Utility API References](reference/utility-apis/README.md) - - [createPlugin](reference/createPlugin.md) - - [createPlugin-feature-flags](reference/createPlugin-feature-flags.md) - - [createPlugin-router](reference/createPlugin-router.md) - - Backend APIs - - [Backend](api/backend.md) -- Tutorials - - [Overview](tutorials/index.md) -- Architecture Decision Records (ADRs) - - [Overview](architecture-decisions/index.md) - - [ADR001 - Architecture Decision Record (ADR) log](architecture-decisions/adr001-add-adr-log.md) - - [ADR002 - Default Software Catalog File Format](architecture-decisions/adr002-default-catalog-file-format.md) - - [ADR003 - Avoid Default Exports and Prefer Named Exports](architecture-decisions/adr003-avoid-default-exports.md) - - [ADR004 - Module Export Structure](architecture-decisions/adr004-module-export-structure.md) - - [ADR005 - Catalog Core Entities](architecture-decisions/adr005-catalog-core-entities.md) - - [ADR006 - Avoid React.FC and React.SFC](architecture-decisions/adr006-avoid-react-fc.md) - - [ADR007 - Use MSW for Mocking Network Requests](architecture-decisions/adr007-use-msw-to-mock-service-requests.md) - - [ADR008 - Default Catalog File Name](architecture-decisions/adr008-default-catalog-file-name.md) -- [Contribute](../CONTRIBUTING.md) -- [Support](overview/support.md) -- [FAQ](FAQ.md) +The Backstage documentation is available at https://backstage.io/docs diff --git a/docs/api/utility-apis.md b/docs/api/utility-apis.md index ff8c91e178..0e09da3f83 100644 --- a/docs/api/utility-apis.md +++ b/docs/api/utility-apis.md @@ -71,18 +71,22 @@ import { AlertApiForwarder, ErrorApiForwarder, ErrorAlerter, + ConfigApi, } from '@backstage/core'; -const builder = ApiRegistry.builder(); +const apis = (config: ConfigApi) => { + const builder = ApiRegistry.builder(); -// The alert API is a self-contained implementation that shows alerts to the user. -const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); + // The alert API is a self-contained implementation that shows alerts to the user. + const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); -// The error API uses the alert API to send error notifications to the user. -builder.add(errorApiRef, new ErrorAlerter(alertApi, new ErrorApiForwarder())); + // The error API uses the alert API to send error notifications to the user. + builder.add(errorApiRef, new ErrorAlerter(alertApi, new ErrorApiForwarder())); + return builder.build(); +}; const app = createApp({ - apis: apiBuilder.build(), + apis, // ... other config }); ``` diff --git a/docs/architecture-decisions/index.md b/docs/architecture-decisions/index.md index b02e8deabb..91d2c668d7 100644 --- a/docs/architecture-decisions/index.md +++ b/docs/architecture-decisions/index.md @@ -4,8 +4,6 @@ title: Architecture Decision Records (ADR) sidebar_label: Overview --- -# - The substantial architecture decisions made in the Backstage project lives here. For more information about ADRs, when to write them, and why, please see [this blog post](https://engineering.atspotify.com/2020/04/14/when-should-i-write-an-architecture-decision-record/). @@ -25,7 +23,10 @@ Records should be stored under the `architecture-decisions` directory. - Submit a pull request - Address and integrate feedback from the community - Eventually, assign a number -- Add the full path of the ADR to the [`mkdocs.yml`](/mkdocs.yml) +- Add the path of the ADR to the microsite sidebar in + [`sidebars.json`](https://github.com/spotify/backstage/blob/master/microsite/sidebars.json) +- Add the path of the ADR to the + [`mkdocs.yml`](https://github.com/spotify/backstage/blob/master/mkdocs.yml) - Merge the pull request ## Superseding an ADR diff --git a/docs/assets/dls/designheader-updated.png b/docs/assets/dls/designheader-updated.png new file mode 100644 index 0000000000..56c5a56abb Binary files /dev/null and b/docs/assets/dls/designheader-updated.png differ diff --git a/docs/getting-started/create-app_output.png b/docs/assets/getting-started/create-app_output.png similarity index 100% rename from docs/getting-started/create-app_output.png rename to docs/assets/getting-started/create-app_output.png diff --git a/docs/getting-started/create-plugin_output.png b/docs/assets/getting-started/create-plugin_output.png similarity index 100% rename from docs/getting-started/create-plugin_output.png rename to docs/assets/getting-started/create-plugin_output.png diff --git a/docs/assets/pop.png b/docs/assets/pop.png new file mode 100644 index 0000000000..441126f33c Binary files /dev/null and b/docs/assets/pop.png differ diff --git a/docs/features/software-catalog/bsc-edit.png b/docs/assets/software-catalog/bsc-edit.png similarity index 100% rename from docs/features/software-catalog/bsc-edit.png rename to docs/assets/software-catalog/bsc-edit.png diff --git a/docs/features/software-catalog/bsc-extend.png b/docs/assets/software-catalog/bsc-extend.png similarity index 100% rename from docs/features/software-catalog/bsc-extend.png rename to docs/assets/software-catalog/bsc-extend.png diff --git a/docs/features/software-catalog/bsc-register-1.png b/docs/assets/software-catalog/bsc-register-1.png similarity index 100% rename from docs/features/software-catalog/bsc-register-1.png rename to docs/assets/software-catalog/bsc-register-1.png diff --git a/docs/features/software-catalog/bsc-register-2.png b/docs/assets/software-catalog/bsc-register-2.png similarity index 100% rename from docs/features/software-catalog/bsc-register-2.png rename to docs/assets/software-catalog/bsc-register-2.png diff --git a/docs/features/software-catalog/bsc-search.png b/docs/assets/software-catalog/bsc-search.png similarity index 100% rename from docs/features/software-catalog/bsc-search.png rename to docs/assets/software-catalog/bsc-search.png diff --git a/docs/features/software-catalog/bsc-starred.png b/docs/assets/software-catalog/bsc-starred.png similarity index 100% rename from docs/features/software-catalog/bsc-starred.png rename to docs/assets/software-catalog/bsc-starred.png diff --git a/docs/assets/software-catalog/software-model-core-entities.png b/docs/assets/software-catalog/software-model-core-entities.png new file mode 100644 index 0000000000..b718b7527c Binary files /dev/null and b/docs/assets/software-catalog/software-model-core-entities.png differ diff --git a/docs/assets/techdocs/documentation-template.png b/docs/assets/techdocs/documentation-template.png new file mode 100644 index 0000000000..1f44ad27c6 Binary files /dev/null and b/docs/assets/techdocs/documentation-template.png differ diff --git a/docs/assets/techdocs/techdocs_big_picture.png b/docs/assets/techdocs/techdocs_big_picture.png new file mode 100644 index 0000000000..ffe437180f Binary files /dev/null and b/docs/assets/techdocs/techdocs_big_picture.png differ diff --git a/docs/auth/add-auth-provider.md b/docs/auth/add-auth-provider.md index a2830dcf4f..7e4f5c8e2f 100644 --- a/docs/auth/add-auth-provider.md +++ b/docs/auth/add-auth-provider.md @@ -48,22 +48,35 @@ provider class which implements a handler for the chosen framework. #### Adding an OAuth based provider If we're adding an `OAuth` based provider we would implement the -[OAuthProviderHandlers](#OAuthProviderHandlers) interface. +[OAuthProviderHandlers](#OAuthProviderHandlers) interface. By implementing this +interface we can use the `OAuthProvider` class provided by `lib/oauth`, meaning +we don't need to implement the full +[AuthProviderRouteHandlers](#AuthProviderRouteHandlers) interface that providers +otherwise need to implement. -The provider class takes the provider's configuration as a class parameter. It -also imports the `Strategy` from the passport package. +The provider class takes the provider's options as a class parameter. It also +imports the `Strategy` from the passport package. ```ts import { Strategy as ProviderAStrategy } from 'passport-provider-a'; +export type ProviderAProviderOptions = OAuthProviderOptions & { + // extra options here +} + export class ProviderAAuthProvider implements OAuthProviderHandlers { - private readonly providerConfig: AuthProviderConfig; private readonly _strategy: ProviderAStrategy; - constructor(providerConfig: AuthProviderConfig) { - this.providerConfig = providerConfig; + constructor(options: ProviderAProviderOptions) { this._strategy = new ProviderAStrategy( - { ...providerConfig.options }, + { + clientID: options.clientId, + clientSecret: options.clientSecret, + callbackURL: options.callbackUrl, + passReqToCallback: false as true, + response_type: 'code', + /// ... etc + } verifyFunction, // See the "Verify Callback" section ); } @@ -82,14 +95,18 @@ An non-`OAuth` based provider could implement [AuthProviderRouteHandlers](#AuthProviderRouteHandlers) instead. ```ts +type ProviderAOptions = { + // ... +}; + export class ProviderAAuthProvider implements AuthProviderRouteHandlers { - private readonly providerConfig: AuthProviderConfig; private readonly _strategy: ProviderAStrategy; - constructor(providerConfig: AuthProviderConfig) { - this.providerConfig = providerConfig; + constructor(options: ProviderAOptions) { this._strategy = new ProviderAStrategy( - { ...providerConfig.options }, + { + // ... + }, verifyFunction, // See the "Verify Callback" section ); } @@ -101,31 +118,61 @@ export class ProviderAAuthProvider implements AuthProviderRouteHandlers { } ``` -#### Create method +#### Factory function -Each provider exports a create method that creates the provider instance, -optionally extending a supported authorization framework. This method exists to -allow for flexibility if additional frameworks are supported in the future. +Each provider exports a factory function that instantiates the provider. The +factory should implement [AuthProviderFactory](#AuthProviderFactory), which +passes in a object with utilities for configuration, logging, token issuing, +etc. The factory should return an implementation of +[AuthProviderRouteHandlers](#AuthProviderRouteHandlers). -Implementing OAuth by returning an instance of `OAuthProvider` based of the -provider's class: +The factory is what decides the mapping from +[static configuration](../conf/index.md) to the creation of auth providers. For +example, OAuth providers use `OAuthEnvironmentHandler` to allow for multiple +different configurations, one for each environment, which looks like this; ```ts -export function createProviderAProvider(config: AuthProviderConfig) { - const provider = new ProviderAAuthProvider(config); - const oauthProvider = new OAuthProvider(provider, config.provider, true); - return oauthProvider; -} +export const createOktaProvider: AuthProviderFactory = ({ + globalConfig, + config, + tokenIssuer, +}) => + OAuthEnvironmentHandler.mapConfig(config, envConfig => { + // read options from config + const clientId = envConfig.getString('clientId'); + const clientSecret = envConfig.getString('clientSecret'); + + // instantiate our OAuthProviderHandlers implementation + const provider = new OktaAuthProvider({ + audience, + clientId, + clientSecret, + callbackUrl, + }); + + // Wrap the OAuthProviderHandlers with OAuthProvider, which implements AuthProviderRouteHandlers + return OAuthProvider.fromConfig(globalConfig, provider, { + disableRefresh: false, + providerId, + tokenIssuer, + }); + }); ``` -Not extending with OAuth, the main difference here is that the create method is -returning a instance of the class without adding the OAuth authorization -framework to it. +The purpose of the different environments is to allow for a single auth-backend +to serve as the authentication service for multiple different frontend +environments, such as local development, staging, and production. + +The factory function for other providers can be a lot simpler, as they might not +have configuration for each environment. Looking something like this: ```ts -export function createProviderAProvider(config: AuthProviderConfig) { - return new ProviderAAuthProvider(config); -} +export const createProviderAProvider: AuthProviderFactory = ({ config }) => { + const a = config.getString('a'); + const b = config.getString('b'); + + return new ProviderAAuthProvider({ a, b }); +}; ``` #### Verify Callback @@ -144,7 +191,7 @@ export function createProviderAProvider(config: AuthProviderConfig) { > http://www.passportjs.org/docs/configure/ **`plugins/auth-backend/src/providers/providerA/index.ts`** is simply -re-exporting the create method to be used for hooking the provider up to the +re-exporting the factory function to be used for hooking the provider up to the backend. ```ts @@ -153,26 +200,14 @@ export { createProviderAProvider } from './provider'; ### Hook it up to the backend -**`plugins/auth-backend/src/providers/config.ts`** The provider needs to be -configured properly so you need to add it to the list of configured providers, -all of which implement [AuthProviderConfig](#AuthProviderConfig): - -```ts -export const providers = [ - { - provider: 'providerA', # used as an identifier - options: { ... }, # consult the provider documentation for which options you should provide - disableRefresh: true # if the provider lacks refresh tokens - }, -``` - **`plugins/auth-backend/src/providers/factories.ts`** When the `auth-backend` starts it sets up routing for all the available providers by calling -`createAuthProviderRouter` on each provider. You need to import the create -method from the provider and add it to the factory: +`createAuthProviderRouter` on each provider. You need to import the factory +function from the provider and add it to the factory: ```ts import { createProviderAProvider } from './providerA'; + const factories: { [providerId: string]: AuthProviderFactory } = { providerA: createProviderAProvider, }; @@ -203,10 +238,21 @@ web browser and you should be able to trigger the authorization flow. ```ts export interface OAuthProviderHandlers { - start(req: express.Request, options: any): Promise; - handler(req: express.Request): Promise; - refresh?(refreshToken: string, scope: string): Promise; - logout?(): Promise; + start( + req: express.Request, + options: Record, + ): Promise; + handler( + req: express.Request, + ): Promise<{ + response: AuthResponse; + refreshToken?: string; + }>; + refresh?( + refreshToken: string, + scope: string, + ): Promise>; + logout?(): Promise; } ``` @@ -221,12 +267,17 @@ export interface AuthProviderRouteHandlers { } ``` -##### AuthProviderConfig +##### AuthProviderFactory ```ts -export type AuthProviderConfig = { - provider: string; - options: any; - disableRefresh?: boolean; +export type AuthProviderFactoryOptions = { + globalConfig: AuthProviderConfig; + config: Config; + logger: Logger; + tokenIssuer: TokenIssuer; }; + +export type AuthProviderFactory = ( + options: AuthProviderFactoryOptions, +) => AuthProviderRouteHandlers; ``` diff --git a/docs/auth/auth-backend-classes.md b/docs/auth/auth-backend-classes.md index 42ee48c5e0..0147c66dd9 100644 --- a/docs/auth/auth-backend-classes.md +++ b/docs/auth/auth-backend-classes.md @@ -1,4 +1,7 @@ -# Authentication Backend Classes Layout and Description +--- +id: auth-backend-classes +title: Auth backend classes +--- ## How Does Authentication Work? @@ -23,7 +26,7 @@ refer to the type documentation under `plugins/auth-backend/src/providers/types.ts`. There are currently two different classes for two authentication mechanisms that -implement this interface: an `OAuthProvider` for [OAuth](https://oauth.net/2/) +implement this interface: an `OAuthAdapter` for [OAuth](https://oauth.net/2/) based mechanisms and a `SAMLAuthProvider` for [SAML](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html) based mechanisms. @@ -47,11 +50,12 @@ OAuth2) providers, you can configure them by setting the right variables in Each authentication provider (except SAML) needs five parameters: an OAuth client ID, a client secret, an authorization endpoint and a token endpoint, and an app origin. The app origin is the URL at which the frontend of the -application is hosted. This is required because the application opens a popup -window to perform the authentication, and once the flow is completed, the popup -window sends a `postMessage` to the frontend application to indicate the result -of the operation. Also this URL is used to verify that authentication requests -are coming from only this endpoint. +application is hosted, and it is read from the `app.baseUrl` config. This is +required because the application opens a popup window to perform the +authentication, and once the flow is completed, the popup window sends a +`postMessage` to the frontend application to indicate the result of the +operation. Also this URL is used to verify that authentication requests are +coming from only this endpoint. These values are configured via the `app-config.yaml` present in the root of your app folder. @@ -61,8 +65,6 @@ auth: providers: google: development: - appOrigin: "http://localhost:3000/" - secure: false clientId: $secret: env: AUTH_GOOGLE_CLIENT_ID @@ -71,8 +73,6 @@ auth: env: AUTH_GOOGLE_CLIENT_SECRET github: development: - appOrigin: "http://localhost:3000/" - secure: false clientId: $secret: env: AUTH_GITHUB_CLIENT_ID @@ -84,8 +84,6 @@ auth: env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL gitlab: development: - appOrigin: "http://localhost:3000/" - secure: false clientId: $secret: ... @@ -93,24 +91,34 @@ auth: ## Technical Notes -### EnvironmentHandler +### OAuthEnvironmentHandler The concept of an "env" is core to the way the auth backend works. It uses an `env` query parameter to identify the environment in which the application is running (`development`, `staging`, `production`, etc). Each runtime can support multiple environments at the same time and the right handler for each request is identified and dispatched to based on the `env` parameter. All -`AuthProviderRouteHandlers` are wrapped within an `EnvironmentHandler`. +`AuthProviderRouteHandlers` are wrapped within an `OAuthEnvironmentHandler`. -An `EnvironmentHandler` takes an ID for each provider that it wraps, the -handlers for each env the provider is supported in, and a function that, given a -`Request` as argument, can extract the information about the env under which it -should be processed. +To instantiate multiple OAuth providers for different environments, use +`OAuthEnvironmentHandler.mapConfig`. It's a helper to iterate over a +configuration object that is a map of environment to configurations. See one of +the existing OAuth providers for an example of how it is used. -Each provider exposes a factory function `createXProvider` (where X is the name -of the provider) that takes the global config, env and other parameters and -returns an `AuthProviderRouteHandlers` for each env, and an `envIdentifier` -function to identify the env of a request. +Given the following configuration: + +```yaml +development: + clientId: abc + clientSecret: secret +production: + clientId: xyz + clientSecret: supersecret +``` + +The `OAuthEnvironmentHandler.mapConfig(config, envConfig => ...)` call will +split the `config` by the top level `development` and `production` keys, and +pass on each block as `envConfig`. For a list of currently available providers, look in the `factories` module located in `plugins/auth-backend/src/providers/factories.ts` diff --git a/docs/auth/oauth.md b/docs/auth/oauth.md index 6e6dc4da80..b7013a310d 100644 --- a/docs/auth/oauth.md +++ b/docs/auth/oauth.md @@ -104,6 +104,8 @@ request an access token. The following diagram visualizes the flow described in the previous section. +![](oauth-popup-flow.svg) + - -![](oauth-popup-flow.svg) diff --git a/docs/dls/design.md b/docs/dls/design.md index c8a94fad4d..b434fc8791 100644 --- a/docs/dls/design.md +++ b/docs/dls/design.md @@ -3,7 +3,7 @@ id: design title: Design --- -![header](../assets/dls/designheader.png) +![header](../assets/dls/designheader-updated.png) Much like Backstage Open Source, this is a _living_ document! We'll keep this updated as we evolve our practices! @@ -116,8 +116,8 @@ components. If you’d like to help build up our design system, you can also add components we’ve designed to the Storybook as well. **[Figma](https://www.figma.com/@backstage)** - we're stoked to be using Figma -Community to share our design assets. You can duplicate our component library -and design your own plugin for Backstage. +Community to share our design assets. You can duplicate our UI Kit and design +your own plugin for Backstage. **[Discord](https://discord.gg/EBHEGzX)** - all design questions should be directed to the _#design_ channel. diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md new file mode 100644 index 0000000000..be6314e494 --- /dev/null +++ b/docs/features/software-catalog/configuration.md @@ -0,0 +1,58 @@ +--- +id: software-catalog-configuration +title: Catalog Configuration +--- + +## Static Location Configuration + +To enable declarative catalog setups, it is possible to add locations to the +catalog via [static configuration](../../conf/index.md). Locations are added to +the catalog under the `catalog.locations` key, for example: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +The locations added through static configuration can not be removed through the +catalog locations API. To remove the locations, you have to remove them from the +configuration. + +## Catalog Rules + +By default the catalog will only allow ingestion of entities with the kind +`Component`, `API` and `Location`. In order to allow entities of other kinds to +be added, you need to add rules to the catalog. Rules are added either in a +separate `catalog.rules` key, or added to statically configured locations. + +For example, given the following configuration: + +```yaml +catalog: + rules: + - allow: [Component, API, Location, Template] + + locations: + - type: github + target: https://github.com/org/example/blob/master/org-data.yaml + rules: + - allow: [Group] +``` + +We are able to add entities of kind `Component`, `API`, `Location`, or +`Template` from any location, and `Group` entities from the `org-data.yaml`, +which will also be read as statically configured location. + +Note that if the `catalog.rules` key is present it will replace the default +value, meaning that you need to add rules for the default kinds if you want +those to still be allowed. + +The following configuration will reject any kind of entities from being added to +the catalog: + +```yaml +catalog: + rules: [] +``` diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index a7d57890dc..e90792c628 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -235,6 +235,9 @@ The `backstage.io/` prefix is reserved for use by Backstage core components. Values can be of any length, but are limited to being strings. +There is a list of [well-known annotations](well-known-annotations.md), but +anybody is free to add more annotations as they see fit. + ### `tags` [optional] A list of single-valued strings, for example to classify catalog entities in @@ -290,7 +293,7 @@ Exactly equal to `backstage.io/v1alpha1` and `Component`, respectively. The type of component as a string, e.g. `website`. This field is required. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface @@ -304,9 +307,9 @@ The current set of well-known and common values for this field is: ### `spec.lifecycle` [required] -The lifecyle state of the component, e.g. `production`. This field is required. +The lifecycle state of the component, e.g. `production`. This field is required. -The software catalog accepts any lifecycle value, but an organisation should +The software catalog accepts any lifecycle value, but an organization should take great care to establish a proper taxonomy for these. The current set of well-known and common values for this field is: @@ -370,8 +373,8 @@ metadata: description: Next.js application skeleton for creating isomorphic web applications. tags: - - Recommended - - React + - recommended + - react spec: owner: web@example.com templater: cookiecutter @@ -418,7 +421,7 @@ potentially search and group templates by these tags. The type of component as a string, e.g. `website`. This field is optional but recommended. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface @@ -508,7 +511,7 @@ Exactly equal to `backstage.io/v1alpha1` and `API`, respectively. The type of the API definition as a string, e.g. `openapi`. This field is required. -The software catalog accepts any type value, but an organisation should take +The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, an OpenAPI type API may be displayed using an OpenAPI viewer tooling in diff --git a/docs/features/software-catalog/extending-the-model.md b/docs/features/software-catalog/extending-the-model.md index ce5ea03b26..8b621f2e44 100644 --- a/docs/features/software-catalog/extending-the-model.md +++ b/docs/features/software-catalog/extending-the-model.md @@ -12,7 +12,7 @@ Backstage natively supports tracking of the following component - Documentation - Other -![](bsc-extend.png) +![](../../assets/software-catalog/bsc-extend.png) Since these types are likely not the only kind of software you will want to track in Backstage, it is possible to @@ -31,9 +31,10 @@ catalog. It might be tempting to put software that doesn't fit into any of the existing types into Other. There are a few reasons why we advice against this; firstly, we have found that it is preferred to match the conceptual model that your -engineers have when describing your sofware. Secondly, Backstage helps your -engineers manage their software by integrating the infratrucure tooling through -plugins. Different plugins are used for managing different types of components. +engineers have when describing your software. Secondly, Backstage helps your +engineers manage their software by integrating the infrastructure tooling +through plugins. Different plugins are used for managing different types of +components. For example, the [Lighthouse plugin](https://github.com/spotify/backstage/tree/master/plugins/lighthouse) diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index fe107d1f59..9b5de16472 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -1,6 +1,7 @@ --- id: software-catalog-overview title: Backstage Service Catalog (alpha) +sidebar_label: Overview --- ## What is a Service Catalog? @@ -47,25 +48,25 @@ There are 3 ways to add components to the catalog: 1. Manually register components 2. Creating new components through Backstage -3. Integrating with and [external source](external-integrations.md) +3. Integrating with an [external source](external-integrations.md) ### Manually register components Users can register new components by going to `/create` and clicking the **REGISTER EXISTING COMPONENT** button: -![](bsc-register-1.png) +![](../../assets/software-catalog/bsc-register-1.png) Backstage expects the full URL to the YAML in your source control. Example: -``` +```bash https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` _More examples can be found [here](https://github.com/spotify/backstage/tree/master/packages/catalog-model/examples)._ -![](bsc-register-2.png) +![](../../assets/software-catalog/bsc-register-2.png) It is important to note that any kind of software can be registered in Backstage. Even if the software is not maintained by your company (SaaS @@ -78,12 +79,28 @@ All software created through the [Backstage Software Templates](../software-templates/index.md) are automatically registered in the catalog. +### Static catalog configuration + +In addition to manually registering components, it is also possible to register +components though [static configuration](../../conf/index.md). For example, the +above example can be added using the following configuration: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +More information about catalog configuration can be found +[here](configuration.md). + ### Updating component metadata Teams owning the components are responsible for maintaining the metadata about them, and do so using their normal Git workflow. -![](bsc-edit.png) +![](../../assets/software-catalog/bsc-edit.png) Once the change has been merged, Backstage will automatically show the updated metadata in the service catalog after a short while. @@ -92,25 +109,25 @@ metadata in the service catalog after a short while. By default the service catalog shows components owned by the team of the logged in user. But you can also switch to _All_ to see all the components across your -companie's software ecosystem. Basic inline _search_ and _column filtering_ -makes it easy to browse a big set of components. +company's software ecosystem. Basic inline _search_ and _column filtering_ makes +it easy to browse a big set of components. -![](bsc-search.png) +![](../../assets/software-catalog/bsc-search.png) ## Starring components For easy and quick access to components you visit frequently, Backstage supports _starring_ of components: -![](bsc-starred.png) +![](../../assets/software-catalog/bsc-starred.png) ## Integrated tooling through plugins -The service catalog is a great way to organise the infrastructure tools you use +The service catalog is a great way to organize the infrastructure tools you use to manage the software. This is how Backstage creates one developer portal for all your tools. Rather than asking teams to jump between different -infrastructure UI’s (and incurring additional cognitive overhead each time they -make a context switch), most of these tools can be organised around the entities +infrastructure UIs (and incurring additional cognitive overhead each time they +make a context switch), most of these tools can be organized around the entities in the catalog. ![tools](https://backstage.io/blog/assets/20-05-20/tabs.png) diff --git a/docs/features/software-catalog/system-model.md b/docs/features/software-catalog/system-model.md index 4b2884e097..4436f60fc2 100644 --- a/docs/features/software-catalog/system-model.md +++ b/docs/features/software-catalog/system-model.md @@ -3,55 +3,26 @@ id: system-model title: System Model --- -We believe that a strong shared understanding and terminology around systems, -software and resources leads to a better Backstage experience. +We believe that a strong shared understanding and terminology around software +and resources leads to a better Backstage experience. _This description originates from [this RFC](https://github.com/spotify/backstage/issues/390). Note that some of the concepts are not yet supported in Backstage._ -## Concepts +## Core Entities -We model our technology using these five concepts (further explained below): +We model software in the Backstage catalogue using these three core entities +(further explained below): + +- **Components** are individual pieces of software + +- **APIs** are the boundaries between different components -- **Domains** are a high-level grouping of systems -- **Systems** encapsulate the implementation of APIs -- **APIs** are the boundaries between different components and systems -- **Components** are pieces of software - **Resources** are physical or virtual infrastructure needed to operate a - system + component -![Software Ecosystem Model_ Public Github version](https://user-images.githubusercontent.com/24575/77633084-39bcde80-6f4f-11ea-8251-f8df561a3652.png) - -### Domain - -While systems are the basic level of encapsulation for resources, components and -APIs, it is often useful to group a collection of systems that share -terminology, domain models, business purpose, or documentation, i.e. they form a -bounded context. - -For example, it would make sense if the different systems in the “Payments” -domain would come with some documentation on how to accept payments for a new -product or use-case, share the same entity types in their APIs, and integrate -well with each other. - -### System - -With increasing complexity in software, we believe that systems form an -important abstraction level to help us reason about software ecosystems. Systems -are a useful concept in that they allow us to ignore the implementation details -of a certain functionality for consumers, while allowing the owning team to make -changes as they see fit (leading to low coupling). - -A system, in this sense, is a collection of resources and components that -exposes one or several APIs. Components and resources in a system are typically -owned by the same team and are expected to co-evolve. As such, systems usually -consist of at most a handful of components. - -For example, a playlist management system might encapsulate a backend service to -update playlists, a backend service to query them, and a database to store them. -It could expose an RPC API, a daily snapshots dataset, and an event stream of -playlist updates. +![](../../assets/software-catalog/software-model-core-entities.png) ### Component @@ -60,34 +31,80 @@ backend service or data pipeline (list not exhaustive). A component can be tracked in source control, or use some existing open source or commercial software. -A component can implement APIs for other components to consume. It might depend -on the resources of the system it belongs to, and APIs from other components or -other systems. All other aspects of the component, e.g. any code dependencies, -must be encapsulated. +A component can implement APIs for other components to consume. In turn it might +depend on APIs implemented by other components, or resources that are attached +to it at runtime. ### API -We believe APIs form an important (maybe the most important) abstraction that -allows large software ecosystems to scale. Thus, APIs are a first class citizen -in the Backstage model and the primary way to discover existing functionality in -the ecosystem. +APIs form an important (maybe the most important) abstraction that allows large +software ecosystems to scale. Thus, APIs are a first class citizen in the +Backstage model and the primary way to discover existing functionality in the +ecosystem. -APIs are implemented by components and form boundaries between components and -systems. They might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a -data schema (eg Avro, TFRecord, ...), or as code interfaces. In any case, APIs -exposed by components need to be in a known machine-readable format so we can -build further tooling and analysis on top. +APIs are implemented by components and form boundaries between components. They +might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a data schema (eg +Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by +components need to be in a known machine-readable format so we can build further +tooling and analysis on top. -Some APIs might be exposed by the system, making them available for any other -Spotify component to consume. Those public APIs must be documented and humanly -discoverable in Backstage. +APIs have a visibility: they are either public (making them available for any +other component to consume), restricted (only available to a whitelisted set of +consumers), or private (only available within their system). As public APIs are +going to be the primary way interaction between components, Backstage supports +documenting, indexing and searching all APIs so we can browse them as +developers. ### Resource -Resources are the infrastructure a system needs to operate, like BigTable -databases, Pub/Sub topics, S3 buckets or CDNs. Modelling them together with -components and systems will better allow us to visualize resource footprint, and -create tooling around them. +Resources are the infrastructure a component needs to operate at runtime, like +BigTable databases, Pub/Sub topics, S3 buckets or CDNs. Modelling them together +with components and systems will better allow us to visualize resource +footprint, and create tooling around them. + +## Ecosystem Modeling + +A large catalogue of components, APIs and resources can be highly granular and +hard to understand as a whole. It might thus be convenient to further categorize +these entities using the following (optional) concepts: + +- **Systems** are a collection of entities that cooperate to perform some + function +- **Domains** relate entities and systems to part of the business + +### System + +With increasing complexity in software, systems form an important abstraction +level to help us reason about software ecosystems. Systems are a useful concept +in that they allow us to ignore the implementation details of a certain +functionality for consumers, while allowing the owning team to make changes as +they see fit (leading to low coupling). + +A system, in this sense, is a collection of resources and components that +exposes one or several public APIs. The main benefit of modelling a system is +that it hides its resources and private APIs between the components for any +consumers. This means that as the owner, you can evolve the implementation, in +terms of components and resources, without your consumers being able to notice. +Typically, a system will consist of at most a handful of components (see Domain +for a grouping of systems). + +For example, a playlist management system might encapsulate a backend service to +update playlists, a backend service to query them, and a database to store them. +It could expose an RPC API, a daily snapshots dataset, and an event stream of +playlist updates. + +### Domain + +While systems are the basic level of encapsulation for related entities, it is +often useful to group a collection of systems that share terminology, domain +models, metrics, KPIs, business purpose, or documentation, i.e. they form a +bounded context. + +For example, it would make sense if the different systems in the “Payments” +domain would come with some documentation on how to accept payments for a new +product or use-case, share the same entity types in their APIs, and integrate +well with each other. Other domains could be “Content Ingestion”, “Ads” or +“Search”. ## Current status diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md new file mode 100644 index 0000000000..18de63d704 --- /dev/null +++ b/docs/features/software-catalog/well-known-annotations.md @@ -0,0 +1,134 @@ +--- +id: well-known-annotations +title: Well-known Annotations on Catalog Entities +sidebar_label: Well-known Annotations +--- + +This section lists a number of well known +[annotations](descriptor-format.md#annotations-optional), that have defined +semantics. They can be attached to catalog entities and consumed by plugins as +needed. + +## Annotations + +This is a (non-exhaustive) list of annotations that are known to be in active +use. + +### backstage.io/managed-by-location + +```yaml +# Example: +metadata: + annotations: + backstage.io/managed-by-location: github:http://github.com/spotify/backstage/catalog-info.yaml +``` + +The value of this annotation is a so called location reference string, that +points to the source from which the entity was originally fetched. This +annotation is added automatically by the catalog as it fetches the data from a +registered location, and is not meant to normally be written by humans. The +annotation may point to any type of generic location that the catalog supports, +so it cannot be relied on to always be specifically of type `github`, nor that +it even represents a single file. Note also that a single location can be the +source of many entities, so it represents a many-to-one relationship. + +The format of the value is `:`. Note that the target may also +contain colons, so it is not advisable to naively split the value on `:` and +expecting a two-item array out of it. The format of the target part is +type-dependent and could conceivably even be an empty string, but the separator +colon is always present. + +### backstage.io/techdocs-ref + +```yaml +# Example: +metadata: + annotations: + backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git +``` + +The value of this annotation is a location reference string (see above). If this +annotation is specified, it is expected to point to a repository that the +TechDocs system can read and generate docs from. + +### backstage.io/jenkins-github-folder + +```yaml +# Example: +metadata: + annotations: + backstage.io/jenkins-github-folder: folder-name/job-name +``` + +The value of this annotation is the path to a job on Jenkins, that builds this +entity. + +Specifying this annotation may enable Jenkins related features in Backstage for +that entity. + +### github.com/project-slug + +```yaml +# Example: +metadata: + annotations: + github.com/project-slug: spotify/backstage +``` + +The value of this annotation is the so-called slug that identifies a project on +[GitHub](https://github.com) that is related to this entity. It is on the format +`/`, and is the same as can be seen in the URL location +bar of the browser when viewing that project. + +Specifying this annotation will enable GitHub related features in Backstage for +that entity. + +### sentry.io/project-slug + +```yaml +# Example: +metadata: + annotations: + sentry.io/project-slug: pump-station +``` + +The value of this annotation is the so-called slug (or alternatively, the ID) of +a [Sentry](https://sentry.io) project within your organization. The organization +slug is currently not configurable on a per-entity basis, but is assumed to be +the same for all entities in the catalog. + +Specifying this annotation may enable Sentry related features in Backstage for +that entity. + +### rollbar.com/project-slug + +```yaml +# Example: +metadata: + annotations: + rollbar.com/project-slug: spotify/pump-station +``` + +The value of this annotation is the so-called slug (or alternatively, the ID) of +a [Rollbar](https://rollbar.com) project within your organization. The value can +be the format of `[organization]/[project-slug]` or just `[project-slug]`. When +the organization slug is omitted the `app-config.yaml` will be used as a +fallback (`rollbar.organization` followed by `organization.name`). + +Specifying this annotation may enable Rollbar related features in Backstage for +that entity. + +## Deprecated Annotations + +The following annotations are deprecated, and only listed here to aid in +migrating away from them. + +### backstage.io/github-actions-id + +This annotation was used for a while to enable the GitHub Actions feature. This +is now instead using the [github.com/project-slug](#github-com-project-slug) +annotation, with the same value format. + +## Links + +- [Descriptor Format: annotations](descriptor-format.md#annotations-optional) diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 2ec4bdd6f3..4f7af7c015 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -22,8 +22,8 @@ metadata: Next.js application skeleton for creating isomorphic web applications. # some tags to display in the frontend tags: - - Recommended - - React + - recommended + - react spec: # which templater key to use in the templaters builder templater: cookiecutter @@ -54,11 +54,34 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. -Currently the catalog supports loading definitions from Github + Local Files. To +Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. -For loading from a file the following command should work when the backend is +You can add the template files to the catalog through +[static location configuration](../software-catalog/configuration.md#static-location-configuration), +for example + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + rules: + - allow: [Template] +``` + +Templates can also be added by posting the to the catalog directly. Note that if +you're doing this, you need to configure the catalog to allow template entities +to be ingested from any source, for example: + +```yaml +catalog: + rules: + - allow: [Component, API, Template] +``` + +For loading from a file, the following command should work when the backend is running: ```sh @@ -69,7 +92,7 @@ curl \ --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" ``` -If loading from a git location, you can run the following +If loading from a Git location, you can run the following ```sh curl \ @@ -83,7 +106,7 @@ This should then have added the catalog, and also should now be listed under the create page at http://localhost:3000/create. Alternatively, if you want to get setup with some mock templates that are -already provided for you, you can run the following to load those templates: +already provided, run the following to load those templates: ``` yarn lerna run mock-data diff --git a/docs/features/software-templates/extending/create-your-own-preparer.md b/docs/features/software-templates/extending/create-your-own-preparer.md index 739c516658..0b3fb31323 100644 --- a/docs/features/software-templates/extending/create-your-own-preparer.md +++ b/docs/features/software-templates/extending/create-your-own-preparer.md @@ -54,7 +54,7 @@ The `protocol` is set on the when added to the service catalog. You can see more about this `PreparerKey` here in [Register your own template](../adding-templates.md) -**note:** Currently the catalog supports loading definitions from Github + Local +**note:** Currently the catalog supports loading definitions from GitHub + Local Files, which translate into the two `PreparerKeys` `file` and `github`. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index 51869dc3ad..111125b49f 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -7,18 +7,18 @@ Publishers are responsible for pushing and storing the templated skeleton after the values have been templated by the `Templater`. See [Create your own templater](./create-your-own-templater.md) for more info. -They recieve a directory or location where the templater has sucessfully run on, -and is now ready to store somewhere. They also get given some other options -which are sent from the frontend, such as the `storePath` which is a string of -where the frontend thinks we should save this templated folder. +They receive a directory or location where the templater has sucessfully run and +is now ready to store somewhere. They also are given some other options which +are sent from the frontend, such as the `storePath` which is a string of where +the frontend thinks we should save this templated folder. Currently we provide the following `publishers`: - `github` This publisher is passed through to the `createRouter` function of the -`@spotify/plugin-scaffolder-backend`. Currently only one publisher is supported, -but PR's are always welcome. +`@spotify/plugin-scaffolder-backend`. Currently, only one publisher is +supported, but PR's are always welcome. An full example backend can be found [here](https://github.com/spotify/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md index ac05eed3b0..9dee66af06 100644 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ b/docs/features/software-templates/extending/create-your-own-templater.md @@ -8,14 +8,14 @@ returned by the preparers, and then executing the templating command on top of the file and returning the completed template path. This may or may not be the same directory as the input directory. -They also recieve additional values from the frontend, which can be used to +They also receive additional values from the frontend, which can be used to interpolate into the skeleton files. Currently we provide the following templaters: - `cookiecutter` -This templater is added the `TemplaterBuilder` and then passed into the +This templater is added to the `TemplaterBuilder` and then passed into the `createRouter` function of the `@spotify/plugin-scaffolder-backend` An full example backend can be found @@ -48,7 +48,7 @@ This `TemplaterKey` is used to select the correct templater from the `spec.templater` in the [Template Entity](../../software-catalog/descriptor-format.md#kind-template). -If you wish to add a new templater you'll need to register it with the +If you wish to add a new templater, you'll need to register it with the `TemplaterBuilder`. ### Creating your own Templater to add to the `TemplaterBuilder` @@ -83,10 +83,10 @@ follows: - `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to be able to run docker containers. -_note_ currently the templaters that we provide are basically docker action +_note_ Currently the templaters that we provide are basically Docker action containers that are run on top of the skeleton folder. This keeps dependencies to a minimal for running backstage scaffolder, but you don't /have/ to use -docker. You could create your own templater that spins up an EC2 instance and +Docker. You could create your own templater that spins up an EC2 instance and downloads the folder and does everything using an AMI if you want. It's entirely up to you! @@ -116,8 +116,8 @@ metadata: description: Next.js application skeleton for creating isomorphic web applications. tags: - - Recommended - - React + - recommended + - react spec: owner: web@example.com templater: handlebars @@ -138,7 +138,7 @@ spec: description: Description of the component ``` -You see that the `spec.templater` is set as `handlebars`, you'll need to +You see that the `spec.templater` is set as `handlebars`, so you'll need to register this with the `TemplaterBuilder` like so: ```ts diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md index 21d8befda1..02d27a5725 100644 --- a/docs/features/software-templates/extending/index.md +++ b/docs/features/software-templates/extending/index.md @@ -5,11 +5,11 @@ title: Extending the Scaffolder Welcome. Take a seat. You're at the Scaffolder Documentation. -So - You wanna create stuff inside your company from some prebaked templates? +So, you want to create stuff inside your company from some prebaked templates? You're at the right place. -This guide is gonna take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through whats going on in the backend to be +This guide is going to take you through how the Scaffolder in Backstage works. +We'll dive into some jargon and run through what's going on in the backend to be able to create these templates. There's also more guides that you might find useful at the bottom of this document. At it's core, theres 3 simple stages. @@ -25,9 +25,9 @@ scaffolder that you will need to know: 3. Publish Each of these steps can be configured for your own use case, but we provide some -sensible defaults too. +sensible defaults, too. -Lets dive a little deeper into these phases. +Let's dive a little deeper into these phases. ### Glossary and Jargon @@ -38,8 +38,8 @@ the router to pick the correct `Preparer` to run for the `Template` entity. **Templater** - The templater is responsible for actually running the chosen templater on top of the previously returned temporary directory from the -**Preprarer**. We advise making these docker containers as it can keep all -dependencies, for example Cookiecutter, self contained and not a dependency on +**Preprarer**. We advise making these Docker containers as it can keep all +dependencies--for example Cookiecutter--self contained and not a dependency on the host machine. **Publisher** - The publisher is responsible for taking the finished directory, @@ -50,11 +50,11 @@ passed through to the scaffolder backend. ### How it works -The main of the heavy lifting is done in the +Most of the heavy lifting is done in the [router.ts](https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) file in the `scaffolder-backend` plugin. -There are 2 routes defined in the router. `POST /v1/jobs` and +There are two routes defined in the router: `POST /v1/jobs` and `GET /v1/job/:jobId` To create a scaffolding job, a JSON object containing the @@ -78,7 +78,7 @@ additional templating values must be posted as the post body. The values should represent something that is valid with the `schema` part of the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) -Once that has been posted, a job will be setup with different stages. And the +Once that has been posted, a job will be setup with different stages, and the job processor will complete each stage before moving onto the next stage, whilst collecting logs and mutating the running job. diff --git a/docs/features/software-templates/index.md b/docs/features/software-templates/index.md index d9a86ee7aa..8989fa35eb 100644 --- a/docs/features/software-templates/index.md +++ b/docs/features/software-templates/index.md @@ -1,11 +1,12 @@ --- id: software-templates-index -title: Software Templates +title: Backstage Software Templates +sidebar_label: Overview --- The Software Templates part of Backstage is a tool that can help you create -Components inside Backstage. It by default has the ability to load skeletons of -code, template in some variables and then publish the template to some location +Components inside Backstage. By default, it has the ability to load skeletons of +code, template in some variables, and then publish the template to some location like GitHub.