diff --git a/.github/ISSUE_TEMPLATE/feature_template.md b/.github/ISSUE_TEMPLATE/feature_template.md index 012b8d7a06..d70622bf52 100644 --- a/.github/ISSUE_TEMPLATE/feature_template.md +++ b/.github/ISSUE_TEMPLATE/feature_template.md @@ -1,7 +1,7 @@ --- name: 'Feature Request' about: 'Suggest new features and changes' -labels: help wanted +labels: enhancement --- diff --git a/.github/workflows/chromatic-storybook-test.yml b/.github/workflows/chromatic-storybook-test.yml index e38b494a73..0bcaf400d0 100644 --- a/.github/workflows/chromatic-storybook-test.yml +++ b/.github/workflows/chromatic-storybook-test.yml @@ -14,7 +14,37 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # Required to retrieve git history - - run: yarn install && yarn build-storybook + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - run: yarn build-storybook + - uses: chromaui/action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/frontend.yml b/.github/workflows/ci.yml similarity index 63% rename from .github/workflows/frontend.yml rename to .github/workflows/ci.yml index dc65b987a8..492b008431 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,11 @@ -name: Frontend CI +name: CI on: pull_request: paths-ignore: - 'microsite/**' jobs: - build: + verify: runs-on: ubuntu-latest strategy: @@ -20,34 +20,52 @@ jobs: - uses: actions/checkout@v2 - name: fetch branch master run: git fetch origin master - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + + # Beginning of yarn setup, keep in sync between all workflows. + # TODO(Rugvip): move this to composite action once all features we use are supported - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + + # Cache every node_modules folder inside the monorepo + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + # We use both yarn.lock and package.json as cache keys to ensure that + # changes to local monorepo packages bust the cache. + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + + # If we get a cache hit for node_modules, there's no need to bring in the global + # yarn cache or run yarn install, as all dependencies will be installed already. + + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: yarn install + if: steps.cache-modules.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + # End of yarn setup - name: check for yarn.lock changes id: yarn-lock run: git diff --quiet origin/master HEAD -- yarn.lock continue-on-error: true - - name: yarn install - run: yarn install --frozen-lockfile - - name: verify doc links run: node docs/verify-links.js @@ -59,8 +77,7 @@ jobs: - name: build changed packages if: ${{ steps.yarn-lock.outcome == 'success' }} - # Need to build all dependencies as well to be able to run tests later - run: yarn lerna -- run build --since origin/master --include-dependencies + run: yarn lerna -- run build --since origin/master - name: build all packages if: ${{ steps.yarn-lock.outcome == 'failure' }} @@ -79,6 +96,3 @@ jobs: - name: verify plugin template run: yarn lerna -- run diff -- --check - - - name: verify storybook - run: yarn workspace storybook build-storybook diff --git a/.github/workflows/cli-win.yml b/.github/workflows/e2e-win.yml similarity index 66% rename from .github/workflows/cli-win.yml rename to .github/workflows/e2e-win.yml index 4e7a186282..5ebf67bc9a 100644 --- a/.github/workflows/cli-win.yml +++ b/.github/workflows/e2e-win.yml @@ -1,12 +1,13 @@ -name: CLI Test Windows +name: E2E Test Windows -# Building on windows is really slow, so this workflow is separate from cli.yml and only builds on changes +# Building on windows is really slow, so this workflow is separate from e2e.yml and only builds on changes # to the cli itself. They're more likely to introduce issues on windows, compared to changes to core and yarn.lock. on: pull_request: paths: - - '.github/workflows/cli-win.yml' + - '.github/workflows/e2e-win.yml' - 'packages/cli/**' + - 'packages/e2e/**' - 'packages/create-app/**' jobs: @@ -25,6 +26,13 @@ jobs: name: Node ${{ matrix.node-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth - name: find location of global yarn cache id: yarn-cache run: echo "::set-output name=dir::$(yarn cache dir)" @@ -32,16 +40,15 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: use node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup + - run: yarn tsc - - run: yarn build - - name: verify app and plugin creation - run: node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js + - name: yarn build + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli --ignore backstage-microsite + - name: run E2E test + run: yarn workspace e2e-test start diff --git a/.github/workflows/cli.yml b/.github/workflows/e2e.yml similarity index 65% rename from .github/workflows/cli.yml rename to .github/workflows/e2e.yml index 158310b5ea..401fe7297a 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/e2e.yml @@ -1,11 +1,10 @@ -name: CLI Test +name: E2E Test Linux on: pull_request: paths-ignore: - 'microsite/**' - jobs: build: runs-on: ${{ matrix.os }} @@ -34,30 +33,44 @@ jobs: name: Node ${{ matrix.node-version }} on ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup + - run: yarn tsc - - run: yarn build - - name: verify app and plugin creation + - name: yarn build + run: yarn build --ignore example-app --ignore example-backend --ignore @techdocs/cli --ignore backstage-microsite + - name: run E2E test + run: | + sudo sysctl fs.inotify.max_user_watches=524288 + yarn workspace e2e-test start env: POSTGRES_HOST: localhost POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - run: | - sudo sysctl fs.inotify.max_user_watches=524288 - node ${{ github.workspace }}/packages/cli/e2e-test/cli-e2e-test.js diff --git a/.github/workflows/master-win.yml b/.github/workflows/master-win.yml new file mode 100644 index 0000000000..296a7511bc --- /dev/null +++ b/.github/workflows/master-win.yml @@ -0,0 +1,45 @@ +name: Master Build Windows + +on: + push: + branches: [master] + +jobs: + build: + runs-on: windows-latest + + strategy: + matrix: + node-version: [12.x] + + env: + CI: true + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: find location of global yarn cache + id: yarn-cache + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + +# Tests are broken on Windows, disabled for now + # - name: test + # run: yarn lerna -- run test diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3ef0c1fee3..354838037e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1,4 +1,4 @@ -name: Master Build +name: Main Master Build on: push: @@ -18,29 +18,34 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup - name: lint run: yarn lerna -- run lint @@ -69,3 +74,11 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" package_root: "packages/core" tag_prefix: "v" + + - name: Discord notification + if: ${{ failure() }} + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} + with: + args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' diff --git a/.github/workflows/microsite-build-check.yml b/.github/workflows/microsite-build-check.yml index 41f43399ca..dc86ca2630 100644 --- a/.github/workflows/microsite-build-check.yml +++ b/.github/workflows/microsite-build-check.yml @@ -21,32 +21,34 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup - name: build microsite run: yarn workspace backstage-microsite build diff --git a/.github/workflows/microsite-with-storybook-deploy.yml b/.github/workflows/microsite-with-storybook-deploy.yml index 84ac6f9490..2c8b8af40c 100644 --- a/.github/workflows/microsite-with-storybook-deploy.yml +++ b/.github/workflows/microsite-with-storybook-deploy.yml @@ -25,32 +25,34 @@ jobs: steps: - uses: actions/checkout@v2 - - name: find location of global yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - - name: cache global yarn cache - uses: actions/cache@v2 - with: - path: ${{ steps.yarn-cache.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: cache node_modules - uses: actions/cache@v2 - with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - name: yarn install run: yarn install --frozen-lockfile + # End of yarn setup - name: build microsite run: yarn workspace backstage-microsite build diff --git a/ADOPTERS.md b/ADOPTERS.md index 7a96198529..dc02ad3b85 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,13 +1,14 @@ -| Organization | Contact | Description of Use | -| --------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | -| [Spotify](https://www.spotify.com) | [@stefanalund](https://github.com/stefanalund) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@RoyJacobs](https://github.com/RoyJacobs) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| Organization | Contact | Description of Use | +| -------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| [Spotify](https://www.spotify.com) | [@stefanalund](https://github.com/stefanalund) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@RoyJacobs](https://github.com/RoyJacobs) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | diff --git a/CHANGELOG.md b/CHANGELOG.md index d295dc38e8..ce39a9f8d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re > Collect changes for the next release below +## v0.1.1-alpha.20 + +- Includes https://github.com/spotify/backstage/pull/2097 to resolve issues with create-plugin command. + ## v0.1.1-alpha.19 ### @backstage/create-app diff --git a/README.md b/README.md index ee9e85fd7f..b5e8704548 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,6 @@ For more information go to [backstage.io](https://backstage.io) or join our [Dis A detailed project roadmap, including already delivered milestones, is available [here](https://backstage.io/docs/overview/roadmap). -## Overview - -The Backstage platform consists of a number of different components: - -- **app** - Main web application that users interact with. It's built up by a number of different _Plugins_. This repo contains an example implementation of an app (located in `packages/app`) and you can easily get started with your own app by [creating one](docs/getting-started/create-an-app.md). -- [**plugins**](https://github.com/spotify/backstage/tree/master/plugins) - Each plugin is treated as a self-contained web app and can include almost any type of content. Plugins all use a common set of platform API's and reusable UI components. Plugins can fetch data either from the _backend_ or through any RESTful API exposed through the _proxy_. -- [**service catalog**](https://github.com/spotify/backstage/tree/master/packages/backend) - Service that holds the model of your software ecosystem, including organisational information and what team owns what software. The backend also has a Plugin model for extending its graph. -- [**proxy**](https://github.com/spotify/backstage/tree/master/plugins/proxy-backend) - Terminates HTTPS and exposes any RESTful API to Plugins. -- **identity** - A backend service that holds your organisation's metadata. - ## Getting Started There are two different ways to get started with Backstage, either by creating a standalone app, or by cloning this repo. Which method you use depends on what you're planning to do. diff --git a/app-config.yaml b/app-config.yaml index 01e2ff33fe..3b3733cf54 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -14,21 +14,15 @@ backend: client: sqlite3 connection: ':memory:' +# See README.md in the proxy-backend plugin for information on the configuration format proxy: - '/circleci/api': - target: 'https://circleci.com/api/v1.1' - changeOrigin: true - pathRewrite: - '^/proxy/circleci/api/': '/' + '/circleci/api': https://circleci.com/api/v1.1 '/jenkins/api': - target: 'http://localhost:8080' - changeOrigin: true + target: http://localhost:8080 headers: Authorization: $secret: env: JENKINS_BASIC_AUTH_HEADER - pathRewrite: - '^/proxy/jenkins/api/': '/' organization: name: Spotify @@ -50,6 +44,46 @@ newrelic: baseUrl: 'https://api.newrelic.com/v2' key: NEW_RELIC_REST_API_KEY +lighthouse: + baseUrl: http://localhost:3003 + +catalog: + processors: + githubApi: + privateToken: + $secret: + env: GITHUB_PRIVATE_TOKEN + bitbucketApi: + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + gitlabApi: + privateToken: + $secret: + env: GITLAB_PRIVATE_TOKEN + azureApi: + privateToken: + $secret: + env: AZURE_PRIVATE_TOKEN + exampleEntityLocations: + github: + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/podcast-api-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/queue-proxy-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/searcher-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-lib-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/www-artist-component.yaml + - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/shuffle-api-component.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml + - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + - https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml + auth: providers: google: @@ -122,3 +156,14 @@ auth: domain: $secret: env: AUTH_AUTH0_DOMAIN + microsoft: + development: + clientId: + $secret: + env: AUTH_MICROSOFT_CLIENT_ID + clientSecret: + $secret: + env: AUTH_MICROSOFT_CLIENT_SECRET + tenantId: + $secret: + env: AUTH_MICROSOFT_TENANT_ID diff --git a/catalog-info.yaml b/catalog-info.yaml index 86b67f95cd..4221ee6dc6 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -7,6 +7,7 @@ metadata: annotations: github.com/project-slug: spotify/backstage backstage.io/github-actions-id: spotify/backstage + backstage.io/techdocs-ref: github:https://github.com/spotify/backstage.git spec: type: library owner: Spotify diff --git a/docs/FAQ.md b/docs/FAQ.md index d230e682a4..4b5df3ace0 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -3,7 +3,7 @@ id: FAQ title: FAQ --- -## Product FAQ: +## Product FAQ ### Can we call Backstage something different? So that it fits our company better? @@ -67,7 +67,7 @@ valuable as you grow. Yes! The Backstage UI is built using Material-UI. With the theming capabilities of Material-UI, you are able to adapt the interface to your brand guidelines. -## Technical FAQ: +## Technical FAQ ### Why Material-UI? diff --git a/docs/README.md b/docs/README.md index b9e79a9e0a..c63a12b589 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,104 +1,3 @@ -# Documentation structure +# Documentation -**Note!** This documentation structure is very much work in progress. If (when, -really 😆) you find broken links or missing content, please create an issue or, -better yet, a pull request. - -# Plugins - -- Overview - - [What is Backstage?](overview/what-is-backstage.md) - - [Backstage architecture](overview/architecture-overview.md) - - [Architecture and terminology](overview/architecture-terminology.md) - - [Roadmap](overview/roadmap.md) - - [Vision](overview/vision.md) -- Getting started - - [Running Backstage locally](getting-started/index.md) - - [Installation](getting-started/installation.md) - - [Local development](getting-started/development-environment.md) - - [Demo deployment](https://backstage-demo.roadie.io) - - Production deployments - - [Create an App](getting-started/create-an-app.md) - - App configuration - - [Configuring App with plugins](getting-started/configure-app-with-plugins.md) - - [Customize the look-and-feel of your App](getting-started/app-custom-theme.md) - - Deployment scenarios - - [Kubernetes](getting-started/deployment-k8s.md) - - [Other](getting-started/deployment-other.md) -- Features - - Software Catalog - - [Overview](features/software-catalog/index.md) - - [System model](features/software-catalog/system-model.md) - - [YAML File Format](features/software-catalog/descriptor-format.md) - - [Extending the model](features/software-catalog/extending-the-model.md) - - [External integrations](features/software-catalog/external-integrations.md) - - [API](features/software-catalog/api.md) - - Software creation templates - - [Overview](features/software-templates/index.md) - - [Adding templates](features/software-templates/adding-templates.md) - - Extending the Scaffolder: - - [Overview](features/software-templates/extending/index.md) - - [Create your own Templater](features/software-templates/extending/create-your-own-templater.md) - - [Create your own Publisher](features/software-templates/extending/create-your-own-publisher.md) - - [Create your own Preparer](features/software-templates/extending/create-your-own-preparer.md) - - Docs-like-code - - [Overview](features/techdocs/README.md) - - [Getting Started](features/techdocs/getting-started.md) - - [Concepts](features/techdocs/concepts.md) - - [Creating and Publishing Documentation](features/techdocs/creating-and-publishing.md) - - [FAQ](features/techdocs/FAQ.md) -- Plugins - - [Overview](plugins/index.md) - - [Existing plugins](plugins/existing-plugins.md) - - [Creating a new plugin](plugins/create-a-plugin.md) - - [Developing a plugin](plugins/plugin-development.md) - - [Structure of a plugin](plugins/structure-of-a-plugin.md) - - Backends and APIs - - [Proxying](plugins/proxying.md) - - [Backstage backend plugin](plugins/backend-plugin.md) - - [Call existing API](plugins/call-existing-api.md) - - Testing - - [Overview](plugins/testing.md) - - Publishing - - [Open source and NPM](plugins/publishing.md) - - [Private/internal (non-open source)](plugins/publish-private.md) -- Configuration - - [Overview](conf/index.md) - - [Reading Configuration](conf/reading.md) - - [Writing Configuration](conf/writing.md) - - [Defining Configuration](conf/defining.md) -- Authentication and identity - - [Overview](auth/index.md) - - [Add auth provider](auth/add-auth-provider.md) - - [Auth backend](auth/auth-backend.md) - - [OAuth](auth/oauth.md) - - [Glossary](auth/glossary.md) -- Designing for Backstage - - [Backstage Design Language System (DLS)](dls/design.md) - - [Storybook -- reusable UI components](http://backstage.io/storybook) - - [Contributing to Storybook](dls/contributing-to-storybook.md) - - [Figma resources](dls/figma.md) -- API references - - TypeScript API - - [Utility APIs](api/utility-apis.md) - - [Utility API References](reference/utility-apis/README.md) - - [createPlugin](reference/createPlugin.md) - - [createPlugin-feature-flags](reference/createPlugin-feature-flags.md) - - [createPlugin-router](reference/createPlugin-router.md) - - Backend APIs - - [Backend](api/backend.md) -- Tutorials - - [Overview](tutorials/index.md) -- Architecture Decision Records (ADRs) - - [Overview](architecture-decisions/index.md) - - [ADR001 - Architecture Decision Record (ADR) log](architecture-decisions/adr001-add-adr-log.md) - - [ADR002 - Default Software Catalog File Format](architecture-decisions/adr002-default-catalog-file-format.md) - - [ADR003 - Avoid Default Exports and Prefer Named Exports](architecture-decisions/adr003-avoid-default-exports.md) - - [ADR004 - Module Export Structure](architecture-decisions/adr004-module-export-structure.md) - - [ADR005 - Catalog Core Entities](architecture-decisions/adr005-catalog-core-entities.md) - - [ADR006 - Avoid React.FC and React.SFC](architecture-decisions/adr006-avoid-react-fc.md) - - [ADR007 - Use MSW for Mocking Network Requests](architecture-decisions/adr007-use-msw-to-mock-service-requests.md) - - [ADR008 - Default Catalog File Name](architecture-decisions/adr008-default-catalog-file-name.md) -- [Contribute](../CONTRIBUTING.md) -- [Support](overview/support.md) -- [FAQ](FAQ.md) +The Backstage documentation is available at https://backstage.io/docs diff --git a/docs/api/utility-apis.md b/docs/api/utility-apis.md index ff8c91e178..b4c2fd161e 100644 --- a/docs/api/utility-apis.md +++ b/docs/api/utility-apis.md @@ -71,18 +71,23 @@ import { AlertApiForwarder, ErrorApiForwarder, ErrorAlerter, + ConfigApi } from '@backstage/core'; -const builder = ApiRegistry.builder(); -// The alert API is a self-contained implementation that shows alerts to the user. -const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); +const apis = (config: ConfigApi) => { + const builder = ApiRegistry.builder(); -// The error API uses the alert API to send error notifications to the user. -builder.add(errorApiRef, new ErrorAlerter(alertApi, new ErrorApiForwarder())); + // The alert API is a self-contained implementation that shows alerts to the user. + const alertApi = builder.add(alertApiRef, new AlertApiForwarder()); + + // The error API uses the alert API to send error notifications to the user. + builder.add(errorApiRef, new ErrorAlerter(alertApi, new ErrorApiForwarder())); + return builder.build(); +} const app = createApp({ - apis: apiBuilder.build(), + apis, // ... other config }); ``` diff --git a/docs/architecture-decisions/index.md b/docs/architecture-decisions/index.md index b02e8deabb..91d2c668d7 100644 --- a/docs/architecture-decisions/index.md +++ b/docs/architecture-decisions/index.md @@ -4,8 +4,6 @@ title: Architecture Decision Records (ADR) sidebar_label: Overview --- -# - The substantial architecture decisions made in the Backstage project lives here. For more information about ADRs, when to write them, and why, please see [this blog post](https://engineering.atspotify.com/2020/04/14/when-should-i-write-an-architecture-decision-record/). @@ -25,7 +23,10 @@ Records should be stored under the `architecture-decisions` directory. - Submit a pull request - Address and integrate feedback from the community - Eventually, assign a number -- Add the full path of the ADR to the [`mkdocs.yml`](/mkdocs.yml) +- Add the path of the ADR to the microsite sidebar in + [`sidebars.json`](https://github.com/spotify/backstage/blob/master/microsite/sidebars.json) +- Add the path of the ADR to the + [`mkdocs.yml`](https://github.com/spotify/backstage/blob/master/mkdocs.yml) - Merge the pull request ## Superseding an ADR diff --git a/docs/getting-started/create-app_output.png b/docs/assets/getting-started/create-app_output.png similarity index 100% rename from docs/getting-started/create-app_output.png rename to docs/assets/getting-started/create-app_output.png diff --git a/docs/getting-started/create-plugin_output.png b/docs/assets/getting-started/create-plugin_output.png similarity index 100% rename from docs/getting-started/create-plugin_output.png rename to docs/assets/getting-started/create-plugin_output.png diff --git a/docs/assets/pop.png b/docs/assets/pop.png new file mode 100644 index 0000000000..441126f33c Binary files /dev/null and b/docs/assets/pop.png differ diff --git a/docs/features/software-catalog/bsc-edit.png b/docs/assets/software-catalog/bsc-edit.png similarity index 100% rename from docs/features/software-catalog/bsc-edit.png rename to docs/assets/software-catalog/bsc-edit.png diff --git a/docs/features/software-catalog/bsc-extend.png b/docs/assets/software-catalog/bsc-extend.png similarity index 100% rename from docs/features/software-catalog/bsc-extend.png rename to docs/assets/software-catalog/bsc-extend.png diff --git a/docs/features/software-catalog/bsc-register-1.png b/docs/assets/software-catalog/bsc-register-1.png similarity index 100% rename from docs/features/software-catalog/bsc-register-1.png rename to docs/assets/software-catalog/bsc-register-1.png diff --git a/docs/features/software-catalog/bsc-register-2.png b/docs/assets/software-catalog/bsc-register-2.png similarity index 100% rename from docs/features/software-catalog/bsc-register-2.png rename to docs/assets/software-catalog/bsc-register-2.png diff --git a/docs/features/software-catalog/bsc-search.png b/docs/assets/software-catalog/bsc-search.png similarity index 100% rename from docs/features/software-catalog/bsc-search.png rename to docs/assets/software-catalog/bsc-search.png diff --git a/docs/features/software-catalog/bsc-starred.png b/docs/assets/software-catalog/bsc-starred.png similarity index 100% rename from docs/features/software-catalog/bsc-starred.png rename to docs/assets/software-catalog/bsc-starred.png diff --git a/docs/assets/software-catalog/software-model-core-entities.png b/docs/assets/software-catalog/software-model-core-entities.png new file mode 100644 index 0000000000..b718b7527c Binary files /dev/null and b/docs/assets/software-catalog/software-model-core-entities.png differ diff --git a/docs/assets/techdocs/documentation-template.png b/docs/assets/techdocs/documentation-template.png new file mode 100644 index 0000000000..1f44ad27c6 Binary files /dev/null and b/docs/assets/techdocs/documentation-template.png differ diff --git a/docs/assets/techdocs/techdocs_big_picture.png b/docs/assets/techdocs/techdocs_big_picture.png new file mode 100644 index 0000000000..ffe437180f Binary files /dev/null and b/docs/assets/techdocs/techdocs_big_picture.png differ diff --git a/docs/auth/auth-backend-classes.md b/docs/auth/auth-backend-classes.md index 42ee48c5e0..9034a3d054 100644 --- a/docs/auth/auth-backend-classes.md +++ b/docs/auth/auth-backend-classes.md @@ -1,4 +1,7 @@ -# Authentication Backend Classes Layout and Description +--- +id: auth-backend-classes +title: Auth backend classes +--- ## How Does Authentication Work? diff --git a/docs/auth/oauth.md b/docs/auth/oauth.md index 6e6dc4da80..b7013a310d 100644 --- a/docs/auth/oauth.md +++ b/docs/auth/oauth.md @@ -104,6 +104,8 @@ request an access token. The following diagram visualizes the flow described in the previous section. +![](oauth-popup-flow.svg) + - -![](oauth-popup-flow.svg) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md new file mode 100644 index 0000000000..d4cc7fc0c2 --- /dev/null +++ b/docs/features/software-catalog/configuration.md @@ -0,0 +1,57 @@ +--- +id: software-catalog-configuration +title: Catalog Configuration +--- + +## Static Location Configuration + +To enable declarative catalog setups, it is possible to add locations to the +catalog via [static configuration](../../conf/index.md). Locations are added to +the catalog under the `catalog.locations` key, for example: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +The locations added through static configuration can not be removed through the +catalog locations API. To remove the locations, you have to remove them from the +configuration. + +## Catalog Rules + +By default the catalog will only allow ingestion of entities with the kind +`Component` and `API`. In order to allow entities of other kinds to be added, +you need to add rules to the catalog. Rules are added either in a separate +`catalog.rules` key, or added to statically configured locations. + +For example, given the following configuration: + +```yaml +catalog: + rules: + - allow: [Component, API, System] + + locations: + - type: github + target: https://github.com/org/example/blob/master/org-data.yaml + allow: [Group] +``` + +We are able to add entities of kind `Component`, `API`, or `System` from any +location, and `Group` entities from the `org-data.yaml`, which will also be read +as statically configured location. + +Note that if the `catalog.rules` key is present it will replace the default +value, meaning that you need to add rules for `Component` and `API` kinds if you +want those to be allowed. + +The following configuration will reject any kind of entities from being added to +the catalog: + +```yaml +catalog: + rules: [] +``` diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index a7d57890dc..78dfcf365c 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -370,8 +370,8 @@ metadata: description: Next.js application skeleton for creating isomorphic web applications. tags: - - Recommended - - React + - recommended + - react spec: owner: web@example.com templater: cookiecutter diff --git a/docs/features/software-catalog/extending-the-model.md b/docs/features/software-catalog/extending-the-model.md index ce5ea03b26..8b621f2e44 100644 --- a/docs/features/software-catalog/extending-the-model.md +++ b/docs/features/software-catalog/extending-the-model.md @@ -12,7 +12,7 @@ Backstage natively supports tracking of the following component - Documentation - Other -![](bsc-extend.png) +![](../../assets/software-catalog/bsc-extend.png) Since these types are likely not the only kind of software you will want to track in Backstage, it is possible to @@ -31,9 +31,10 @@ catalog. It might be tempting to put software that doesn't fit into any of the existing types into Other. There are a few reasons why we advice against this; firstly, we have found that it is preferred to match the conceptual model that your -engineers have when describing your sofware. Secondly, Backstage helps your -engineers manage their software by integrating the infratrucure tooling through -plugins. Different plugins are used for managing different types of components. +engineers have when describing your software. Secondly, Backstage helps your +engineers manage their software by integrating the infrastructure tooling +through plugins. Different plugins are used for managing different types of +components. For example, the [Lighthouse plugin](https://github.com/spotify/backstage/tree/master/plugins/lighthouse) diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index fe107d1f59..bfc3d53d4d 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -1,6 +1,7 @@ --- id: software-catalog-overview title: Backstage Service Catalog (alpha) +sidebar_label: Backstage Service Catalog --- ## What is a Service Catalog? @@ -54,18 +55,18 @@ There are 3 ways to add components to the catalog: Users can register new components by going to `/create` and clicking the **REGISTER EXISTING COMPONENT** button: -![](bsc-register-1.png) +![](../../assets/software-catalog/bsc-register-1.png) Backstage expects the full URL to the YAML in your source control. Example: -``` +```bash https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml ``` _More examples can be found [here](https://github.com/spotify/backstage/tree/master/packages/catalog-model/examples)._ -![](bsc-register-2.png) +![](../../assets/software-catalog/bsc-register-2.png) It is important to note that any kind of software can be registered in Backstage. Even if the software is not maintained by your company (SaaS @@ -78,12 +79,28 @@ All software created through the [Backstage Software Templates](../software-templates/index.md) are automatically registered in the catalog. +### Static catalog configuration + +In addition to manually registering components, it is also possible to register +components though [static configuration](../../conf/index.md). For example, the +above example can be added using the following configuration: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +More information about catalog configuration can be found +[here](configuration.md). + ### Updating component metadata Teams owning the components are responsible for maintaining the metadata about them, and do so using their normal Git workflow. -![](bsc-edit.png) +![](../../assets/software-catalog/bsc-edit.png) Once the change has been merged, Backstage will automatically show the updated metadata in the service catalog after a short while. @@ -92,25 +109,25 @@ metadata in the service catalog after a short while. By default the service catalog shows components owned by the team of the logged in user. But you can also switch to _All_ to see all the components across your -companie's software ecosystem. Basic inline _search_ and _column filtering_ -makes it easy to browse a big set of components. +company's software ecosystem. Basic inline _search_ and _column filtering_ makes +it easy to browse a big set of components. -![](bsc-search.png) +![](../../assets/software-catalog/bsc-search.png) ## Starring components For easy and quick access to components you visit frequently, Backstage supports _starring_ of components: -![](bsc-starred.png) +![](../../assets/software-catalog/bsc-starred.png) ## Integrated tooling through plugins -The service catalog is a great way to organise the infrastructure tools you use +The service catalog is a great way to organize the infrastructure tools you use to manage the software. This is how Backstage creates one developer portal for all your tools. Rather than asking teams to jump between different -infrastructure UI’s (and incurring additional cognitive overhead each time they -make a context switch), most of these tools can be organised around the entities +infrastructure UIs (and incurring additional cognitive overhead each time they +make a context switch), most of these tools can be organized around the entities in the catalog. ![tools](https://backstage.io/blog/assets/20-05-20/tabs.png) diff --git a/docs/features/software-catalog/system-model.md b/docs/features/software-catalog/system-model.md index 4b2884e097..4436f60fc2 100644 --- a/docs/features/software-catalog/system-model.md +++ b/docs/features/software-catalog/system-model.md @@ -3,55 +3,26 @@ id: system-model title: System Model --- -We believe that a strong shared understanding and terminology around systems, -software and resources leads to a better Backstage experience. +We believe that a strong shared understanding and terminology around software +and resources leads to a better Backstage experience. _This description originates from [this RFC](https://github.com/spotify/backstage/issues/390). Note that some of the concepts are not yet supported in Backstage._ -## Concepts +## Core Entities -We model our technology using these five concepts (further explained below): +We model software in the Backstage catalogue using these three core entities +(further explained below): + +- **Components** are individual pieces of software + +- **APIs** are the boundaries between different components -- **Domains** are a high-level grouping of systems -- **Systems** encapsulate the implementation of APIs -- **APIs** are the boundaries between different components and systems -- **Components** are pieces of software - **Resources** are physical or virtual infrastructure needed to operate a - system + component -![Software Ecosystem Model_ Public Github version](https://user-images.githubusercontent.com/24575/77633084-39bcde80-6f4f-11ea-8251-f8df561a3652.png) - -### Domain - -While systems are the basic level of encapsulation for resources, components and -APIs, it is often useful to group a collection of systems that share -terminology, domain models, business purpose, or documentation, i.e. they form a -bounded context. - -For example, it would make sense if the different systems in the “Payments” -domain would come with some documentation on how to accept payments for a new -product or use-case, share the same entity types in their APIs, and integrate -well with each other. - -### System - -With increasing complexity in software, we believe that systems form an -important abstraction level to help us reason about software ecosystems. Systems -are a useful concept in that they allow us to ignore the implementation details -of a certain functionality for consumers, while allowing the owning team to make -changes as they see fit (leading to low coupling). - -A system, in this sense, is a collection of resources and components that -exposes one or several APIs. Components and resources in a system are typically -owned by the same team and are expected to co-evolve. As such, systems usually -consist of at most a handful of components. - -For example, a playlist management system might encapsulate a backend service to -update playlists, a backend service to query them, and a database to store them. -It could expose an RPC API, a daily snapshots dataset, and an event stream of -playlist updates. +![](../../assets/software-catalog/software-model-core-entities.png) ### Component @@ -60,34 +31,80 @@ backend service or data pipeline (list not exhaustive). A component can be tracked in source control, or use some existing open source or commercial software. -A component can implement APIs for other components to consume. It might depend -on the resources of the system it belongs to, and APIs from other components or -other systems. All other aspects of the component, e.g. any code dependencies, -must be encapsulated. +A component can implement APIs for other components to consume. In turn it might +depend on APIs implemented by other components, or resources that are attached +to it at runtime. ### API -We believe APIs form an important (maybe the most important) abstraction that -allows large software ecosystems to scale. Thus, APIs are a first class citizen -in the Backstage model and the primary way to discover existing functionality in -the ecosystem. +APIs form an important (maybe the most important) abstraction that allows large +software ecosystems to scale. Thus, APIs are a first class citizen in the +Backstage model and the primary way to discover existing functionality in the +ecosystem. -APIs are implemented by components and form boundaries between components and -systems. They might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a -data schema (eg Avro, TFRecord, ...), or as code interfaces. In any case, APIs -exposed by components need to be in a known machine-readable format so we can -build further tooling and analysis on top. +APIs are implemented by components and form boundaries between components. They +might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a data schema (eg +Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by +components need to be in a known machine-readable format so we can build further +tooling and analysis on top. -Some APIs might be exposed by the system, making them available for any other -Spotify component to consume. Those public APIs must be documented and humanly -discoverable in Backstage. +APIs have a visibility: they are either public (making them available for any +other component to consume), restricted (only available to a whitelisted set of +consumers), or private (only available within their system). As public APIs are +going to be the primary way interaction between components, Backstage supports +documenting, indexing and searching all APIs so we can browse them as +developers. ### Resource -Resources are the infrastructure a system needs to operate, like BigTable -databases, Pub/Sub topics, S3 buckets or CDNs. Modelling them together with -components and systems will better allow us to visualize resource footprint, and -create tooling around them. +Resources are the infrastructure a component needs to operate at runtime, like +BigTable databases, Pub/Sub topics, S3 buckets or CDNs. Modelling them together +with components and systems will better allow us to visualize resource +footprint, and create tooling around them. + +## Ecosystem Modeling + +A large catalogue of components, APIs and resources can be highly granular and +hard to understand as a whole. It might thus be convenient to further categorize +these entities using the following (optional) concepts: + +- **Systems** are a collection of entities that cooperate to perform some + function +- **Domains** relate entities and systems to part of the business + +### System + +With increasing complexity in software, systems form an important abstraction +level to help us reason about software ecosystems. Systems are a useful concept +in that they allow us to ignore the implementation details of a certain +functionality for consumers, while allowing the owning team to make changes as +they see fit (leading to low coupling). + +A system, in this sense, is a collection of resources and components that +exposes one or several public APIs. The main benefit of modelling a system is +that it hides its resources and private APIs between the components for any +consumers. This means that as the owner, you can evolve the implementation, in +terms of components and resources, without your consumers being able to notice. +Typically, a system will consist of at most a handful of components (see Domain +for a grouping of systems). + +For example, a playlist management system might encapsulate a backend service to +update playlists, a backend service to query them, and a database to store them. +It could expose an RPC API, a daily snapshots dataset, and an event stream of +playlist updates. + +### Domain + +While systems are the basic level of encapsulation for related entities, it is +often useful to group a collection of systems that share terminology, domain +models, metrics, KPIs, business purpose, or documentation, i.e. they form a +bounded context. + +For example, it would make sense if the different systems in the “Payments” +domain would come with some documentation on how to accept payments for a new +product or use-case, share the same entity types in their APIs, and integrate +well with each other. Other domains could be “Content Ingestion”, “Ads” or +“Search”. ## Current status diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 2ec4bdd6f3..67467bd98e 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -22,8 +22,8 @@ metadata: Next.js application skeleton for creating isomorphic web applications. # some tags to display in the frontend tags: - - Recommended - - React + - recommended + - react spec: # which templater key to use in the templaters builder templater: cookiecutter @@ -54,11 +54,11 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. -Currently the catalog supports loading definitions from Github + Local Files. To +Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. -For loading from a file the following command should work when the backend is +For loading from a file, the following command should work when the backend is running: ```sh @@ -69,7 +69,7 @@ curl \ --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" ``` -If loading from a git location, you can run the following +If loading from a Git location, you can run the following ```sh curl \ @@ -83,7 +83,7 @@ This should then have added the catalog, and also should now be listed under the create page at http://localhost:3000/create. Alternatively, if you want to get setup with some mock templates that are -already provided for you, you can run the following to load those templates: +already provided, run the following to load those templates: ``` yarn lerna run mock-data diff --git a/docs/features/software-templates/extending/create-your-own-preparer.md b/docs/features/software-templates/extending/create-your-own-preparer.md index 739c516658..0b3fb31323 100644 --- a/docs/features/software-templates/extending/create-your-own-preparer.md +++ b/docs/features/software-templates/extending/create-your-own-preparer.md @@ -54,7 +54,7 @@ The `protocol` is set on the when added to the service catalog. You can see more about this `PreparerKey` here in [Register your own template](../adding-templates.md) -**note:** Currently the catalog supports loading definitions from Github + Local +**note:** Currently the catalog supports loading definitions from GitHub + Local Files, which translate into the two `PreparerKeys` `file` and `github`. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index 51869dc3ad..391f6f0658 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -7,8 +7,8 @@ Publishers are responsible for pushing and storing the templated skeleton after the values have been templated by the `Templater`. See [Create your own templater](./create-your-own-templater.md) for more info. -They recieve a directory or location where the templater has sucessfully run on, -and is now ready to store somewhere. They also get given some other options +They receive a directory or location where the templater has sucessfully run +and is now ready to store somewhere. They also are given some other options which are sent from the frontend, such as the `storePath` which is a string of where the frontend thinks we should save this templated folder. @@ -17,7 +17,7 @@ Currently we provide the following `publishers`: - `github` This publisher is passed through to the `createRouter` function of the -`@spotify/plugin-scaffolder-backend`. Currently only one publisher is supported, +`@spotify/plugin-scaffolder-backend`. Currently, only one publisher is supported, but PR's are always welcome. An full example backend can be found diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md index ac05eed3b0..9dee66af06 100644 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ b/docs/features/software-templates/extending/create-your-own-templater.md @@ -8,14 +8,14 @@ returned by the preparers, and then executing the templating command on top of the file and returning the completed template path. This may or may not be the same directory as the input directory. -They also recieve additional values from the frontend, which can be used to +They also receive additional values from the frontend, which can be used to interpolate into the skeleton files. Currently we provide the following templaters: - `cookiecutter` -This templater is added the `TemplaterBuilder` and then passed into the +This templater is added to the `TemplaterBuilder` and then passed into the `createRouter` function of the `@spotify/plugin-scaffolder-backend` An full example backend can be found @@ -48,7 +48,7 @@ This `TemplaterKey` is used to select the correct templater from the `spec.templater` in the [Template Entity](../../software-catalog/descriptor-format.md#kind-template). -If you wish to add a new templater you'll need to register it with the +If you wish to add a new templater, you'll need to register it with the `TemplaterBuilder`. ### Creating your own Templater to add to the `TemplaterBuilder` @@ -83,10 +83,10 @@ follows: - `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to be able to run docker containers. -_note_ currently the templaters that we provide are basically docker action +_note_ Currently the templaters that we provide are basically Docker action containers that are run on top of the skeleton folder. This keeps dependencies to a minimal for running backstage scaffolder, but you don't /have/ to use -docker. You could create your own templater that spins up an EC2 instance and +Docker. You could create your own templater that spins up an EC2 instance and downloads the folder and does everything using an AMI if you want. It's entirely up to you! @@ -116,8 +116,8 @@ metadata: description: Next.js application skeleton for creating isomorphic web applications. tags: - - Recommended - - React + - recommended + - react spec: owner: web@example.com templater: handlebars @@ -138,7 +138,7 @@ spec: description: Description of the component ``` -You see that the `spec.templater` is set as `handlebars`, you'll need to +You see that the `spec.templater` is set as `handlebars`, so you'll need to register this with the `TemplaterBuilder` like so: ```ts diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md index 21d8befda1..02d27a5725 100644 --- a/docs/features/software-templates/extending/index.md +++ b/docs/features/software-templates/extending/index.md @@ -5,11 +5,11 @@ title: Extending the Scaffolder Welcome. Take a seat. You're at the Scaffolder Documentation. -So - You wanna create stuff inside your company from some prebaked templates? +So, you want to create stuff inside your company from some prebaked templates? You're at the right place. -This guide is gonna take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through whats going on in the backend to be +This guide is going to take you through how the Scaffolder in Backstage works. +We'll dive into some jargon and run through what's going on in the backend to be able to create these templates. There's also more guides that you might find useful at the bottom of this document. At it's core, theres 3 simple stages. @@ -25,9 +25,9 @@ scaffolder that you will need to know: 3. Publish Each of these steps can be configured for your own use case, but we provide some -sensible defaults too. +sensible defaults, too. -Lets dive a little deeper into these phases. +Let's dive a little deeper into these phases. ### Glossary and Jargon @@ -38,8 +38,8 @@ the router to pick the correct `Preparer` to run for the `Template` entity. **Templater** - The templater is responsible for actually running the chosen templater on top of the previously returned temporary directory from the -**Preprarer**. We advise making these docker containers as it can keep all -dependencies, for example Cookiecutter, self contained and not a dependency on +**Preprarer**. We advise making these Docker containers as it can keep all +dependencies--for example Cookiecutter--self contained and not a dependency on the host machine. **Publisher** - The publisher is responsible for taking the finished directory, @@ -50,11 +50,11 @@ passed through to the scaffolder backend. ### How it works -The main of the heavy lifting is done in the +Most of the heavy lifting is done in the [router.ts](https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) file in the `scaffolder-backend` plugin. -There are 2 routes defined in the router. `POST /v1/jobs` and +There are two routes defined in the router: `POST /v1/jobs` and `GET /v1/job/:jobId` To create a scaffolding job, a JSON object containing the @@ -78,7 +78,7 @@ additional templating values must be posted as the post body. The values should represent something that is valid with the `schema` part of the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) -Once that has been posted, a job will be setup with different stages. And the +Once that has been posted, a job will be setup with different stages, and the job processor will complete each stage before moving onto the next stage, whilst collecting logs and mutating the running job. diff --git a/docs/features/software-templates/index.md b/docs/features/software-templates/index.md index d9a86ee7aa..1c67676b3e 100644 --- a/docs/features/software-templates/index.md +++ b/docs/features/software-templates/index.md @@ -4,8 +4,8 @@ title: Software Templates --- The Software Templates part of Backstage is a tool that can help you create -Components inside Backstage. It by default has the ability to load skeletons of -code, template in some variables and then publish the template to some location +Components inside Backstage. By default, it has the ability to load skeletons of +code, template in some variables, and then publish the template to some location like GitHub.