Merge branch 'master' into mcalus3/add-catalog-import-plugin

2020-10-25 20:41:35 +01:00
parent ac3b033665 ae567d8125
commit 877bfc0243
302 changed files with 3853 additions and 2212 deletions
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog': minor
+---
+
+Add client side paging for catalog table
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-cost-insights': minor
+---
+
+Added getLastCompleteBillingDate to the CostInsightsApi to reason about completeness of billing data
@@ -0,0 +1,21 @@
+---
+'@backstage/backend-common': minor
+'@backstage/cli': minor
+'@backstage/config-loader': minor
+'example-backend': patch
+'@backstage/create-app': patch
+---
+
+**BREAKING CHANGE**
+
+The existing loading of additional config files like `app-config.development.yaml` using APP_ENV or NODE_ENV has been removed.
+Instead, the CLI and backend process now accept one or more `--config` flags to load config files.
+
+Without passing any flags, `app-config.yaml` and, if it exists, `app-config.local.yaml` will be loaded.
+If passing any `--config <path>` flags, only those files will be loaded, **NOT** the default `app-config.yaml` one.
+
+The old behaviour of for example `APP_ENV=development` can be replicated using the following flags:
+
+```bash
+--config ../../app-config.yaml --config ../../app-config.development.yaml
+```
@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+Add forwardRef to the SidebarItem
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder': patch
+---
+
+fix the accordion details design when job stage fail
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Fix CodeOwnersProcessor to handle non team users
@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+rename stories folder top Chip
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog': patch
+---
+
+Remove "in default" in component name
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-techdocs-backend': patch
+---
+
+Replacing the hard coded `baseApiUrl` by reading the value from configuration to enable private GitHub setup for TechDocs.
@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+update ItemCard component and it's story
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-cost-insights': minor
+---
+
+Remove product filters from query parameters
@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+make ErrorPage responsive + fix the test case
@@ -0,0 +1,5 @@
+---
+'@backstage/core': patch
+---
+
+fix the responsive of page story
@@ -0,0 +1,8 @@
+---
+'@backstage/backend-common': minor
+'example-backend': patch
+'@backstage/cli': patch
+'@backstage/create-app': patch
+---
+
+Change loadBackendConfig to return the config directly
@@ -14,3 +14,4 @@
 /packages/techdocs-container                @spotify/techdocs-core
 /.github/workflows/techdocs.yml             @spotify/techdocs-core
 /.github/workflows/techdocs-pypi.yml        @spotify/techdocs-core
+/.changeset/cost-insights-*                 @spotify/silver-lining
@@ -6,6 +6,16 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re

 ## Next Release

+### @backstage/cli
+
+- Configuration files are no longer selected through `APP_ENV` or `NODE_ENV`. The default configuration files are `app-config.yaml` and, fix it exists, `app-config.local.yaml` in the repo root. To load a different set of files, use `--config <path>` arguments.
+
+### @backstage/backend-common
+
+- Configuration files are no longer selected through `APP_ENV` or `NODE_ENV`. The default configuration files are `app-config.yaml` and, fix it exists, `app-config.local.yaml` in the repo root. To load a different set of files, use `--config <path>` arguments.
+
+## v0.1.1-alpha.25
+
 > Collect changes for the next release below

 ### @backstage/cli
@@ -1,13 +0,0 @@
-app:
-  baseUrl: http://localhost:3000
-
-backend:
-  baseUrl: http://localhost:7000
-  listen:
-    port: 7000
-  cors:
-    origin: http://localhost:3000
-    methods: [GET, POST, PUT, DELETE]
-    credentials: true
-  csp:
-    connect-src: ["'self'", 'http:', 'https:']
@@ -1,6 +1,6 @@
 app:
  title: Backstage Example App
-  baseUrl: http://localhost:7000
+  baseUrl: http://localhost:3000
  googleAnalyticsTrackingId: # UA-000000-0

 backend:
@@ -10,8 +10,12 @@ backend:
  database:
    client: sqlite3
    connection: ':memory:'
+  cors:
+    origin: http://localhost:3000
+    methods: [GET, POST, PUT, DELETE]
+    credentials: true
  csp:
-    connect-src: ["'self'", 'https:']
+    connect-src: ["'self'", 'http:', 'https:']

 # See README.md in the proxy-backend plugin for information on the configuration format
 proxy:
@@ -146,6 +150,9 @@ catalog:
    # Backstage example templates
    - type: url
      target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml
+    # Backstage example groups and users
+    - type: url
+      target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme-corp.yaml

 scaffolder:
  github:
@@ -28,6 +28,9 @@ spec:
          image: example-backend:latest
          imagePullPolicy: Never

+          command: [node, packages/backend]
+          args: [--config, app-config.yaml, --config, k8s-config.yaml]
+
          env:
            # We set this to development to make the backend start with incomplete configuration. In a production
            # deployment you will want to make sure that you have a full configuration, and remove any plugins that
@@ -35,10 +38,6 @@ spec:
            - name: NODE_ENV
              value: development

-              # This makes us load in `app-config.production.yaml` if there is one.
-            - name: APP_ENV
-              value: production
-
              # This makes it possible for the app to reach the backend when serving through `kubectl proxy`
              # If you expose the service using for example an ingress controller, you should
              # switch this out or remove it.
@@ -54,8 +53,8 @@ spec:

          volumeMounts:
            - name: config-volume
-              mountPath: /usr/src/app/app-config.local.yaml
-              subPath: app-config.local.yaml
+              mountPath: /usr/src/app/k8s-config.yaml
+              subPath: k8s-config.yaml

          resources:
            limits:
@@ -77,7 +76,7 @@ spec:
            name: backstage-config
            items:
              - key: app-config
-                path: app-config.local.yaml
+                path: k8s-config.yaml
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -20,7 +20,7 @@ Each of these methods is hosted at an endpoint `/auth/[provider]/method`, where
 ```

 For more information on how these methods are used and for which purpose, refer
-to the documentation [here](oauth.md).
+to the [OAuth documentation](oauth.md).

 For details on the parameters, input and output conditions for each method,
 refer to the type documentation under
@@ -38,7 +38,7 @@ Currently OAuth is assumed to be the de facto authentication mechanism for
 Backstage based applications.

 Backstage comes with a "batteries-included" set of supported commonly used OAuth
-providers: Okta, Github, Google, Gitlab, and a generic OAuth2 provider.
+providers: Okta, GitHub, Google, GitLab, and a generic OAuth2 provider.

 All of these use the authorization flow of OAuth2 to implement authentication.

@@ -15,11 +15,11 @@ allowing for customization.

 ## Supplying Configuration

-Configuration is stored in `app-config.yaml` files, with support for suffixes
-such as `app-config.production.yaml` to override values for specific
-environments. The configuration files themselves contain plain YAML, but with
-support for loading in secrets from various sources using for example `$env` and
-`$file` keys.
+Configuration is stored in YAML files where the defaults are `app-config.yaml`
+and `app-config.local.yaml` for local overrides. Other sets of files can by
+loaded by passing `--config <path>` flags. The configuration files themselves
+contain plain YAML, but with support for loading in secrets from various sources
+using for example `$env` and `$file` keys.

 It is also possible to supply configuration through environment variables, for
 example `APP_CONFIG_app_baseUrl=https://staging.example.com`. However these
@@ -55,20 +55,28 @@ picked up by the serve tasks of `@backstage/cli` for local development, and are
 injected by the entrypoint of the nginx container serving the frontend in a
 production build.

-## File Resolution
+## Configuration Files

 It is possible to have multiple configuration files, both to support different
 environments, but also to define configuration that is local to specific
-packages.
+packages. The configuration files to load are selected using a `--config <path>`
+flag, and it is possible to load any number of files. Paths are relative to the
+working directory of the executed process, for example `package/backend`. This
+means that to select a config file in the repo root when running the backend,
+you would use `--config ../../my-config.yaml`.

-All `app-config.yaml` files inside the monorepo root and package root are
-considered, as are files with additional `local` and environment affixes such as
-`development`, for example `app-config.local.yaml`,
-`app-config.production.yaml`, and `app-config.development.local.yaml`. Which
-environment config files are loaded is determined by the `APP_ENV` environment
-variable, or `NODE_ENV` if it is not set. Local configuration files are always
-loaded, but are meant for local development overrides and should typically be
-`.gitignore`'d.
+If no `config` flags are specified, the default behavior is to load
+`app-config.yaml` and, if it exists, `app-config.local.yaml` from the repo root.
+In the provided project setup, `app-config.local.yaml` is `.gitignore`'d, making
+it a good place to add config overrides and secrets for local development.
+
+Note that if any config flags are provided, the default `app-config.yaml` files
+are NOT loaded. To include them you need to explicitly include them with a flag,
+for example:
+
+```
+yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml
+```

 All loaded configuration files are merged together using the following rules:

@@ -84,10 +92,10 @@ order:

 - Configuration from the `APP_CONFIG_` environment variables has the highest
  priority, followed by files.
- Files inside package directories have higher priority than those in the root
-  directory.
- Files with environment affixes have higher priority than ones without.
- Files with the `local` affix have higher priority than ones without.
+- Files loaded with config flags are ordered by priority, where the last flag
+  has the highest priority.
+- If no config flags are provided, `app-config.local.yaml` has higher priority
+  than `app-config.yaml`.

 ## Secrets

@@ -19,7 +19,7 @@ are then harvested and visualized in Backstage.

 ## How it works

-Backstage and the Backstage Service Catalog makes it easy for one team to manage
+Backstage and the Backstage Service Catalog make it easy for one team to manage
 10 services — and makes it possible for your company to manage thousands of
 them.

@@ -5,8 +5,8 @@ description: Documentation on Adding your own Templates
 ---

 Templates are stored in the **Service Catalog** under a kind `Template`. The
-minimum that the template skeleton needs is a `template.yaml` but it would be
-good to also have some files in there that can be templated in.
+minimum that is needed to define a template is a `template.yaml` file, but it
+would be good to also have some files in there that can be templated in.

 A simple `template.yaml` definition might look something like this:

@@ -61,7 +61,7 @@ support to load the location will also need to be added to the Catalog.

 You can add the template files to the catalog through
 [static location configuration](../software-catalog/configuration.md#static-location-configuration),
-for example
+for example:

 ```yaml
 catalog:
@@ -100,11 +100,11 @@ curl \
  --location \
  --request POST 'localhost:7000/api/catalog/locations' \
  --header 'Content-Type: application/json' \
-  --data-raw "{\"type\": \"github\", \"target\": \"https://${YOUR GITHUB REPO}blob/master/${PATH TO FOLDER}/template.yaml\"}"
+  --data-raw "{\"type\": \"github\", \"target\": \"https://${GITHUB URL}/${YOUR GITHUB ORG/REPO}/blob/master/${PATH TO FOLDER}/template.yaml\"}"
 ```

-This should then have added the catalog, and also should now be listed under the
-create page at http://localhost:3000/create.
+This should then have been added the catalog, and be listed under the create
+page at http://localhost:3000/create.

 The `type` field which is chosen in the request to add the `template.yaml` to
 the Service Catalog here, will become the `PreparerKey` which will be used to
@@ -15,10 +15,10 @@ location protocols:
 - `github://`

 These two are added to the `PreparersBuilder` and then passed into the
-`createRouter` function of the `@spotify/plugin-scaffolder-backend`
+`createRouter` function of the `@spotify/plugin-scaffolder-backend`.

-A full example backend can be found
-[here](https://github.com/spotify/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts),
+A full example backend can be found in
+[`scaffolder.ts`](https://github.com/spotify/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts),
 but it looks something like the following

 ```ts
@@ -56,7 +56,7 @@ when added to the service catalog. You can see more about this `PreparerKey`
 here in [Register your own template](../adding-templates.md)

 **note:** Currently the catalog supports loading definitions from GitHub + Local
-Files, which translate into the two `PreparerKeys` `file` and `github`. To load
+Files, which translate into the two `PreparerKeys`: `file` and `github`. To load
 from other places, not only will there need to be another preparer, but the
 support to load the location will also need to be added to the Catalog.

@@ -39,17 +39,17 @@ internally.
 ![Enter some variables](../../assets/software-templates/template-picked.png)

 After filling in these variables, you'll get some more fields to fill out which
-are required for backstage usage: the owner, (which is a `user` in the backstage
+are required for backstage usage: the owner (which is a `user` in the backstage
 system), the `storePath` (which right now must be a GitHub Organisation or
-GitHub user), a non-existing github repository name in the format
-`organisation/reponame`, and a GitHub team or user account which should be
+GitHub user and a non-existing GitHub repository name in the format
+`organisation/reponame`), and a GitHub team or user account which should be
 granted admin access to the repository.

 ![Enter backstage vars](../../assets/software-templates/template-picked-2.png)

 ### Run!

-Once you've entered values and confirmed, you'll then get a modal with live
+Once you've entered values and confirmed, you'll then get a popup box with live
 progress of what is currently happening with the creation of your template.

 ![Templating Running](../../assets/software-templates/running.png)
@@ -70,6 +70,6 @@ you to the registered component in the catalog:

 ![Catalog](../../assets/software-templates/go-to-catalog.png)

-And then you'll also be able to see it in the Catalog View table
+And then you'll also be able to see it in the Catalog View table:

 ![Catalog](../../assets/software-templates/added-to-the-catalog-list.png)
@@ -211,22 +211,16 @@ docs on creating private GitHub access tokens is available
 Note that the need for private GitHub access tokens will be replaced with GitHub
 Apps integration further down the line.

-#### Github
+#### GitHub

-The Github access token is retrieved from environment variables via the config.
+The GitHub access token is retrieved from environment variables via the config.
 The config file needs to specify what environment variable the token is
 retrieved from. Your config should have the following objects.

 You can configure who can see the new repositories that the scaffolder creates
 by specifying `visibility` option. Valid options are `public`, `private` and
-`internal`. `internal` options is for GitHub Enterprise clients, which means
-public within the organization.
-
-#### Gitlab
-
-For Gitlab, we currently support the configuration of the GitLab publisher and
-allows to configure the private access token and the base URL of a GitLab
-instance:
+`internal`. The `internal` option is for GitHub Enterprise clients, which means
+public within the enterprise.

 ```yaml
 scaffolder:
@@ -234,6 +228,16 @@ scaffolder:
    token:
      $env: GITHUB_TOKEN
    visibility: public # or 'internal' or 'private'
+```
+
+#### GitLab
+
+For GitLab, we currently support the configuration of the GitLab publisher and
+allows to configure the private access token and the base URL of a GitLab
+instance:
+
+```yaml
+scaffolder:
  gitlab:
    api:
      baseUrl: https://gitlab.com
@@ -82,7 +82,9 @@ for companies to adopt. This involves (something like) the following work items.
 - “Solidify” work and “Mkdocs stabilization” work that has come out of our Q3
  end-to-end work.
 - Improve/simplify the get up and running process.
- Introduce doc template Software Templates.
+- Introduce new documentation templates.
+- Extend the already existing docs-template to have options of different
+  documentation types.
 - Enable companies to choose their own storage (S3 for example).
 - Enable companies to choose their own source code hosting provider (GitHub,
  GitLab, and so).
@@ -23,8 +23,8 @@ Requests towards this repo.

 Backstage provides the `@backstage/create-app` package to scaffold standalone
 instances of Backstage. You will need to have
-[NodeJS](https://nodejs.org/en/download/) Active LTS Release installed
-(currently v12), [yarn](https://classic.yarnpkg.com/en/docs/install) and
+[Node.js](https://nodejs.org/en/download/) Active LTS Release installed
+(currently v12), [Yarn](https://classic.yarnpkg.com/en/docs/install) and
 [Python](https://www.python.org/downloads/) (although you likely have it
 already). You will also need to have
 [Docker](https://docs.docker.com/engine/install/) installed to use some features
@@ -38,8 +38,8 @@ npx @backstage/create-app
 ```

 You will be taken through a wizard to create your app, and the output should
-look something like this. You can read more about this process
-[here](https://backstage.io/docs/getting-started/create-an-app).
+look something like this. You can read more about this process in
+[Create an app](https://backstage.io/docs/getting-started/create-an-app).

 ### Contributing to Backstage

@@ -8,7 +8,7 @@ description: Documentation on How to run Backstage Locally

 - Node.js

-First make sure you are using NodeJS with an Active LTS Release, currently v12.
+First make sure you are using Node.js with an Active LTS Release, currently v12.
 This is made easy with a version manager such as
 [nvm](https://github.com/nvm-sh/nvm) which allows for version switching.

@@ -23,10 +23,10 @@ node --version
 > v12.18.3
 ```

- yarn
+- Yarn

 Please refer to the
-[installation instructions for yarn](https://classic.yarnpkg.com/en/docs/install/).
+[installation instructions for Yarn](https://classic.yarnpkg.com/en/docs/install/).

 - Docker

@@ -48,7 +48,7 @@ management. [[live demo](https://backstage-demo.roadie.io/)]
 ![UI with different components highlighted](../assets/architecture-overview/core-vs-plugin-components-highlighted.png)

 Each plugin typically makes itself available in the UI on a dedicated URL. For
-example, the lighthouse plugin is registered with the UI on `/lighthouse`.
+example, the Lighthouse plugin is registered with the UI on `/lighthouse`.
 [[live demo](https://backstage-demo.roadie.io/lighthouse)]

 ![The lighthouse plugin UI](../assets/architecture-overview/lighthouse-plugin.png)
@@ -76,7 +76,7 @@ Plugins can be enabled, and passed configuration in `apis.ts`. For example,
 [here](https://github.com/spotify/backstage/blob/master/packages/app/src/apis.ts)
 is that file in the Backstage sample app.

-This is how the lighthouse plugin would be enabled in a typical Backstage
+This is how the Lighthouse plugin would be enabled in a typical Backstage
 application:

 ```tsx
@@ -108,7 +108,7 @@ Architecturally, plugins can take three forms:
 #### Standalone plugins

 Standalone plugins run entirely in the browser.
-[The tech radar plugin](https://backstage-demo.roadie.io/tech-radar), for
+[The Tech Radar plugin](https://backstage-demo.roadie.io/tech-radar), for
 example, simply renders hard-coded information. It doesn't make any API requests
 to other services.

@@ -124,9 +124,9 @@ simple.
 Service backed plugins make API requests to a service which is within the
 purview of the organisation running Backstage.

-The lighthouse plugin, for example, makes requests to the
+The Lighthouse plugin, for example, makes requests to the
 [lighthouse-audit-service](https://github.com/spotify/lighthouse-audit-service).
-The lighthouse-audit-service is a microservice which runs a copy of Google's
+The `lighthouse-audit-service` is a microservice which runs a copy of Google's
 [Lighthouse library](https://github.com/GoogleChrome/lighthouse/) and stores the
 results in a PostgreSQL database.

@@ -144,11 +144,11 @@ Third-party backed plugins are similar to service backed plugins. The main
 difference is that the service which backs the plugin is hosted outside of the
 ecosystem of the company hosting Backstage.

-The Circle CI plugin is an example of a third-party backed plugin. Circle CI is
-a SaaS service which can be used without any knowledge of Backstage. It has an
-API which a Backstage plugin consumes to display content.
+The CircleCI plugin is an example of a third-party backed plugin. CircleCI is a
+SaaS service which can be used without any knowledge of Backstage. It has an API
+which a Backstage plugin consumes to display content.

-Requests which go to Circle CI from the users browser are passed through a proxy
+Requests which go to CircleCI from the users browser are passed through a proxy
 service that Backstage provides. Without this, the requests would be blocked by
 Cross Origin Resource Sharing policies which prevent a browser page served at
 [https://example.com](https://example.com) from serving resources hosted at
@@ -180,7 +180,7 @@ separate docker images.

 1. The frontend container
 2. The backend container
-3. The lighthouse audit service container
+3. The Lighthouse audit service container

 ![Boxes around the architecture to indicate how it is containerised](../assets/architecture-overview/containerised.png)

@@ -204,7 +204,7 @@ yarn run docker-build
 This will create a container called `example-backend`.

 The lighthouse-audit-service container is already publicly available in Docker
-Hub and can be downloaded and ran with
+Hub and can be downloaded and run with

 ```bash
 docker run spotify/lighthouse-audit-service:latest
@@ -73,7 +73,7 @@ export const plugin = createPlugin({
 ```

 This is where the plugin is created and where it hooks into the app by declaring
-what component should be shown on what url. See reference docs for
+what component should be shown on what URL. See reference docs for
 [createPlugin](../reference/createPlugin.md) or
 [router](../reference/createPlugin-router.md).

@@ -101,11 +101,11 @@ Backstage CLI.

 ## Talking to the outside world

-If your plugin needs to communicate with services outside the backstage
+If your plugin needs to communicate with services outside the Backstage
 environment you will probably face challenges like CORS policies and/or
 backend-side authorization. To smooth this process out you can use proxy -
-either the one you already have (like nginx/haproxy/etc) or the proxy-backend
-plugin that we provide for the backstage backend.
+either the one you already have (like Nginx, HAProxy, etc.) or the proxy-backend
+plugin that we provide for the Backstage backend.
 [Read more](https://github.com/spotify/backstage/blob/master/plugins/proxy-backend/README.md)

 [Back to Getting Started](../README.md)
@@ -71,38 +71,38 @@ Let's look at them individually.

 ### `packages/`

-These are all the packages that is we use within the project.
-[Plugins](#plugins) are separated out into their own folder, see further down.
+These are all the packages that we use within the project. [Plugins](#plugins)
+are separated out into their own folder, see further down.

 - [`app/`](https://github.com/spotify/backstage/tree/master/packages/app) - This
  is our take on how an App could look like, bringing together a set of packages
  and plugins into a working Backstage App. This is not a published package, and
-  the main goals is to provide a demo of what an App could look like, and also
-  enabling local development.
+  the main goals are to provide a demo of what an App could look like and to
+  enable local development.

 - [`backend/`](https://github.com/spotify/backstage/tree/master/packages/backend) -
-  Every standalone backstage project will have both an `app` _and_ a `backend`
+  Every standalone Backstage project will have both an `app` _and_ a `backend`
  package. The `backend` uses plugins to construct a working backend that the
  frontend (`app`) can use.

 - [`backend-common/`](https://github.com/spotify/backstage/tree/master/packages/backend-common) -
  There are no "core" packages in the backend. Instead we have `backend-common`
-  which contains helper middleware´s and other utils.
+  which contains helper middleware and other utils.

 - [`catalog-model/`](https://github.com/spotify/backstage/tree/master/packages/catalog-model) -
-  You can considers this to be a library for working with the catalog of sorts.
+  You can consider this to be a library for working with the catalog of sorts.
  It contains the definition of an
  [Entity](https://backstage.io/docs/features/software-catalog/references#docsNav),
-  as well as validation an other logic related to it. This package can be used
+  as well as validation and other logic related to it. This package can be used
  in both the frontend and the backend.

 - [`cli/`](https://github.com/spotify/backstage/tree/master/packages/cli) - One
  of the biggest packages in our project, the `cli` is used to build, serve,
  diff, create-plugins and more. In the early days of this project, we started
-  out with calling tools directly - such as `eslint` - through package.json. But
-  as it was tricky to have a good development experience around that when we
+  out with calling tools directly - such as `eslint` - through `package.json`.
+  But as it was tricky to have a good development experience around that when we
  change named tooling, we opted for wrapping those in our own cli. That way
-  everything looks the same in package.json. Much like
+  everything looks the same in `package.json`. Much like
  [react-scripts](https://github.com/facebook/create-react-app/tree/master/packages/react-scripts).

 - [`cli-common/`](https://github.com/spotify/backstage/tree/master/packages/cli-common) -
@@ -119,20 +119,21 @@ These are all the packages that is we use within the project.

 * [`config-loader/`](https://github.com/spotify/backstage/tree/master/packages/config-loader) -
  This package is used to read config objects. It does not know how to merge,
-  this only reads files and passes them on to the config. As this part os only
+  but only reads files and passes them on to the config. As this part is only
  used by the backend, we chose to separate `config` and `config-loader` into
  two different packages.

 - [`core/`](https://github.com/spotify/backstage/tree/master/packages/core) -
  This package contains our visual React components, some of which you can find
-  [here](https://backstage.io/storybook/?path=/story/plugins-examples--plugin-with-data).
+  in
+  [plugin examples](https://backstage.io/storybook/?path=/story/plugins-examples--plugin-with-data).
  Apart from that it re-exports everything from [`core-api`] so that users only
  need to rely on one package.

 * [`core-api/`](https://github.com/spotify/backstage/tree/master/packages/core-api) -
-  This package contains apis and definitions of such. It is it's own package
+  This package contains APIs and definitions of such. It is it's own package
  because we needed to split our `test-utils` package. It's an implementation
-  detail that we try to hide from our users, and no-one should have to depend on
+  detail that we try to hide from our users, and no one should have to depend on
  it directly.

 * [`test-utils/`](https://github.com/spotify/backstage/tree/master/packages/test-utils) -
@@ -140,7 +141,7 @@ These are all the packages that is we use within the project.
  `core-api`.

 * [`test-utils-core/`](https://github.com/spotify/backstage/tree/master/packages/test-utils-core) -
-  This package contains more general purpose testing facilities for testing an
+  This package contains more general purpose testing facilities for testing a
  Backstage App.

 * [`create-app/`](https://github.com/spotify/backstage/tree/master/packages/create-app) -
@@ -181,7 +182,7 @@ These are all the packages that is we use within the project.

 ### `plugins/`

-Most of the functionality of an Backstage App comes from plugins. Even core
+Most of the functionality of a Backstage App comes from plugins. Even core
 features can be plugins, take the
 [catalog](https://github.com/spotify/backstage/tree/master/plugins/catalog) as
 an example.
@@ -199,7 +200,7 @@ through the

 ## Packages outside of the monorepo

-For convenience we include packages in our project that is not part of our
+For convenience we include packages in our project that are not part of our
 monorepo setup.

 - [`microsite/`](https://github.com/spotify/backstage/blob/master/microsite) -
@@ -220,20 +221,17 @@ future.
  that this folder use the public registry.

 - [`.vale.ini`](https://github.com/spotify/backstage/tree/master/.vale.ini) -
-  [Spell checker](https://github.com/errata-ai/vale) for markdown files
+  [Spell checker](https://github.com/errata-ai/vale) for Markdown files.

 - [`.yarnrc`](https://github.com/spotify/backstage/tree/master/.yarnrc) -
-  Enforces "our" version of yarn.
+  Enforces "our" version of Yarn.

 - [`app-config.yaml`](https://github.com/spotify/backstage/tree/master/app-config.yaml) -
-  Configuration for the app, both frontend and backend
-
- [`app-config.development.yaml`](https://github.com/spotify/backstage/tree/master/app-config.development.yaml) -
-  Used for overriding configuration when developing locally.
+  Configuration for the app, both frontend and backend.

 - [`catalog-info.yaml`](https://github.com/spotify/backstage/tree/master/catalog-info.yaml) -
  Description of Backstage in the Backstage Entity format.

 - [`lerna.json`](https://github.com/spotify/backstage/tree/master/lerna.json) -
-  [lerna](https://github.com/lerna/lerna) monorepo config. We are using
+  [Lerna](https://github.com/lerna/lerna) monorepo config. We are using
  `yarn workspaces`, so this will only be used for executing scripts.
@@ -179,7 +179,7 @@ const onSave = async () => {
 ```

 Now it's much simpler for users to change the theme tune, as they no longer need
-to go look up a track ID and edit a yaml file. Instead, they can now stay inside
+to go look up a track ID and edit a YAML file. Instead, they can now stay inside
 Backstage and search for the track and request the change from there. In
 addition, the requested change can be reviewed by the regular process of each
 organization.
@@ -11,7 +11,7 @@ title: Monorepo App Setup With Authentication
 > own environment. It starts with a skeleton install and verifying of the
 > monorepo's functionality. Next, GitHub authentication is added and tested.
 >
-> This document assumes you have NodeJS 12 active along with Yarn and Python.
+> This document assumes you have Node.js 12 active along with Yarn and Python.
 > Please note, that at the time of this writing, the current version is
 > 0.1.1-alpha.21. This guide can still be used with future versions, just,
 > verify as you go. If you run into issues, you can compare your setup with mine
@@ -20,7 +20,7 @@ title: Adding Custom Plugin to Existing Monorepo App
 > functionality, extend the Sidebar to make our life easy. Finally, we add
 > custom code to display GitHub repository information.
 >
-> This document assumes you have NodeJS 12 active along with Yarn and Python.
+> This document assumes you have Node.js 12 active along with Yarn and Python.
 > Please note, that at the time of this writing, the current version is
 > 0.1.1-alpha.21. This guide can still be used with future versions, just,
 > verify as you go. If you run into issues, you can compare your setup with mine
@@ -161,7 +161,7 @@ export default ExampleFetchComponent;

 # The Graph Model

-GitHub has a graphql API available for interacting. Let's start by adding our
+GitHub has a GraphQL API available for interacting. Let's start by adding our
 basic repository query

 1. Add the query const statement outside ExampleFetchComponent
@@ -0,0 +1,79 @@
+---
+title: New Cost Insights plugin: The engineer’s solution to taming cloud costs
+author: Janisa Anandamohan
+authorURL: https://twitter.com/janisa_a
+---
+
+How did Spotify save millions on cloud costs within a matter of months?? We made cost optimization just another part of the daily development process. Our newly open sourced [Cost Insights plugin](https://github.com/spotify/backstage/tree/master/plugins/cost-insights) makes a team’s cloud costs visible — and actionable — right inside Backstage. So engineers can see the impact of their cloud usage (down to a product and resource level) and make optimizations wherever and whenever it makes sense. By managing cloud costs from the ground up, you can make smarter decisions that let you continue to build and scale quickly, without wasting resources.
+
+<iframe width="780" height="440" src="https://www.youtube.com/embed/YLAd5hdXR_Q" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
+Are we turning engineers into accountants? Nope, we’re just letting engineers do what they do best, in the place that feels natural to them: inside Backstage.
+
+<!--truncate-->
+
+## Why put a cost management tool in the hands of engineers?
+
+Engineers are closest to the metal in terms of knowing why a specific feature, product, or service is using cloud resources. So they’re in the best position to understand how costs impact ongoing development (and vice versa).
+
+If you manage costs top-down from a 10,000-foot view of your cloud infrastructure, you’re likely making decisions far removed from products, especially in larger organizations. Set a broad cost-cutting goal, and you could be creating unintended consequences — curtailing spending at the expense of growth or experimentation.
+
+## Ground-level intelligence, data-driven solutions
+
+Our hypothesis at Spotify was, if you bring spending data into an engineer’s everyday development workflow, they’ll naturally look for cost optimizations just like they look for any other optimization. And the cost optimizations will be more efficient and effective, because the decisions are informed at the ground level.
+
+The problem is that most cloud platforms don’t provide cost data at a granular enough level to make those decisions. And the bigger your organization (say, two-thousand-microservices and four-thousand-data-pipelines big, like Spotify), then the less you can attribute these large, fuzzy numbers to the right team, let alone a shipping product or internal service.
+
+That’s where Cost Insights comes in. Instead of making cost management and product development separate departments on the org chart, Backstage brings them together — with a level of detail and specificity engineers relate and respond to.
+
+## How to turn dollars into sense
+
+It’s not enough to make costs visible. To be useful, the numbers need to be relevant, relatable, and actionable. In other words, not just cost information, but insights. There are several ways the plugin puts data from your cloud provider in a more helpful context.
+
+### Use business metrics to evaluate costs
+
+Cost Insights will show you trends at a glance and also let you compare costs quarter over quarter. More importantly, you can also evaluate costs against business metrics that you care most about. In the example below, should the upward slope shown in the first screen be cause for worry? Perhaps not — if you switch views, you’ll see that cost per daily average user (DAU) is actually going down. Exactly what you hope to see as you scale.
+
+![Comparing costs to DAU](assets/20-10-22/cost-insights-1-dau.gif)
+_(Note: Screens are examples; they do not show real data.)_
+
+### Illustrate costs with relatable, real-world comparisons
+
+In addition to dollar amounts, Cost Insights allows teams to visualize and convert cost overages into more relatable terms. In the example below, we equate the growth in costs for virtual machine instances (100% increase) to developer time spent (about 1 engineer). We use this particular comparison in the plugin because we found it resonated with our own engineers — providing a useful perspective for spending increases. You can configure what the “cost of an engineer” means to your organization. Or engineers can build in their own comparisons — cups of coffee, carbon offset credits, electric luxury vehicles — whatever makes costs more tangible for them.
+
+![Cost growth as engineering time](assets/20-10-22/cost-insights-2-engineer.png)
+_(Note: Screens are examples; they do not show real data.)_
+
+### Tie spending to specific products and resources
+
+The more detailed the cost data, the more relevant, actionable, and helpful it is. Cost Insights allows you to attribute costs to products and resources in a way that makes sense to your engineers. For example, here we see a breakdown of data processing costs by individual pipelines. This allows your team to target optimizations more precisely.
+
+![Data Processing costs by pipeline](assets/20-10-22/cost-insights-3-data.png)
+_(Note: Screens are examples; they do not show real data.)_
+
+## Driving down costs without slowing down development
+
+When it comes to cutting costs, we actually want to guard against over-optimization. Growth and costs can go hand in hand. The trick is knowing when one is out of balance and needs addressing. Our product highlights when there’s been a large increase in spending, so that engineers are thinking about cost only when they must and aren’t distracted from their set goals and priorities.
+
+Engineers can then determine for themselves if the time invested in an optimization was valuable compared to the costs saved. Cost Insights puts the decision in our engineers’ hands for them to choose when to focus on growth efforts and when to focus on cost. Control, as ever, remains with our developers, where we think it belongs.
+
+## Getting started
+
+You can begin working with the Cost Insights plugin today on [GitHub](https://github.com/spotify/backstage/tree/master/plugins/cost-insights). We include an example client with static data in the expected format. The `CostInsightsApi` should talk with a cloud billing backend that aggregates billing data from your cloud provider.
+
+The current release of Cost Insights includes:
+
+- Daily cost graph by team or billing account
+- Cost comparisons against configurable business metrics (including an option for Daily Active Users)
+- Insights panels — configurable for the cloud products your company uses
+- Cost alerts and recommendations
+- Selectable time periods for month-over-month or quarter-over-quarter comparisons
+- Conversion of cost growth into “cost of average engineer” to help optimization trade-off decisions
+
+Our hope is to help other companies translate their cloud cost in a relatable way for their engineers to better understand their impact and accurately identify their opportunities for optimizations.
+
+And if you’re interested in contributing to our outstanding issues, you can find them in the issues queue, filtered under the [‘cost-insights’ label](https://github.com/spotify/backstage/labels/cost-insights).
+
+## Ready for DevSecCostOpsPlus (and whatever’s next)
+
+There’s DevOps, there’s DevSecOps, and then there’s Backstage: one frontend for all your infrastructure. From building, testing, and deploying to monitoring and security — Backstage helps you manage your entire tech organization and provides a seamless developer experience for engineers, from end to end to end. And now that also extends to cost management for your cloud infrastructure and tooling. Happy building and [happy optimizing](https://github.com/spotify/backstage/tree/master/plugins/cost-insights).
@@ -0,0 +1,9 @@
+---
+title: Security Insights
+author: roadie.io
+authorUrl: https://roadie.io/
+category: Security
+description: View Security Insights for your components in Backstage.
+documentation: https://roadie.io/backstage/plugins/security-insights
+iconUrl: https://roadie.io/static/7f13bb8d861d8dedc5112fb939d215f9/351f2/GitHub-Mark-Light-120px-plus.png
+npmPackageName: '@roadiehq/backstage-plugin-security-insights'
@@ -78,7 +78,41 @@ const Background = props => {
        </Block.Container>
      </Block>

-      <Block className="stripe-bottom bg-black-grey">
+      <Block className="stripe bg-black-grey">
+        <Block.Container style={{ justifyContent: 'flex-start' }}>
+          <Block.TextBox>
+            <Block.Title>Control cloud costs</Block.Title>
+            <Block.Paragraph>
+              How do you control cloud costs while maintaining the speed and
+              independence of your development teams? With the{' '}
+              <a href="https://backstage.io/plugins">Cost Insights plugin</a>{' '}
+              for Backstage, managing cloud costs becomes just another part of
+              an engineer’s daily development process. They get a clear view of
+              their spending — and can decide for themselves how they want to
+              optimize it. Learn more about the{' '}
+              <a href="https://backstage.io/blog/2020/10/22/cost-insights-plugin">
+                Cost Insights plugin
+              </a>
+              .
+            </Block.Paragraph>
+            <Block.LinkButton href="https://youtu.be/YLAd5hdXR_Q">
+              Watch now
+            </Block.LinkButton>
+          </Block.TextBox>
+          <Block.MediaFrame>
+            <iframe
+              width="560"
+              height="315"
+              src="https://www.youtube.com/embed/YLAd5hdXR_Q"
+              frameborder="0"
+              allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture"
+              allowfullscreen
+            ></iframe>
+          </Block.MediaFrame>
+        </Block.Container>
+      </Block>
+
+      <Block className="stripe bg-black">
        <Block.Container style={{ justifyContent: 'flex-start' }}>
          <Block.TextBox>
            <Block.Title id="techdocs-demo">
@@ -96,7 +130,7 @@ const Background = props => {
              </a>
              .
            </Block.Paragraph>
-            <Block.LinkButton href={'https://youtu.be/mOLCgdPw1iA'}>
+            <Block.LinkButton href="https://youtu.be/mOLCgdPw1iA">
              Watch now
            </Block.LinkButton>
          </Block.TextBox>
@@ -16,9 +16,8 @@ const {
 const pluginsDirectory = require('path').join(process.cwd(), 'data/plugins');
 const pluginMetadata = fs
  .readdirSync(pluginsDirectory)
-  .map(file =>
-    yaml.safeLoad(fs.readFileSync(`./data/plugins/${file}`, 'utf8')),
-  );
+  .map(file => yaml.safeLoad(fs.readFileSync(`./data/plugins/${file}`, 'utf8')))
+  .sort((a, b) => a.title.toLowerCase().localeCompare(b.title.toLowerCase()));
 const truncate = text =>
  text.length > 170 ? text.substr(0, 170) + '...' : text;

@@ -23,6 +23,7 @@ import {
  Cost,
  CostInsightsApi,
  DateAggregation,
+  DEFAULT_DATE_FORMAT,
  Duration,
  exclusiveEndDateOf,
  Group,
@@ -38,18 +39,32 @@ import {
  UnlabeledDataflowData,
 } from '@backstage/plugin-cost-insights';

-function durationOf(intervals: string): Duration {
-  const match = intervals.match(/\/(?<duration>P\d+[DM])\//);
-  const { duration } = match!.groups!;
-  return duration as Duration;
+type IntervalFields = {
+  duration: Duration;
+  endDate: string;
+};
+
+function parseIntervals(intervals: string): IntervalFields {
+  const match = intervals.match(
+    /\/(?<duration>P\d+[DM])\/(?<date>\d{4}-\d{2}-\d{2})/,
+  );
+  if (Object.keys(match?.groups || {}).length !== 2) {
+    throw new Error(`Invalid intervals: ${intervals}`);
+  }
+  const { duration, date } = match!.groups!;
+  return {
+    duration: duration as Duration,
+    endDate: date,
+  };
 }

 function aggregationFor(
-  duration: Duration,
+  intervals: string,
  baseline: number,
 ): DateAggregation[] {
-  const days = dayjs(exclusiveEndDateOf(duration)).diff(
-    inclusiveStartDateOf(duration),
+  const { duration, endDate } = parseIntervals(intervals);
+  const days = dayjs(exclusiveEndDateOf(duration, endDate)).diff(
+    inclusiveStartDateOf(duration, endDate),
    'day',
  );

@@ -57,10 +72,10 @@ function aggregationFor(
    (values: DateAggregation[], i: number): DateAggregation[] => {
      const last = values.length ? values[values.length - 1].amount : baseline;
      values.push({
-        date: dayjs(inclusiveStartDateOf(duration))
+        date: dayjs(inclusiveStartDateOf(duration, endDate))
          .add(i, 'day')
-          .format('YYYY-MM-DD'),
-        amount: last + (baseline / 20) * (Math.random() * 2 - 1),
+          .format(DEFAULT_DATE_FORMAT),
+        amount: Math.max(0, last + (baseline / 20) * (Math.random() * 2 - 1)),
      });
      return values;
    },
@@ -99,6 +114,12 @@ export class ExampleCostInsightsClient implements CostInsightsApi {
    return new Promise(resolve => setTimeout(resolve, 0, res));
  }

+  getLastCompleteBillingDate(): Promise<string> {
+    return Promise.resolve(
+      dayjs().subtract(1, 'day').format(DEFAULT_DATE_FORMAT),
+    );
+  }
+
  async getUserGroups(userId: string): Promise<Group[]> {
    const groups: Group[] = await this.request({ userId }, [
      { id: 'pied-piper' },
@@ -121,10 +142,10 @@ export class ExampleCostInsightsClient implements CostInsightsApi {
    metric: string,
    intervals: string,
  ): Promise<MetricData> {
-    const aggregation = aggregationFor(
-      durationOf(intervals),
-      100_000,
-    ).map(entry => ({ ...entry, amount: Math.round(entry.amount) }));
+    const aggregation = aggregationFor(intervals, 100_000).map(entry => ({
+      ...entry,
+      amount: Math.round(entry.amount),
+    }));

    const cost: MetricData = await this.request(
      { metric, intervals },
@@ -140,7 +161,7 @@ export class ExampleCostInsightsClient implements CostInsightsApi {
  }

  async getGroupDailyCost(group: string, intervals: string): Promise<Cost> {
-    const aggregation = aggregationFor(durationOf(intervals), 8_000);
+    const aggregation = aggregationFor(intervals, 8_000);
    const groupDailyCost: Cost = await this.request(
      { group, intervals },
      {
@@ -154,7 +175,7 @@ export class ExampleCostInsightsClient implements CostInsightsApi {
  }

  async getProjectDailyCost(project: string, intervals: string): Promise<Cost> {
-    const aggregation = aggregationFor(durationOf(intervals), 1_500);
+    const aggregation = aggregationFor(intervals, 1_500);
    const projectDailyCost: Cost = await this.request(
      { project, intervals },
      {
@@ -257,8 +278,8 @@ export class ExampleCostInsightsClient implements CostInsightsApi {
  async getAlerts(group: string): Promise<Alert[]> {
    const projectGrowthData: ProjectGrowthData = {
      project: 'example-project',
-      periodStart: 'Q2 2020',
-      periodEnd: 'Q3 2020',
+      periodStart: '2020-Q2',
+      periodEnd: '2020-Q3',
      aggregation: [60_000, 120_000],
      change: {
        ratio: 1,
@@ -32,10 +32,12 @@
    "@backstage/cli-common": "^0.1.1-alpha.25",
    "@backstage/config": "^0.1.1-alpha.25",
    "@backstage/config-loader": "^0.1.1-alpha.25",
+    "@backstage/test-utils": "^0.1.1-alpha.25",
    "@types/cors": "^2.8.6",
    "@types/express": "^4.17.6",
    "compression": "^1.7.4",
    "cors": "^2.8.5",
+    "cross-fetch": "^3.0.6",
    "express": "^4.17.1",
    "express-prom-bundle": "^6.1.0",
    "express-promise-router": "^3.0.3",
@@ -44,8 +46,8 @@
    "knex": "^0.21.1",
    "lodash": "^4.17.15",
    "logform": "^2.1.1",
+    "minimist": "^1.2.5",
    "morgan": "^1.10.0",
-    "node-fetch": "^2.6.0",
    "prom-client": "^12.0.0",
    "selfsigned": "^1.10.7",
    "stoppable": "^1.1.0",
@@ -63,8 +65,8 @@
    "@backstage/cli": "^0.1.1-alpha.25",
    "@types/compression": "^1.7.0",
    "@types/http-errors": "^1.6.3",
+    "@types/minimist": "^1.2.0",
    "@types/morgan": "^1.9.0",
-    "@types/node-fetch": "^2.5.7",
    "@types/stoppable": "^1.1.0",
    "@types/supertest": "^2.0.8",
    "@types/webpack-env": "^1.15.2",
@@ -72,8 +74,7 @@
    "get-port": "^5.1.1",
    "http-errors": "^1.7.3",
    "jest": "^26.0.1",
-    "jest-fetch-mock": "^3.0.3",
-    "msw": "^0.20.5",
+    "msw": "^0.21.2",
    "supertest": "^4.0.2"
  },
  "files": [
@@ -14,19 +14,38 @@
 * limitations under the License.
 */

+import { resolve as resolvePath } from 'path';
+import parseArgs from 'minimist';
+import { Logger } from 'winston';
 import { findPaths } from '@backstage/cli-common';
+import { Config, ConfigReader } from '@backstage/config';
 import { loadConfig } from '@backstage/config-loader';

+type Options = {
+  logger: Logger;
+  // process.argv or any other overrides
+  argv: string[];
+};
+
 /**
 * Load configuration for a Backend
 */
-export async function loadBackendConfig() {
+export async function loadBackendConfig(options: Options): Promise<Config> {
+  const args = parseArgs(options.argv);
+  const configOpts: string[] = [args.config ?? []].flat();
+
  /* eslint-disable-next-line no-restricted-syntax */
  const paths = findPaths(__dirname);
  const configs = await loadConfig({
    env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development',
-    rootPaths: [paths.targetRoot, paths.targetDir],
+    configRoot: paths.targetRoot,
+    configPaths: configOpts.map(opt => resolvePath(opt)),
    shouldReadSecrets: true,
  });
-  return configs;
+
+  options.logger.info(
+    `Loaded config from ${configs.map(c => c.context).join(', ')}`,
+  );
+
+  return ConfigReader.fromConfigs(configs);
 }
@@ -19,14 +19,13 @@ import { setupServer } from 'msw/node';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '../logging';
 import { AzureUrlReader } from './AzureUrlReader';
+import { msw } from '@backstage/test-utils';

 const logger = getVoidLogger();

 describe('AzureUrlReader', () => {
  const worker = setupServer();
-
-  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
-  afterAll(() => worker.close());
+  msw.setupDefaultHandlers(worker);

  beforeEach(() => {
    worker.use(
@@ -41,7 +40,6 @@ describe('AzureUrlReader', () => {
      ),
    );
  });
-  afterEach(() => worker.resetHandlers());

  const createConfig = (token?: string) =>
    new ConfigReader(
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch';
+import fetch from 'cross-fetch';
 import { Config } from '@backstage/config';
 import { NotFoundError } from '../errors';
 import { ReaderFactory, UrlReader } from './types';
@@ -76,7 +76,7 @@ export class AzureUrlReader implements UrlReader {

    // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
    if (response.ok && response.status !== 203) {
-      return response.buffer();
+      return Buffer.from(await response.text());
    }

    const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
@@ -19,14 +19,14 @@ import { setupServer } from 'msw/node';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '../logging';
 import { BitbucketUrlReader } from './BitbucketUrlReader';
+import { msw } from '@backstage/test-utils';

 const logger = getVoidLogger();

 describe('BitbucketUrlReader', () => {
  const worker = setupServer();

-  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
-  afterAll(() => worker.close());
+  msw.setupDefaultHandlers(worker);

  beforeEach(() => {
    worker.use(
@@ -41,7 +41,6 @@ describe('BitbucketUrlReader', () => {
      ),
    );
  });
-  afterEach(() => worker.resetHandlers());

  const createConfig = (username?: string, appPassword?: string) =>
    new ConfigReader(
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch';
+import fetch from 'cross-fetch';
 import { Config } from '@backstage/config';
 import { ReaderFactory, UrlReader } from './types';
 import { NotFoundError } from '../errors';
@@ -84,7 +84,7 @@ export class BitbucketUrlReader implements UrlReader {
    }

    if (response.ok) {
-      return response.buffer();
+      return Buffer.from(await response.text());
    }

    const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import fetch, { Response } from 'node-fetch';
+import fetch from 'cross-fetch';
 import { NotFoundError } from '../errors';
 import { UrlReader } from './types';

@@ -31,7 +31,7 @@ export class FetchUrlReader implements UrlReader {
    }

    if (response.ok) {
-      return response.buffer();
+      return Buffer.from(await response.text());
    }

    const message = `could not read ${url}, ${response.status} ${response.statusText}`;
@@ -16,7 +16,7 @@

 import { Config } from '@backstage/config';
 import parseGitUri from 'git-url-parse';
-import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch';
+import fetch from 'cross-fetch';
 import { NotFoundError } from '../errors';
 import { ReaderFactory, UrlReader } from './types';

@@ -219,7 +219,7 @@ export class GithubUrlReader implements UrlReader {
    }

    if (response.ok) {
-      return response.buffer();
+      return Buffer.from(await response.text());
    }

    const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`;
@@ -19,14 +19,14 @@ import { setupServer } from 'msw/node';
 import { ConfigReader } from '@backstage/config';
 import { getVoidLogger } from '../logging';
 import { GitlabUrlReader } from './GitlabUrlReader';
+import { msw } from '@backstage/test-utils';

 const logger = getVoidLogger();

 describe('GitlabUrlReader', () => {
  const worker = setupServer();

-  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
-  afterAll(() => worker.close());
+  msw.setupDefaultHandlers(worker);

  beforeEach(() => {
    worker.use(
@@ -44,7 +44,6 @@ describe('GitlabUrlReader', () => {
      ),
    );
  });
-  afterEach(() => worker.resetHandlers());

  const createConfig = (token?: string) =>
    new ConfigReader(
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import fetch, { RequestInit, Response } from 'node-fetch';
+import fetch from 'cross-fetch';
 import { Config } from '@backstage/config';
 import { NotFoundError } from '../errors';
 import { ReaderFactory, UrlReader } from './types';
@@ -77,7 +77,7 @@ export class GitlabUrlReader implements UrlReader {
    }

    if (response.ok) {
-      return response.buffer();
+      return Buffer.from(await response.text());
    }

    const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
@@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import { ConfigReader } from '@backstage/config';
+import { Config } from '@backstage/config';
 import compression from 'compression';
 import cors from 'cors';
 import express, { Router } from 'express';
@@ -77,7 +77,7 @@ export class ServiceBuilderImpl implements ServiceBuilder {
    this.module = moduleRef;
  }

-  loadConfig(config: ConfigReader): ServiceBuilder {
+  loadConfig(config: Config): ServiceBuilder {
    const backendConfig = config.getOptionalConfig('backend');
    if (!backendConfig) {
      return this;
@@ -33,7 +33,7 @@ import {
  SingleHostDiscovery,
  UrlReaders,
 } from '@backstage/backend-common';
-import { ConfigReader } from '@backstage/config';
+import { Config } from '@backstage/config';
 import healthcheck from './plugins/healthcheck';
 import auth from './plugins/auth';
 import catalog from './plugins/catalog';
@@ -47,7 +47,7 @@ import graphql from './plugins/graphql';
 import app from './plugins/app';
 import { PluginEnvironment } from './types';

-function makeCreateEnv(config: ConfigReader) {
+function makeCreateEnv(config: Config) {
  const root = getRootLogger();
  const reader = UrlReaders.default({ logger: root, config });
  const discovery = SingleHostDiscovery.fromConfig(config);
@@ -64,9 +64,11 @@ function makeCreateEnv(config: ConfigReader) {
 }

 async function main() {
-  const configs = await loadBackendConfig();
-  const configReader = ConfigReader.fromConfigs(configs);
-  const createEnv = makeCreateEnv(configReader);
+  const config = await loadBackendConfig({
+    argv: process.argv,
+    logger: getRootLogger(),
+  });
+  const createEnv = makeCreateEnv(config);

  const healthcheckEnv = useHotMemoize(module, () => createEnv('healthcheck'));
  const catalogEnv = useHotMemoize(module, () => createEnv('catalog'));
@@ -93,7 +95,7 @@ async function main() {
  apiRouter.use(notFoundHandler());

  const service = createServiceBuilder(module)
-    .loadConfig(configReader)
+    .loadConfig(config)
    .addRouter('', await healthcheck(healthcheckEnv))
    .addRouter('/api', apiRouter)
    .addRouter('', await app(appEnv));
@@ -0,0 +1,9 @@
+apiVersion: backstage.io/v1alpha1
+kind: Location
+metadata:
+  name: acme-corp
+  description: A collection of all Backstage example Groups
+spec:
+  type: github
+  targets:
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/org.yaml
@@ -0,0 +1,11 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: backstage
+  description: The backstage sub-department
+spec:
+  type: sub-department
+  parent: infrastructure
+  ancestors: [infrastructure, acme-corp]
+  children: [team-a, team-b]
+  descendants: [team-a, team-b]
@@ -0,0 +1,11 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: boxoffice
+  description: The boxoffice sub-department
+spec:
+  type: sub-department
+  parent: infrastructure
+  ancestors: [infrastructure, acme-corp]
+  children: [team-c, team-d]
+  descendants: [team-c, team-d]
@@ -0,0 +1,11 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: infrastructure
+  description: The infra department
+spec:
+  type: department
+  parent: acme-corp
+  ancestors: [acme-corp]
+  children: [backstage, boxoffice]
+  descendants: [backstage, boxoffice, team-a, team-b, team-c, team-d]
@@ -0,0 +1,27 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: acme-corp
+  description: The acme-corp organization
+spec:
+  type: organization
+  ancestors: []
+  children: [infrastructure]
+  descendants:
+    [infrastructure, backstage, boxoffice, team-a, team-b, team-c, team-d]
+---
+apiVersion: backstage.io/v1alpha1
+kind: Location
+metadata:
+  name: example-groups
+  description: A collection of all Backstage example Groups
+spec:
+  type: github
+  targets:
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/infrastructure-group.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/boxoffice-group.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/backstage-group.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/team-a-group.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/team-b-group.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/team-c-group.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme/team-d-group.yaml
@@ -0,0 +1,44 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: team-a
+  description: Team A
+spec:
+  type: team
+  parent: backstage
+  ancestors: [backstage, infrastructure, acme-corp]
+  children: []
+  descendants: []
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: breanna.davison
+spec:
+  profile:
+    displayName: Breanna Davison
+    email: breanna-davison@example.com
+    picture: https://example.com/staff/breanna.jpeg
+  memberOf: [team-a]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: janelle.dawe
+spec:
+  profile:
+    displayName: Janelle Dawe
+    email: janelle-dawe@example.com
+    picture: https://example.com/staff/janelle.jpeg
+  memberOf: [team-a]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: nigel.manning
+spec:
+  profile:
+    displayName: Nigel Manning
+    email: nigel-manning@example.com
+    picture: https://example.com/staff/nigel.jpeg
+  memberOf: [team-a]
@@ -0,0 +1,66 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: team-b
+  description: Team B
+spec:
+  type: team
+  parent: backstage
+  ancestors: [backstage, infrastructure, acme-corp]
+  children: []
+  descendants: []
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: amelia.park
+spec:
+  profile:
+    displayName: Amelia Park
+    email: amelia-park@example.com
+    picture: https://example.com/staff/amelia.jpeg
+  memberOf: [team-b]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: colette.brock
+spec:
+  profile:
+    displayName: Colette Brock
+    email: colette-brock@example.com
+    picture: https://example.com/staff/colette.jpeg
+  memberOf: [team-b]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: jenny.doe
+spec:
+  profile:
+    displayName: Jenny Doe
+    email: jenny-doe@example.com
+    picture: https://example.com/staff/jenny.jpeg
+  memberOf: [team-b]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: jonathon.page
+spec:
+  profile:
+    displayName: Jonathon Page
+    email: jonathon-page@example.com
+    picture: https://example.com/staff/jonathon.jpeg
+  memberOf: [team-b]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: justine.barrow
+spec:
+  profile:
+    displayName: Justine Barrow
+    email: justine-barrow@example.com
+    picture: https://example.com/staff/justine.jpeg
+  memberOf: [team-b]
@@ -0,0 +1,66 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: team-c
+  description: Team C
+spec:
+  type: team
+  parent: boxoffice
+  ancestors: [boxoffice, infrastructure, acme-corp]
+  children: []
+  descendants: []
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: calum.leavy
+spec:
+  profile:
+    displayName: Calum Leavy
+    email: calum-leavy@example.com
+    picture: https://example.com/staff/calum.jpeg
+  memberOf: [team-c]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: frank.tiernan
+spec:
+  profile:
+    displayName: Frank Tiernan
+    email: frank-tiernan@example.com
+    picture: https://example.com/staff/frank.jpeg
+  memberOf: [team-c]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: peadar.macmahon
+spec:
+  profile:
+    displayName: Peadar MacMahon
+    email: peadar-macmahon@example.com
+    picture: https://example.com/staff/peadar.jpeg
+  memberOf: [team-c]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: sarah.gilroy
+spec:
+  profile:
+    displayName: Sarah Gilroy
+    email: sarah-gilroy@example.com
+    picture: https://example.com/staff/sarah.jpeg
+  memberOf: [team-c]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: tara.macgovern
+spec:
+  profile:
+    displayName: Tara MacGovern
+    email: tara-macgovern@example.com
+    picture: https://example.com/staff/tara.jpeg
+  memberOf: [team-c]
@@ -0,0 +1,33 @@
+apiVersion: backstage.io/v1alpha1
+kind: Group
+metadata:
+  name: team-d
+  description: Team D
+spec:
+  type: team
+  parent: boxoffice
+  ancestors: [boxoffice, infrastructure, acme-corp]
+  children: []
+  descendants: []
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: eva.macdowell
+spec:
+  profile:
+    displayName: Eva MacDowell
+    email: eva-macdowell@example.com
+    picture: https://example.com/staff/eva.jpeg
+  memberOf: [team-d]
+---
+apiVersion: backstage.io/v1alpha1
+kind: User
+metadata:
+  name: lucy.sheehan
+spec:
+  profile:
+    displayName: Lucy Sheehan
+    email: lucy-sheehan@example.com
+    picture: https://example.com/staff/lucy.jpeg
+  memberOf: [team-d]
@@ -6,7 +6,8 @@ metadata:
 spec:
  type: github
  targets:
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/hello-world-api.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/streetlights-api.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/spotify-api.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/swapi-graphql.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/apis/hello-world-api.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/apis/petstore-api.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/apis/spotify-api.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/apis/streetlights-api.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/apis/swapi-graphql.yaml
@@ -6,11 +6,12 @@ metadata:
 spec:
  type: github
  targets:
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/podcast-api-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/queue-proxy-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/searcher-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-lib-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/www-artist-component.yaml
-    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/shuffle-api-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/artist-lookup-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/petstore-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/playback-order-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/podcast-api-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/queue-proxy-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/searcher-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/playback-lib-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/www-artist-component.yaml
+    - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/components/shuffle-api-component.yaml
@@ -15,6 +15,7 @@
 */

 import { JsonObject } from '@backstage/config';
+import { EntityName } from '../types';

 /**
 * The format envelope that's common to all versions/kinds of entity.
@@ -42,6 +43,11 @@ export type Entity = {
   * The specification data describing the entity itself.
   */
  spec?: JsonObject;
+
+  /**
+   * The relations that this entity has with other entities.
+   */
+  relations?: EntityRelation[];
 };

 /**
@@ -120,3 +126,38 @@ export type EntityMeta = JsonObject & {
   */
  tags?: string[];
 };
+
+/**
+ * A relation of a specific type to another entity in the catalog.
+ */
+export type EntityRelation = {
+  /**
+   * The type of the relation.
+   */
+  type: string;
+
+  /**
+   * The target entity of this relation.
+   */
+  target: EntityName;
+};
+
+/**
+ * Holds the relation data for entities.
+ */
+export type EntityRelationSpec = {
+  /**
+   * The source entity of this relation.
+   */
+  source: EntityName;
+
+  /**
+   * The type of the relation.
+   */
+  type: string;
+
+  /**
+   * The target entity of this relation.
+   */
+  target: EntityName;
+};
@@ -18,7 +18,12 @@ export {
  ENTITY_DEFAULT_NAMESPACE,
  ENTITY_META_GENERATED_FIELDS,
 } from './constants';
-export type { Entity, EntityMeta } from './Entity';
+export type {
+  Entity,
+  EntityMeta,
+  EntityRelation,
+  EntityRelationSpec,
+} from './Entity';
 export * from './policies';
 export {
  getEntityName,
@@ -15,7 +15,12 @@
 */

 import { EntityPolicy } from '../../types';
-import { makeValidator, Validators } from '../../validation';
+import {
+  CommonValidatorFunctions,
+  KubernetesValidatorFunctions,
+  makeValidator,
+  Validators,
+} from '../../validation';
 import { Entity } from '../Entity';

 /**
@@ -50,7 +55,47 @@ export class FieldFormatEntityPolicy implements EntityPolicy {
      }

      if (!isValid) {
-        throw new Error(`${field} "${value}" is not valid`);
+        let expectation;
+        switch (
+          validator.name as
+            | keyof typeof KubernetesValidatorFunctions
+            | keyof typeof CommonValidatorFunctions
+        ) {
+          case 'isValidLabelValue':
+          case 'isValidObjectName':
+            expectation =
+              'a string that is sequences of [a-zA-Z0-9] separated by any of [-_.], at most 63 characters in total';
+            break;
+          case 'isValidLabelKey':
+          case 'isValidApiVersion':
+          case 'isValidAnnotationKey':
+            expectation = 'a valid prefix and/or suffix';
+            break;
+          case 'isValidNamespace':
+          case 'isValidDnsLabel':
+            expectation =
+              'a string that is sequences of [a-zA-Z0-9] separated by [-], at most 63 characters in total';
+            break;
+          case 'isValidAnnotationValue':
+            expectation = 'a string';
+            break;
+          case 'isValidKind':
+            expectation =
+              'a string that is a sequence of [a-zA-Z][a-z0-9A-Z], at most 63 characters in total';
+            break;
+          default:
+            expectation = undefined;
+            break;
+        }
+
+        // ensure that if there are other/future validators, the error message defaults to a general "is not valid, visit link"
+        const message = expectation
+          ? ` expected ${expectation} but found "${value}".`
+          : '';
+
+        throw new Error(
+          `"${field}" is not valid;${message} To learn more about catalog file format, visit: https://github.com/spotify/backstage/blob/master/docs/architecture-decisions/adr002-default-catalog-file-format.md`,
+        );
      }
    }

@@ -44,3 +44,4 @@ export type {
  UserEntityV1alpha1 as UserEntity,
  UserEntityV1alpha1,
 } from './UserEntityV1alpha1';
+export * from './relations';
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+Naming rules for relations in priority order:
+
+1. Use at most two words. One main verb and a specifier, e.g. "ownerOf"
+2. Reading out "<source-kind> <type> <target-kind>" should make sense in English.
+3. Maintain symmetry between pairs, e.g. "ownedBy" and "ownerOf" rather than "owns".
+*/
+
+/**
+ * An ownership relation where the owner is usually an organizational
+ * entity (user or group), and the other entity can be anything.
+ */
+export const RELATION_OWNED_BY = 'ownedBy';
+export const RELATION_OWNER_OF = 'ownerOf';
+
+/**
+ * A relation with an API entity, typically from a component or system
+ */
+export const RELATION_CONSUMES_API = 'consumesApi';
+export const RELATION_PROVIDES_API = 'providesApi';
+
+/**
+ * A relation denoting a dependency on another entity.
+ */
+export const RELATION_DEPENDS_ON = 'dependsOn';
+export const RELATION_DEPENDENCY_OF = 'dependencyOf';
+
+/**
+ * A parent/child relation to build up a tree, used for example to describe
+ * the organizational structure between groups.
+ */
+export const RELATION_PARENT_OF = 'parentOf';
+export const RELATION_CHILD_OF = 'childOf';
+
+/**
+ * A membership relation, typically for users in a group.
+ */
+export const RELATION_MEMBER_OF = 'memberOf';
+export const RELATION_HAS_MEMBER = 'hasMember';
@@ -18,8 +18,9 @@ export type LocationSpec = {
  type: string;
  target: string;
  // When using repo importer plugin, location is being created before the component yaml file is merged to the main branch.
-  // This flag is then set to disable validation that prevents creation of location if target file does not yet exist.
-  pendingLocation?: boolean;
+  // This flag is then set to indicate that the file can be not present.
+  // default value: 'required'.
+  presence?: 'optional' | 'required';
 };

 export type Location = {
@@ -21,7 +21,7 @@ export const locationSpecSchema = yup
  .object<LocationSpec>({
    type: yup.string().required(),
    target: yup.string().required(),
-    pendingLocation: yup.bool(),
+    presence: yup.string(),
  })
  .noUnknown()
  .required();
@@ -36,7 +36,7 @@
    "@lerna/project": "^3.18.0",
    "@rollup/plugin-commonjs": "^13.0.0",
    "@rollup/plugin-json": "^4.0.2",
-    "@rollup/plugin-node-resolve": "^8.1.0",
+    "@rollup/plugin-node-resolve": "^9.0.0",
    "@rollup/plugin-yaml": "^2.1.1",
    "@spotify/eslint-config-base": "^8.0.0",
    "@spotify/eslint-config-react": "^8.0.0",
@@ -114,6 +114,7 @@
    "@types/http-proxy": "^1.17.4",
    "@types/inquirer": "^7.3.1",
    "@types/mini-css-extract-plugin": "^0.9.1",
+    "@types/mock-fs": "^4.13.0",
    "@types/node": "^13.7.2",
    "@types/ora": "^3.2.0",
    "@types/react-dev-utils": "^9.0.4",
@@ -15,22 +15,15 @@
 */

 import { Command } from 'commander';
-import { loadConfig } from '@backstage/config-loader';
-import { ConfigReader } from '@backstage/config';
-import { paths } from '../../lib/paths';
 import { buildBundle } from '../../lib/bundler';
 import { parseParallel, PARALLEL_ENV_VAR } from '../../lib/parallel';
+import { loadCliConfig } from '../../lib/config';

 export default async (cmd: Command) => {
-  const appConfigs = await loadConfig({
-    env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production',
-    rootPaths: [paths.targetRoot, paths.targetDir],
-  });
  await buildBundle({
    entry: 'src/index',
    parallel: parseParallel(process.env[PARALLEL_ENV_VAR]),
    statsJsonEnabled: cmd.stats,
-    config: ConfigReader.fromConfigs(appConfigs),
-    appConfigs,
+    ...(await loadCliConfig(cmd.config)),
  });
 };
@@ -15,21 +15,14 @@
 */

 import { Command } from 'commander';
-import { loadConfig } from '@backstage/config-loader';
-import { ConfigReader } from '@backstage/config';
-import { paths } from '../../lib/paths';
 import { serveBundle } from '../../lib/bundler';
+import { loadCliConfig } from '../../lib/config';

 export default async (cmd: Command) => {
-  const appConfigs = await loadConfig({
-    env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development',
-    rootPaths: [paths.targetRoot, paths.targetDir],
-  });
  const waitForExit = await serveBundle({
    entry: 'src/index',
    checksEnabled: cmd.check,
-    config: ConfigReader.fromConfigs(appConfigs),
-    appConfigs,
+    ...(await loadCliConfig(cmd.config)),
  });

  await waitForExit();
@@ -14,24 +14,14 @@
 * limitations under the License.
 */

-import { ConfigReader } from '@backstage/config';
-import { loadConfig } from '@backstage/config-loader';
 import { Command } from 'commander';
-import { paths } from '../../lib/paths';
 import { serveBackend } from '../../lib/bundler/backend';

 export default async (cmd: Command) => {
-  const appConfigs = await loadConfig({
-    env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development',
-    rootPaths: [paths.targetRoot, paths.targetDir],
-  });
-
  const waitForExit = await serveBackend({
    entry: 'src/index',
    checksEnabled: cmd.check,
    inspectEnabled: cmd.inspect,
-    config: ConfigReader.fromConfigs(appConfigs),
-    appConfigs,
  });

  await waitForExit();
@@ -15,20 +15,13 @@
 */

 import { Command } from 'commander';
-import { loadConfig } from '@backstage/config-loader';
-import { ConfigReader } from '@backstage/config';
-import { paths } from '../../lib/paths';
 import { stringify as stringifyYaml } from 'yaml';
+import { loadCliConfig } from '../../lib/config';

 export default async (cmd: Command) => {
-  const appConfigs = await loadConfig({
-    env:
-      cmd.env ?? process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development',
-    shouldReadSecrets: cmd.withSecrets ?? false,
-    rootPaths: [paths.targetRoot, paths.targetDir],
-  });
+  const { config } = await loadCliConfig(cmd.config, cmd.withSecrets ?? false);

-  const flatConfig = ConfigReader.fromConfigs(appConfigs).get();
+  const flatConfig = config.get();

  if (cmd.format === 'json') {
    process.stdout.write(`${JSON.stringify(flatConfig, null, 2)}\n`);
@@ -16,14 +16,20 @@

 import fs from 'fs-extra';
 import path from 'path';
+import mockFs from 'mock-fs';
 import os from 'os';
 import del from 'del';
 import { createTemporaryPluginFolder, movePlugin } from './createPlugin';

+const id = 'testPluginMock';
+
 describe('createPlugin', () => {
+  afterAll(() => {
+    mockFs.restore();
+  });
+
  describe('createPluginFolder', () => {
    it('should create a temporary plugin directory in the correct place', async () => {
-      const id = 'testPlugin';
      const tempDir = path.join(os.tmpdir(), id);
      try {
        await createTemporaryPluginFolder(tempDir);
@@ -35,35 +41,27 @@ describe('createPlugin', () => {
    });

    it('should not create a temporary plugin directory if it already exists', async () => {
-      const id = 'testPlugin';
-      const tempDir = path.join(os.tmpdir(), id);
-      try {
-        await createTemporaryPluginFolder(tempDir);
-        await expect(fs.pathExists(tempDir)).resolves.toBe(true);
-        await expect(createTemporaryPluginFolder(tempDir)).rejects.toThrow(
-          /Failed to create temporary plugin directory/,
-        );
-      } finally {
-        await del(tempDir, { force: true });
-      }
+      mockFs({
+        [id]: {},
+      });
+
+      await expect(createTemporaryPluginFolder(id)).rejects.toThrow(
+        /Failed to create temporary plugin directory/,
+      );
    });
  });

  describe('movePlugin', () => {
    it('should move the temporary plugin directory to its final place', async () => {
-      const id = 'testPlugin';
-      const tempDir = path.join(os.tmpdir(), id);
-      const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test-'));
-      const pluginDir = path.join(rootDir, 'plugins', id);
-      try {
-        await createTemporaryPluginFolder(tempDir);
-        await movePlugin(tempDir, pluginDir, id);
-        await expect(fs.pathExists(pluginDir)).resolves.toBe(true);
-        expect(pluginDir).toMatch(path.join('', 'plugins', id));
-      } finally {
-        await del(tempDir, { force: true });
-        await del(rootDir, { force: true });
-      }
+      mockFs({
+        [id]: {},
+      });
+      const tempDir = id;
+      const pluginDir = path.join('test-temp', 'plugins', id);
+
+      await movePlugin(tempDir, pluginDir, id);
+      await expect(fs.pathExists(pluginDir)).resolves.toBe(true);
+      expect(pluginDir).toMatch(path.join('', 'plugins', id));
    });
  });
 });
@@ -18,16 +18,25 @@ import { CommanderStatic } from 'commander';
 import { exitWithError } from '../lib/errors';

 export function registerCommands(program: CommanderStatic) {
+  const configOption = [
+    '--config <path>',
+    'Config files to load instead of app-config.yaml',
+    (opt: string, opts: string[]) => [...opts, opt],
+    Array<string>(),
+  ] as const;
+
  program
    .command('app:build')
    .description('Build an app for a production release')
    .option('--stats', 'Write bundle stats to output directory')
+    .option(...configOption)
    .action(lazy(() => import('./app/build').then(m => m.default)));

  program
    .command('app:serve')
    .description('Serve an app for local development')
    .option('--check', 'Enable type checking and linting')
+    .option(...configOption)
    .action(lazy(() => import('./app/serve').then(m => m.default)));

  program
@@ -50,6 +59,8 @@ export function registerCommands(program: CommanderStatic) {
    .description('Start local development server with HMR for the backend')
    .option('--check', 'Enable type checking and linting')
    .option('--inspect', 'Enable debugger')
+    // We don't actually use the config in the CLI, just pass them on to the NodeJS process
+    .option(...configOption)
    .action(lazy(() => import('./backend/dev').then(m => m.default)));

  program
@@ -89,12 +100,14 @@ export function registerCommands(program: CommanderStatic) {
    .command('plugin:serve')
    .description('Serves the dev/ folder of a plugin')
    .option('--check', 'Enable type checking and linting')
+    .option(...configOption)
    .action(lazy(() => import('./plugin/serve').then(m => m.default)));

  program
    .command('plugin:export')
    .description('Exports the dev/ folder of a plugin')
    .option('--stats', 'Write bundle stats to output directory')
+    .option(...configOption)
    .action(lazy(() => import('./plugin/export').then(m => m.default)));

  program
@@ -131,14 +144,11 @@ export function registerCommands(program: CommanderStatic) {
  program
    .command('config:print')
    .option('--with-secrets', 'Include secrets in the printed configuration')
-    .option(
-      '--env <env>',
-      'The environment to print configuration for [APP_ENV or NODE_ENV or development]',
-    )
    .option(
      '--format <format>',
      'Format to print the configuration in, either json or yaml [yaml]',
    )
+    .option(...configOption)
    .description('Print the app configuration for the current package')
    .action(lazy(() => import('./config/print').then(m => m.default)));

@@ -15,20 +15,13 @@
 */

 import { Command } from 'commander';
-import { loadConfig } from '@backstage/config-loader';
-import { ConfigReader } from '@backstage/config';
-import { paths } from '../../lib/paths';
 import { buildBundle } from '../../lib/bundler';
+import { loadCliConfig } from '../../lib/config';

 export default async (cmd: Command) => {
-  const appConfigs = await loadConfig({
-    env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production',
-    rootPaths: [paths.targetRoot, paths.targetDir],
-  });
  await buildBundle({
    entry: 'dev/index',
    statsJsonEnabled: cmd.stats,
-    config: ConfigReader.fromConfigs(appConfigs),
-    appConfigs,
+    ...(await loadCliConfig(cmd.config)),
  });
 };
@@ -15,21 +15,14 @@
 */

 import { Command } from 'commander';
-import { loadConfig } from '@backstage/config-loader';
-import { ConfigReader } from '@backstage/config';
-import { paths } from '../../lib/paths';
 import { serveBundle } from '../../lib/bundler';
+import { loadCliConfig } from '../../lib/config';

 export default async (cmd: Command) => {
-  const appConfigs = await loadConfig({
-    env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development',
-    rootPaths: [paths.targetRoot, paths.targetDir],
-  });
  const waitForExit = await serveBundle({
    entry: 'dev/index',
    checksEnabled: cmd.check,
-    config: ConfigReader.fromConfigs(appConfigs),
-    appConfigs,
+    ...(await loadCliConfig(cmd.config)),
  });

  await waitForExit();
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export const pluginsFileContent = `    
+export { plugin as WelcomePlugin } from '@backstage/plugin-welcome';
+export { plugin as LighthousePlugin } from '@backstage/plugin-lighthouse';`;
+
+export const codeownersFileContent = `
+*                                           @spotify/backstage-core
+/docs/features/techdocs                     @spotify/techdocs-core
+/plugins/cost-insights                      @spotify/silver-lining
+`;
+
+export const packageFileContent = {
+  name: 'example-app',
+  version: '0.1.1',
+  dependencies: {},
+  devDependencies: {},
+  scripts: {},
+};
@@ -16,13 +16,9 @@

 import fse from 'fs-extra';
 import path from 'path';
-import os from 'os';
+import mockFs from 'mock-fs';
 import { paths } from '../../lib/paths';
-import {
-  addExportStatement,
-  capitalize,
-  createTemporaryPluginFolder,
-} from '../create-plugin/createPlugin';
+import { addExportStatement, capitalize } from '../create-plugin/createPlugin';
 import { addCodeownersEntry } from '../../lib/codeowners';
 import {
  removeReferencesFromAppPackage,
@@ -31,124 +27,162 @@ import {
  removeSymLink,
  removePluginFromCodeOwners,
 } from './removePlugin';
+import {
+  codeownersFileContent,
+  packageFileContent,
+  pluginsFileContent,
+} from './file-mocks';

 const BACKSTAGE = `@backstage`;
 const testPluginName = 'yarn-test-package';
 const testPluginPackage = `${BACKSTAGE}/plugin-${testPluginName}`;
-const tempDir = path.join(os.tmpdir(), 'remove-plugin-test');
+const tempDir = '/remove-plugin-test';

 const removeEmptyLines = (file: string): string =>
  file.split(/\r?\n/).filter(Boolean).join('\n');

-const createTestPackageFile = async (
-  testFilePath: string,
-  packageFile: string,
-) => {
-  // Copy contents of package file for test
-  const packageFileContent = JSON.parse(fse.readFileSync(packageFile, 'utf8'));
+const createTestPackageFile = async (testFilePath: string) => {
+  const testFileContent = {
+    ...packageFileContent,
+    dependencies: {
+      ...packageFileContent.dependencies,
+      [testPluginPackage]: '0.1.0',
+    },
+  };

-  packageFileContent.dependencies[testPluginPackage] = '0.1.0';
-  fse.createFileSync(testFilePath);
-  fse.writeFileSync(
-    testFilePath,
-    `${JSON.stringify(packageFileContent, null, 2)}\n`,
-    'utf8',
-  );
+  mockFs({
+    packages: {
+      app: {
+        'package.json': `${JSON.stringify(packageFileContent, null, 2)}\n`,
+      },
+    },
+    [tempDir]: {
+      [testFilePath]: `${JSON.stringify(testFileContent, null, 2)}\n`,
+    },
+  });
  return;
 };

 const createTestPluginFile = async (
-  testFilePath: string,
-  pluginsFilePath: string,
+  testFileName: string,
+  pluginsFileName: string,
 ) => {
-  // Copy contents of package file for test
-  fse.copyFileSync(pluginsFilePath, testFilePath);
+  mockFs({
+    [tempDir]: {
+      [testFileName]: `${pluginsFileContent}\n`,
+      [pluginsFileName]: `${pluginsFileContent}\n`,
+    },
+    packages: {
+      app: {
+        src: {
+          'plugin.ts': `${pluginsFileContent}\n`,
+        },
+      },
+    },
+  });
+
  const pluginNameCapitalized = testPluginName
    .split('-')
    .map(name => capitalize(name))
    .join('');
-  const exportStatement = `export { plugin as ${pluginNameCapitalized}} from @backstage/plugin-${testPluginName}`;
-  await addExportStatement(testFilePath, exportStatement);
+  const exportStatement = `export { plugin as ${pluginNameCapitalized}} from ${testPluginPackage}`;
+  await addExportStatement(path.join(tempDir, testFileName), exportStatement);
 };

 const mkTestPluginDir = (testDirPath: string) => {
-  fse.mkdirSync(testDirPath);
-  for (let i = 0; i < 50; i++)
-    fse.createFileSync(path.join(testDirPath, `testFile${i}.ts`));
+  const pluginFiles: { [index: string]: string } = {};
+  for (let i = 0; i < 50; i++) {
+    pluginFiles[`testFile${i}.ts`] = '';
+  }
+
+  mockFs({
+    [testDirPath]: pluginFiles,
+  });
 };

 describe('removePlugin', () => {
  beforeAll(() => {
    // Create temporary directory for all tests
-    createTemporaryPluginFolder(tempDir);
+    mockFs({
+      [tempDir]: {
+        'package.json': packageFileContent,
+        src: {
+          'plugin.ts': pluginsFileContent,
+        },
+      },
+    });
  });

  afterAll(() => {
-    // Remove temporary directory
-    fse.removeSync(tempDir);
+    mockFs.restore();
  });

  describe('Remove Plugin Dependencies', () => {
-    const appPath = paths.resolveTargetRoot('packages', 'app');
-    const githubDir = paths.resolveTargetRoot('.github');
-
    it('removes plugin references from /packages/app/package.json', async () => {
      // Set up test
-      const packageFilePath = path.join(appPath, 'package.json');
-      const testFilePath = path.join(tempDir, 'test.json');
-      createTestPackageFile(testFilePath, packageFilePath);
-      try {
-        await removeReferencesFromAppPackage(testFilePath, testPluginName);
-        const testFileContent = removeEmptyLines(
-          fse.readFileSync(testFilePath, 'utf8'),
-        );
-        const packageFileContent = removeEmptyLines(
-          fse.readFileSync(packageFilePath, 'utf8'),
-        );
-        expect(testFileContent).toBe(packageFileContent);
-      } finally {
-        fse.removeSync(testFilePath);
-      }
-    });
+      const testFilePath = 'test.json';
+      createTestPackageFile(testFilePath);
+      await removeReferencesFromAppPackage(
+        path.join(tempDir, testFilePath),
+        testPluginName,
+      );
+      const testFileContent = removeEmptyLines(
+        fse.readFileSync(path.join(tempDir, testFilePath), 'utf8'),
+      );

+      const mockedPackageFileContent = removeEmptyLines(
+        fse.readFileSync(path.join('packages', 'app', 'package.json'), 'utf8'),
+      );
+      expect(testFileContent).toBe(mockedPackageFileContent);
+    });
    it('removes plugin exports from /packages/app/src/package.json', async () => {
-      const testFilePath = path.join(tempDir, 'test.ts');
-      const pluginsFilePaths = path.join(appPath, 'src', 'plugins.ts');
-      createTestPluginFile(testFilePath, pluginsFilePaths);
-      try {
-        await removeReferencesFromPluginsFile(testFilePath, testPluginName);
-        const testFileContent = removeEmptyLines(
-          fse.readFileSync(testFilePath, 'utf8'),
-        );
-        const pluginsFileContent = removeEmptyLines(
-          fse.readFileSync(pluginsFilePaths, 'utf8'),
-        );
-        expect(testFileContent).toBe(pluginsFileContent);
-      } finally {
-        fse.removeSync(testFilePath);
-      }
+      const testFileName = 'test.ts';
+      const pluginsFileName = 'plugin.ts';
+      createTestPluginFile(testFileName, pluginsFileName);
+      await removeReferencesFromPluginsFile(
+        path.join(tempDir, testFileName),
+        testPluginName,
+      );
+      const testFileContent = removeEmptyLines(
+        fse.readFileSync(path.join(tempDir, testFileName), 'utf8'),
+      );
+      const mockedPluginsFileContent = removeEmptyLines(
+        fse.readFileSync(
+          path.join('packages', 'app', 'src', pluginsFileName),
+          'utf8',
+        ),
+      );
+      expect(testFileContent).toBe(mockedPluginsFileContent);
    });

    it('removes codeOwners references', async () => {
-      const testFilePath = path.join(tempDir, 'test');
-      const codeownersPath = path.join(githubDir, 'CODEOWNERS');
-      try {
-        fse.copySync(codeownersPath, testFilePath);
-        const testFileContent = removeEmptyLines(
-          fse.readFileSync(testFilePath, 'utf8'),
-        );
-        const codeOwnersFileContent = removeEmptyLines(
-          fse.readFileSync(codeownersPath, 'utf8'),
-        );
-        await addCodeownersEntry(testFilePath!, `/plugins/${testPluginName}`, [
-          '@thisIsAtestTeam',
-          'test@gmail.com',
-        ]);
-        await removePluginFromCodeOwners(testFilePath, testPluginName);
-        expect(testFileContent).toBe(codeOwnersFileContent);
-      } finally {
-        if (fse.existsSync(testFilePath)) fse.removeSync(testFilePath);
-      }
+      const testFileName = 'test';
+      const testFilePath = path.join(tempDir, testFileName);
+
+      const mockedCodeownersPath = path.join('.github', 'CODEOWNERS');
+
+      mockFs({
+        [tempDir]: {
+          [testFileName]: '',
+        },
+        '.github': {
+          CODEOWNERS: codeownersFileContent,
+        },
+      });
+      fse.copySync(mockedCodeownersPath, testFilePath);
+      const testFileContent = removeEmptyLines(
+        fse.readFileSync(testFilePath, 'utf8'),
+      );
+      const codeOwnersFileContent = removeEmptyLines(
+        fse.readFileSync(mockedCodeownersPath, 'utf8'),
+      );
+      await addCodeownersEntry(
+        testFilePath!,
+        path.join('plugins', testPluginName),
+        ['@thisIsAtestTeam', 'test@gmail.com'],
+      );
+      await removePluginFromCodeOwners(testFilePath, testPluginName);
+      expect(testFileContent).toBe(codeOwnersFileContent);
    });
  });

@@ -161,34 +195,40 @@ describe('removePlugin', () => {

    describe('Removes Plugin Directory', () => {
      it('removes plugin directory from /plugins', async () => {
-        try {
-          mkTestPluginDir(testDirPath);
-          expect(fse.existsSync(testDirPath)).toBeTruthy();
-          await removePluginDirectory(testDirPath);
-          expect(fse.existsSync(testDirPath)).toBeFalsy();
-        } finally {
-          if (fse.existsSync(testDirPath)) fse.removeSync(testDirPath);
-        }
+        mkTestPluginDir(testDirPath);
+        expect(fse.existsSync(testDirPath)).toBeTruthy();
+        await removePluginDirectory(testDirPath);
+        expect(fse.existsSync(testDirPath)).toBeFalsy();
      });
    });

    describe('Removes System Link', () => {
      it('removes system link from @backstage', async () => {
-        const scopedDir = paths.resolveTargetRoot('node_modules', '@backstage');
+        const symLink = `plugin-${testPluginName}`;
        const testSymLinkPath = path.join(
-          scopedDir,
-          `plugin-${testPluginName}`,
+          '/',
+          'node_modules',
+          '@backstage',
+          symLink,
        );
-        try {
-          mkTestPluginDir(testDirPath);
-          fse.ensureSymlinkSync(testSymLinkPath, testDirPath);
+        const mockedTestDirPath = path.join('/', 'plugins', testPluginName);

-          await removeSymLink(testSymLinkPath);
-          expect(fse.existsSync(testSymLinkPath)).toBeFalsy();
-        } finally {
-          if (fse.existsSync(testDirPath)) fse.removeSync(testDirPath);
-          if (fse.existsSync(testSymLinkPath)) fse.removeSync(testSymLinkPath);
-        }
+        mockFs({
+          '/plugins': {
+            [testPluginName]: {},
+          },
+          '/node_modules': {
+            '@backstage': {
+              [symLink]: mockFs.symlink({
+                path: mockedTestDirPath,
+              }),
+            },
+          },
+        });
+
+        expect(fse.existsSync(testSymLinkPath)).toBeTruthy();
+        await removeSymLink(testSymLinkPath);
+        expect(fse.existsSync(testSymLinkPath)).toBeFalsy();
      });
    });
  });
@@ -17,13 +17,9 @@
 import webpack from 'webpack';
 import { createBackendConfig } from './config';
 import { resolveBundlingPaths } from './paths';
-import { ServeOptions } from './types';
+import { BackendServeOptions } from './types';

-export async function serveBackend(
-  options: ServeOptions & {
-    inspectEnabled: boolean;
-  },
-) {
+export async function serveBackend(options: BackendServeOptions) {
  const paths = resolveBundlingPaths(options);
  const config = await createBackendConfig(paths, {
    ...options,
--- a/Show More
+++ b/Show More