Merge branch 'master' into ryanv/product-panel-improvements

.changeset/add-config-schema-declarations.md

@@ -0,0 +1,13 @@
+---
+'@backstage/backend-common': patch
+'@backstage/cli': patch
+'@backstage/core': patch
+'@backstage/plugin-cost-insights': patch
+'@backstage/plugin-lighthouse': patch
+'@backstage/plugin-rollbar': patch
+'@backstage/plugin-sentry': patch
+'@backstage/plugin-techdocs': patch
+'@backstage/plugin-user-settings': patch
+---
+
+Added configuration schema

.changeset/add-config-schema-support.md

@@ -0,0 +1,27 @@
+---
+'@backstage/backend-common': minor
+'@backstage/cli': minor
+'@backstage/config-loader': minor
+---
+
+Added support for loading and validating configuration schemas, as well as declaring config visibility through schemas.
+
+The new `loadConfigSchema` function exported by `@backstage/config-loader` allows for the collection and merging of configuration schemas from all nearby dependencies of the project.
+
+A configuration schema is declared using the `https://backstage.io/schema/config-v1` JSON Schema meta schema, which is based on draft07. The only difference to the draft07 schema is the custom `visibility` keyword, which is used to indicate whether the given config value should be visible in the frontend or not. The possible values are `frontend`, `backend`, and `secret`, where `backend` is the default. A visibility of `secret` has the same scope at runtime, but it will be treated with more care in certain contexts, and defining both `frontend` and `secret` for the same value in two different schemas will result in an error during schema merging.
+
+Packages that wish to contribute configuration schema should declare it in a root `"configSchema"` field in `package.json`. The field can either contain an inlined JSON schema, or a relative path to a schema file. Schema files can be in either `.json` or `.d.ts` format.
+
+TypeScript configuration schema files should export a single `Config` type, for example:
+
+```ts
+export interface Config {
+  app: {
+    /**
+     * Frontend root URL
+     * @visibility frontend
+     */
+    baseUrl: string;
+  };
+}
+```

.changeset/app-backend-config.md

@@ -0,0 +1,7 @@
+---
+'@backstage/plugin-app-backend': minor
+---
+
+Use new config schema support to automatically inject config with frontend visibility, in addition to the existing env schema injection.
+
+This removes the confusing behavior where configuration was only injected into the app at build time. Any runtime configuration (except for environment config) in the backend used to only apply to the backend itself, and not be injected into the frontend.

.changeset/cost-insights-ten-carpets-return.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-cost-insights': minor
+---
+
+remove cost insights currency feature flag

.changeset/modern-boats-knock.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Support specifying listen host/port for frontend

.changeset/shaggy-turkeys-warn.md

@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-client': minor
+---
+
+Changed the getEntities interface to (1) nest parameters in an object, (2) support field selection, and (3) return an object with an items field for future extension

.changeset/silent-suns-bathe.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Extracted pushToRemote function for reuse between publishers

.github/CODEOWNERS

@@ -9,5 +9,6 @@
 /plugins/cost-insights                      @backstage/silver-lining
 /plugins/cloudbuild                         @trivago/ebarrios
 /plugins/techdocs                           @backstage/techdocs-core
+/plugins/search                             @backstage/techdocs-core
 /plugins/techdocs-backend                   @backstage/techdocs-core
 /.changeset/cost-insights-*                 @backstage/silver-lining

.github/styles/vocab.txt

@@ -76,6 +76,7 @@ http
 https
 img
 incentivised
+inlined
 inlinehilite
 interop
 javascript
@@ -176,6 +177,7 @@ src
 subkey
 superfences
 Superfences
+superset
 talkdesk
 Talkdesk
 tasklist

.github/workflows/ci.yml

@@ -72,6 +72,9 @@ jobs:
       - name: prettier
         run: yarn prettier:check
 
+      - name: validate config
+        run: yarn backstage-cli config:check
+
       - name: lint
         run: yarn lerna -- run lint --since origin/master
 

.github/workflows/e2e-win.yml

@@ -32,7 +32,7 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - name: Add msbuild to PATH
-        uses: microsoft/setup-msbuild@v1.0.1
+        uses: microsoft/setup-msbuild@v1.0.2
       - name: yarn install
         run: yarn install --frozen-lockfile
 

.github/workflows/master.yml

@@ -47,6 +47,9 @@ jobs:
         run: yarn install --frozen-lockfile
       # End of yarn setup
 
+      - name: validate config
+        run: yarn backstage-cli config:check
+
       - name: lint
         run: yarn lerna -- run lint
 

app-config.yaml

@@ -270,7 +270,7 @@ costInsights:
 homepage:
   clocks:
     - label: UTC
-      timzone: UTC
+      timezone: UTC
     - label: NYC
       timezone: 'America/New_York'
     - label: STO

docs/conf/defining.md

@@ -4,16 +4,109 @@ title: Defining Configuration for your Plugin
 description: Documentation on Defining Configuration for your Plugin
 ---
 
-There is currently no tooling support or helpers for defining plugin
-configuration. But it's on the roadmap.
+Configuration in Backstage is organized via a configuration schema, which in
+turn is defined using a superset of
+[JSON Schema Draft-07](https://json-schema.org/specification-links.html#draft-7).
+Each plugin or package within a Backstage app can contribute to the schema,
+which during validation is stitched together into a single schema.
 
-Meanwhile, document the config values that you are reading in your plugin
-README.
+## Schema Collection and Definition
 
-## Format
+Schemas are collected from all packages and dependencies in each repo that are a
+part of the Backstage ecosystem, including transitive dependencies. The current
+definition of "part of the ecosystem" is that a package has at least one
+dependency in the `@backstage` namespace, but this is subject to change.
+
+Each package is searched for a schema at a single point of entry, a top-level
+`"configSchema"` field in `package.json`. The field can either contain an
+inlined JSON schema, or a relative path to a schema file. Supported schema file
+formats are `.json` or `.d.ts`.
+
+> When defining a schema file, be sure to include the file in your
+> `package.json` > `"files"` field as well!
+
+TypeScript configuration schema files should export a single `Config` type, for
+example:
+
+```ts
+export interface Config {
+  app: {
+    /**
+     * Frontend root URL
+     * @visibility frontend
+     */
+    baseUrl: string;
+  };
+}
+```
+
+Separate `.json` schema files can use a top-level
+`"$schema": "https://backstage.io/schema/config-v1"` declaration in order to
+receive schema validation and autocompletion. For example:
+
+```json
+{
+  "$schema": "https://backstage.io/schema/config-v1",
+  "type": "object",
+  "properties": {
+    "app": {
+      "type": "object",
+      "properties": {
+        "baseUrl": {
+          "type": "string",
+          "description": "Frontend root URL",
+          "visibility": "frontend"
+        }
+      },
+      "required": ["baseUrl"]
+    },
+    "required": ["app"]
+  }
+}
+```
+
+## Visibility
+
+The `https://backstage.io/schema/config-v1` meta schema is a superset of JSON
+Schema Draft 07. The single addition is a custom `visibility` keyword, which is
+used to indicate whether the given config value should be visible in the
+frontend or not. The possible values are `frontend`, `backend`, and `secret`,
+where `backend` is the default. A visibility of `secret` has the same scope at
+runtime, but it will be treated with more care in certain contexts, and defining
+both `frontend` and `secret` for the same value in two different schemas will
+result in an error during schema merging.
+
+The visibility only applies to the direct parent of where the keyword is placed
+in the schema. For example, if you set the visibility to `frontend` for a subset
+of the schema with `type: "object"`, but none of the descendants, only an empty
+object will be available in the frontend. The full ancestry does not need to
+have correctly defined visibilities however, so it is enough to only for example
+declare the visibility of a leaf node of `type: "string"`.
+
+## Validation
+
+Schemas can be validated using the `backstage-cli config:check` command. If you
+want to validate anything else than the default `app-config.yaml`, be sure to
+pass in all of the configuration files as `--config <path>` options as well.
+
+To validate and examine the frontend configuration, use the
+`backstage-cli config:print --frontend` command. Just like for validation you
+may need to pass in all files using one or multiple `--config <path>` options.
+
+## Guidelines
+
+> Make limited use of static configuration. The first question to ask is whether
+> a particular option actually needs to be static configuration, or if it might
+> just as well be a TypeScript API. In general, options that you want to be able
+> to change for different deployment environments should be static
+> configuration, while it should otherwise be avoided.
 
 When defining configuration for your plugin, keep keys camelCased and stick to
-existing casing conventions such as `baseUrl`.
+existing casing conventions such as `baseUrl` rather than `baseURL`.
 
 It is also usually best to prefer objects over arrays, as it makes it possible
 to override individual values using separate files or environment variables.
+
+Avoid creating new top-level fields as much as possible. Either place your
+configuration within an existing known top-level block, or create a single new
+one using e.g. the name of the product that the plugin integrates.

docs/conf/index.md

@@ -33,6 +33,20 @@ values that are common between the two only need to be defined once. Such as the
 
 For more details, see [Writing Configuration](./writing.md).
 
+## Configuration Schema
+
+The configuration is validated using JSON Schema definitions. Each plugin and
+package can provide pieces of the configuration schema, which are stitched
+together to form a complete schema during validation. The configuration schema
+is also used to select what configuration is available in the frontend using a
+custom `visibility` keyword, as configuration is by default only available in
+the backend.
+
+You can validate your configuration against the schema using
+`backstage-cli config:check`, and define a schema for your own plugin either
+using JSON Schema or TypeScript. For more information, see
+[Defining Configuration](./defining.md).
+
 ## Reading Configuration
 
 As a plugin developer, you likely end up wanting to define configuration that
@@ -49,5 +63,5 @@ More details are provided in dedicated sections of the documentation.
   plugin.
 - [Writing Configuration](./writing.md): How to provide configuration for your
   Backstage deployment.
-- [Defining Configuration](./defining.md): How to define configuration for users
-  of your plugin.
+- [Defining Configuration](./defining.md): How to define a configuration schema
+  for users of your plugin or package.

docs/conf/writing.md

@@ -97,10 +97,10 @@ order:
 - If no config flags are provided, `app-config.local.yaml` has higher priority
   than `app-config.yaml`.
 
-## Secrets
+## Secrets and Dynamic Data
 
-Secrets are supported via special secret keys that are prefixed with `$`, which
-in turn provide a number of different ways to read in secrets. To load a
+Secrets are supported via special data loading keys that are prefixed with `$`,
+which in turn provide a number of different ways to read in secrets. To load a
 configuration value as a secret, supply an object with one of the special secret
 keys, for example `$env` or `$file`. A full list of supported secret keys can be
 found below. For example, the following will read the config key
@@ -117,10 +117,6 @@ will return the value of the environment variable `MY_SECRET_KEY` when the
 backend started up. All secrets are loaded at startup, so changing the contents
 of secret files or environment variables will not be reflected at runtime.
 
-Note that secrets will never be included in the frontend bundle or development
-builds. When loading configuration you have to explicitly enable reading of
-secrets, which is only done for the backend configuration.
-
 As hinted at, secrets can be loaded from a bunch of different sources, and can
 be extended with more. Below is a list of the currently supported methods for
 loading secrets.

docs/features/software-catalog/descriptor-format.md

@@ -2,8 +2,8 @@
 id: descriptor-format
 title: Descriptor Format of Catalog Entities
 sidebar_label: YAML File Format
-description: Documentation on Descriptor Format of Catalog Entities which
-describes the default data shape and semantics of catalog entities
+# prettier-ignore
+description: Documentation on Descriptor Format of Catalog Entities which describes the default data shape and semantics of catalog entities
 ---
 
 This section describes the default data shape and semantics of catalog entities.

docs/features/software-catalog/external-integrations.md

@@ -1,8 +1,8 @@
 ---
 id: external-integrations
 title: External integrations
-description: Documentation on External integrations to integrate systems
-with Backstage
+# prettier-ignore
+description: Documentation on External integrations to integrate systems with Backstage
 ---
 
 Backstage natively supports importing catalog data through the use of

docs/features/software-catalog/index.md

@@ -2,8 +2,8 @@
 id: software-catalog-overview
 title: Backstage Service Catalog (alpha)
 sidebar_label: Overview
-description: The Backstage Service Catalog — actually, a software catalog, since
-it includes more than just services
+# prettier-ignore
+description: The Backstage Service Catalog — actually, a software catalog, since it includes more than just services
 ---
 
 ## What is a Service Catalog?

docs/features/software-catalog/well-known-annotations.md

@@ -2,9 +2,8 @@
 id: well-known-annotations
 title: Well-known Annotations on Catalog Entities
 sidebar_label: Well-known Annotations
-description: Documentation that lists a number of well known Annotations, that
-have defined semantics. They can be attached to catalog entities and consumed
-by plugins as needed.
+# prettier-ignore
+description: Documentation that lists a number of well known Annotations, that have defined semantics. They can be attached to catalog entities and consumed by plugins as needed.
 ---
 
 This section lists a number of well known

docs/features/software-catalog/well-known-relations.md

@@ -2,9 +2,8 @@
 id: well-known-relations
 title: Well-known Relations between Catalog Entities
 sidebar_label: Well-known Relations
-description: Documentation that lists a number of well known Relations, that
-have defined semantics. They can be attached to catalog entities and consumed
-by plugins as needed.
+# prettier-ignore
+description: Documentation that lists a number of well known Relations, that have defined semantics. They can be attached to catalog entities and consumed by plugins as needed.
 ---
 
 This section lists a number of well known

docs/features/techdocs/README.md

@@ -122,3 +122,12 @@ migrate Spotify's existing TechDocs features to open source.
   https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend
 [techdocs/container]: https://github.com/backstage/techdocs-container
 [techdocs/cli]: https://github.com/backstage/techdocs-cli
+
+## Feedback
+
+We have created a sweet and short TechDocs user survey -
+https://docs.google.com/forms/d/e/1FAIpQLSdn5Vn3MQhCdyYRuW8cMzZkMQF0bFxXYN168gZRvESLfJWVVg/viewform
+
+This is to gather inputs from you (the Backstage community) which will help us
+best serve TechDocs adopters and existing users. Your inputs will shape our
+roadmap and we will share it in the open.

packages/app/package.json

@@ -25,6 +25,7 @@
     "@backstage/plugin-rollbar": "^0.2.1",
     "@backstage/plugin-scaffolder": "^0.3.0",
     "@backstage/plugin-sentry": "^0.2.1",
+    "@backstage/plugin-search": "^0.2.0",
     "@backstage/plugin-tech-radar": "^0.3.0",
     "@backstage/plugin-techdocs": "^0.2.1",
     "@backstage/plugin-user-settings": "^0.2.1",
@@ -49,7 +50,7 @@
     "zen-observable": "^0.8.15"
   },
   "devDependencies": {
-    "@testing-library/cypress": "^6.0.0",
+    "@testing-library/cypress": "^7.0.1",
     "@testing-library/jest-dom": "^5.10.1",
     "@testing-library/react": "^10.4.1",
     "@testing-library/user-event": "^12.0.7",

packages/app/src/plugins.ts

@@ -39,3 +39,4 @@ export { plugin as CostInsights } from '@backstage/plugin-cost-insights';
 export { plugin as GitHubInsights } from '@roadiehq/backstage-plugin-github-insights';
 export { plugin as UserSettings } from '@backstage/plugin-user-settings';
 export { plugin as BuildKite } from '@roadiehq/backstage-plugin-buildkite';
+export { plugin as Search } from '@backstage/plugin-search';

packages/backend-common/config.d.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface Config {
+  app: {
+    baseUrl: string; // defined in core, but repeated here without doc
+  };
+
+  backend: {
+    baseUrl: string; // defined in core, but repeated here without doc
+
+    /** Address that the backend should listen to. */
+    listen:
+      | string
+      | {
+          /** Address of the interface that the backend should bind to. */
+          address?: string;
+          /** Port that the backend should listen to. */
+          port?: number;
+        };
+
+    /** HTTPS configuration for the backend. If omitted the backend will serve HTTP */
+    https?: {
+      /** Certificate configuration or parameters for generating a self-signed certificate */
+      certificate?:
+        | {
+            /** Algorithm to use to generate a self-signed certificate */
+            algorithm: string;
+            keySize?: number;
+            days?: number;
+          }
+        | {
+            /** PEM encoded certificate. Use $file to load in a file */
+            cert: string;
+            /**
+             * PEM encoded certificate key. Use $file to load in a file.
+             * @visibility secret
+             */
+            key: string;
+          };
+    };
+
+    /** Database connection configuration, select database type using the `client` field */
+    database:
+      | {
+          client: 'sqlite3';
+          connection: ':memory:' | string;
+        }
+      | {
+          client: 'pg';
+          /**
+           * PostgreSQL connection string or knex configuration object.
+           * @secret
+           */
+          connection: string | object;
+        };
+
+    cors?: {
+      origin?: string | string[];
+      methods?: string | string[];
+      allowedHeaders?: string | string[];
+      exposedHeaders?: string | string[];
+      credentials?: boolean;
+      maxAge?: number;
+      preflightContinue?: boolean;
+      optionsSuccessStatus?: number;
+    };
+
+    /**  */
+    csp?: object;
+  };
+
+  /** Configuration for integrations towards various external repository provider systems */
+  integrations?: {
+    /** Integration configuration for Azure */
+    azure?: Array<{
+      /** The hostname of the given Azure instance */
+      host: string;
+      /**
+       * Token used to authenticate requests.
+       * @visibility secret
+       */
+      token?: string;
+    }>;
+
+    /** Integration configuration for BitBucket */
+    bitbucket?: Array<{
+      /** The hostname of the given Bitbucket instance */
+      host: string;
+      /**
+       * Token used to authenticate requests.
+       * @visibility secret
+       */
+      token?: string;
+      /** The base url for the BitBucket API, for example https://api.bitbucket.org/2.0 */
+      apiBaseUrl?: string;
+      /**
+       * The username to use for authenticated requests.
+       * @visibility secret
+       */
+      username?: string;
+      /**
+       * BitBucket app password used to authenticate requests.
+       * @visibility secret
+       */
+      appPassword?: string;
+    }>;
+
+    /** Integration configuration for GitHub */
+    github?: Array<{
+      /** The hostname of the given GitHub instance */
+      host: string;
+      /**
+       * Token used to authenticate requests.
+       * @visibility secret
+       */
+      token?: string;
+      /** The base url for the GitHub API, for example https://api.github.com */
+      apiBaseUrl?: string;
+      /** The base url for GitHub raw resources, for example https://raw.githubusercontent.com */
+      rawBaseUrl?: string;
+    }>;
+
+    /** Integration configuration for GitLab */
+    gitlab?: Array<{
+      /** The hostname of the given GitLab instance */
+      host: string;
+      /**
+       * Token used to authenticate requests.
+       * @visibility secret
+       */
+      token?: string;
+    }>;
+  };
+}

packages/backend-common/package.json

@@ -89,6 +89,8 @@
     "supertest": "^4.0.2"
   },
   "files": [
-    "dist"
-  ]
+    "dist",
+    "config.d.ts"
+  ],
+  "configSchema": "config.d.ts"
 }

packages/backend-common/src/config.ts

@@ -40,7 +40,6 @@ export async function loadBackendConfig(options: Options): Promise<Config> {
     env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development',
     configRoot: paths.targetRoot,
     configPaths: configOpts.map(opt => resolvePath(opt)),
-    shouldReadSecrets: true,
   });
 
   options.logger.info(

packages/backend/src/plugins/app.ts

@@ -17,9 +17,13 @@
 import { createRouter } from '@backstage/plugin-app-backend';
 import { PluginEnvironment } from '../types';
 
-export default async function createPlugin({ logger }: PluginEnvironment) {
+export default async function createPlugin({
+  logger,
+  config,
+}: PluginEnvironment) {
   return await createRouter({
     logger,
+    config,
     appPackageName: 'example-app',
   });
 }

packages/catalog-client/src/CatalogClient.test.ts

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
+import { Entity } from '@backstage/catalog-model';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
 import { CatalogClient } from './CatalogClient';
-import { Entity } from '@backstage/catalog-model';
-import { DiscoveryApi } from './types';
+import { CatalogListResponse, DiscoveryApi } from './types';
 
 const server = setupServer();
 const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base';
@@ -40,7 +40,7 @@ describe('CatalogClient', () => {
   });
 
   describe('getEntities', () => {
-    const defaultResponse: Entity[] = [
+    const defaultServiceResponse: Entity[] = [
       {
         apiVersion: '1',
         kind: 'Component',
@@ -58,22 +58,26 @@ describe('CatalogClient', () => {
         },
       },
     ];
+    const defaultResponse: CatalogListResponse<Entity> = {
+      items: defaultServiceResponse,
+    };
 
     beforeEach(() => {
       server.use(
         rest.get(`${mockBaseUrl}/entities`, (_, res, ctx) => {
-          return res(ctx.json(defaultResponse));
+          return res(ctx.json(defaultServiceResponse));
         }),
       );
     });
 
     it('should entities from correct endpoint', async () => {
-      const entities = await client.getEntities();
-      expect(entities).toEqual(defaultResponse);
+      const response = await client.getEntities();
+      expect(response).toEqual(defaultResponse);
     });
 
     it('builds entity search filters properly', async () => {
       expect.assertions(2);
+
       server.use(
         rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => {
           expect(req.url.search).toBe('?filter=a=1,b=2,b=3,%C3%B6=%3D');
@@ -81,13 +85,32 @@ describe('CatalogClient', () => {
         }),
       );
 
-      const entities = await client.getEntities({
-        a: '1',
-        b: ['2', '3'],
-        ö: '=',
+      const response = await client.getEntities({
+        filter: {
+          a: '1',
+          b: ['2', '3'],
+          ö: '=',
+        },
       });
 
-      expect(entities).toEqual([]);
+      expect(response.items).toEqual([]);
+    });
+
+    it('builds entity field selectors properly', async () => {
+      expect.assertions(2);
+
+      server.use(
+        rest.get(`${mockBaseUrl}/entities`, (req, res, ctx) => {
+          expect(req.url.search).toBe('?fields=a.b,%C3%B6');
+          return res(ctx.json([]));
+        }),
+      );
+
+      const response = await client.getEntities({
+        fields: ['a.b', 'ö'],
+      });
+
+      expect(response.items).toEqual([]);
     });
   });
 });

packages/catalog-client/src/CatalogClient.ts

@@ -25,6 +25,8 @@ import {
   AddLocationRequest,
   AddLocationResponse,
   CatalogApi,
+  CatalogEntitiesRequest,
+  CatalogListResponse,
   DiscoveryApi,
 } from './types';
 
@@ -35,55 +37,33 @@ export class CatalogClient implements CatalogApi {
     this.discoveryApi = options.discoveryApi;
   }
 
-  private async getRequired(path: string): Promise<any> {
-    const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`;
-    const response = await fetch(url);
-
-    if (!response.ok) {
-      const payload = await response.text();
-      const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`;
-      throw new Error(message);
-    }
-
-    return await response.json();
-  }
-
-  private async getOptional(path: string): Promise<any | undefined> {
-    const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`;
-    const response = await fetch(url);
-
-    if (!response.ok) {
-      if (response.status === 404) {
-        return undefined;
-      }
-
-      const payload = await response.text();
-      const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`;
-      throw new Error(message);
-    }
-
-    return await response.json();
-  }
-
   async getLocationById(id: String): Promise<Location | undefined> {
     return await this.getOptional(`/locations/${id}`);
   }
 
   async getEntities(
-    filter?: Record<string, string | string[]>,
-  ): Promise<Entity[]> {
-    let path = `/entities`;
-    if (filter) {
-      const parts: string[] = [];
-      for (const [key, value] of Object.entries(filter)) {
-        for (const v of [value].flat()) {
-          parts.push(`${encodeURIComponent(key)}=${encodeURIComponent(v)}`);
-        }
+    request?: CatalogEntitiesRequest,
+  ): Promise<CatalogListResponse<Entity>> {
+    const { filter = {}, fields = [] } = request ?? {};
+    const params: string[] = [];
+
+    const filterParts: string[] = [];
+    for (const [key, value] of Object.entries(filter)) {
+      for (const v of [value].flat()) {
+        filterParts.push(`${encodeURIComponent(key)}=${encodeURIComponent(v)}`);
       }
-      path += `?filter=${parts.join(',')}`;
+    }
+    if (filterParts.length) {
+      params.push(`filter=${filterParts.join(',')}`);
     }
 
-    return await this.getRequired(path);
+    if (fields.length) {
+      params.push(`fields=${fields.map(encodeURIComponent).join(',')}`);
+    }
+
+    const query = params.length ? `?${params.join('&')}` : '';
+    const entities: Entity[] = await this.getRequired(`/entities${query}`);
+    return { items: entities };
   }
 
   async getEntityByName(compoundName: EntityName): Promise<Entity | undefined> {
@@ -153,4 +133,38 @@ export class CatalogClient implements CatalogApi {
     }
     return undefined;
   }
+
+  //
+  // Private methods
+  //
+
+  private async getRequired(path: string): Promise<any> {
+    const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      const payload = await response.text();
+      const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`;
+      throw new Error(message);
+    }
+
+    return await response.json();
+  }
+
+  private async getOptional(path: string): Promise<any | undefined> {
+    const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+      if (response.status === 404) {
+        return undefined;
+      }
+
+      const payload = await response.text();
+      const message = `Request failed with ${response.status} ${response.statusText}, ${payload}`;
+      throw new Error(message);
+    }
+
+    return await response.json();
+  }
 }

packages/catalog-client/src/types.ts

@@ -16,10 +16,21 @@
 
 import { Entity, EntityName, Location } from '@backstage/catalog-model';
 
+export type CatalogEntitiesRequest = {
+  filter?: Record<string, string | string[]> | undefined;
+  fields?: string[] | undefined;
+};
+
+export type CatalogListResponse<T> = {
+  items: T[];
+};
+
 export interface CatalogApi {
   getLocationById(id: String): Promise<Location | undefined>;
   getEntityByName(name: EntityName): Promise<Entity | undefined>;
-  getEntities(filter?: Record<string, string | string[]>): Promise<Entity[]>;
+  getEntities(
+    request?: CatalogEntitiesRequest,
+  ): Promise<CatalogListResponse<Entity>>;
   addLocation(location: AddLocationRequest): Promise<AddLocationResponse>;
   getLocationByEntity(entity: Entity): Promise<Location | undefined>;
   removeEntityByUid(uid: string): Promise<void>;

packages/cli/package.json

@@ -76,6 +76,7 @@
     "jest": "^26.0.1",
     "jest-css-modules": "^2.1.0",
     "jest-esm-transformer": "^1.0.0",
+    "lodash": "^4.17.19",
     "mini-css-extract-plugin": "^0.9.0",
     "ora": "^4.0.3",
     "raw-loader": "^4.0.1",
@@ -146,5 +147,47 @@
     "watch": "./src",
     "exec": "bin/backstage-cli",
     "ext": "ts"
+  },
+  "configSchema": {
+    "$schema": "https://backstage.io/schema/config-v1",
+    "title": "@backstage/cli",
+    "type": "object",
+    "properties": {
+      "app": {
+        "type": "object",
+        "properties": {
+          "baseUrl": {
+            "type": "string",
+            "visibility": "frontend"
+          },
+          "title": {
+            "type": "string",
+            "visibility": "frontend"
+          },
+          "googleAnalyticsTrackingId": {
+            "type": "string",
+            "visibility": "frontend",
+            "description": "Tracking ID for Google Analytics",
+            "example": "UA-000000-0"
+          },
+          "listen": {
+            "type": "object",
+            "description": "Listening configuration for local development",
+            "properties": {
+              "host": {
+                "type": "number",
+                "visibility": "frontend",
+                "description": "The host that the frontend should be bound to. Only used for local development."
+              },
+              "post": {
+                "type": "number",
+                "visibility": "frontend",
+                "description": "The port that the frontend should be bound to. Only used for local development."
+              }
+            }
+          }
+        }
+      }
+    }
   }
 }

packages/cli/src/commands/config/print.ts

@@ -16,16 +16,52 @@
 
 import { Command } from 'commander';
 import { stringify as stringifyYaml } from 'yaml';
+import { AppConfig, ConfigReader } from '@backstage/config';
 import { loadCliConfig } from '../../lib/config';
+import { ConfigSchema, ConfigVisibility } from '@backstage/config-loader';
 
 export default async (cmd: Command) => {
-  const { config } = await loadCliConfig(cmd.config, cmd.withSecrets ?? false);
-
-  const flatConfig = config.get();
+  const { schema, appConfigs } = await loadCliConfig(cmd.config);
+  const visibility = getVisiblityOption(cmd);
+  const data = serializeConfigData(appConfigs, schema, visibility);
 
   if (cmd.format === 'json') {
-    process.stdout.write(`${JSON.stringify(flatConfig, null, 2)}\n`);
+    process.stdout.write(`${JSON.stringify(data, null, 2)}\n`);
   } else {
-    process.stdout.write(`${stringifyYaml(flatConfig)}\n`);
+    process.stdout.write(`${stringifyYaml(data)}\n`);
   }
 };
+
+function getVisiblityOption(cmd: Command): ConfigVisibility {
+  if (cmd.frontend && cmd.withSecrets) {
+    throw new Error('Not allowed to combine frontend and secret config');
+  }
+  if (cmd.frontend) {
+    return 'frontend';
+  } else if (cmd.withSecrets) {
+    return 'secret';
+  }
+  return 'backend';
+}
+
+function serializeConfigData(
+  appConfigs: AppConfig[],
+  schema: ConfigSchema,
+  visiblity: ConfigVisibility,
+) {
+  if (visiblity === 'frontend') {
+    const frontendConfigs = schema.process(appConfigs, {
+      visiblity: ['frontend'],
+    });
+    return ConfigReader.fromConfigs(frontendConfigs).get();
+  } else if (visiblity === 'secret') {
+    return ConfigReader.fromConfigs(appConfigs).get();
+  }
+
+  const sanitizedConfigs = schema.process(appConfigs, {
+    valueTransform: (value, { visibility }) =>
+      visibility === 'secret' ? '<secret>' : value,
+  });
+
+  return ConfigReader.fromConfigs(sanitizedConfigs).get();
+}

packages/cli/src/commands/config/validate.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Command } from 'commander';
+import { loadCliConfig } from '../../lib/config';
+
+export default async (cmd: Command) => {
+  await loadCliConfig(cmd.config);
+};

packages/cli/src/commands/index.ts

@@ -136,6 +136,7 @@ export function registerCommands(program: CommanderStatic) {
 
   program
     .command('config:print')
+    .option('--frontend', 'Print only the frontend configuration')
     .option('--with-secrets', 'Include secrets in the printed configuration')
     .option(
       '--format <format>',
@@ -145,6 +146,14 @@ export function registerCommands(program: CommanderStatic) {
     .description('Print the app configuration for the current package')
     .action(lazy(() => import('./config/print').then(m => m.default)));
 
+  program
+    .command('config:check')
+    .option(...configOption)
+    .description(
+      'Validate that the given configuration loads and matches schema',
+    )
+    .action(lazy(() => import('./config/validate').then(m => m.default)));
+
   program
     .command('prepack')
     .description('Prepares a package for packaging before publishing')

packages/cli/src/lib/bundler/bundle.ts

@@ -33,14 +33,14 @@ const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024;
 const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024;
 
 export async function buildBundle(options: BuildOptions) {
-  const { statsJsonEnabled } = options;
+  const { statsJsonEnabled, schema: configSchema } = options;
 
   const paths = resolveBundlingPaths(options);
   const config = await createConfig(paths, {
     ...options,
     checksEnabled: false,
     isDev: false,
-    baseUrl: resolveBaseUrl(options.config),
+    baseUrl: resolveBaseUrl(options.frontendConfig),
   });
   const compiler = webpack(config);
 
@@ -56,6 +56,14 @@ export async function buildBundle(options: BuildOptions) {
     });
   }
 
+  if (configSchema) {
+    await fs.writeJson(
+      resolvePath(paths.targetDist, '.config-schema.json'),
+      configSchema.serialize(),
+      { spaces: 2 },
+    );
+  }
+
   const { stats } = await build(compiler, isCi).catch(error => {
     console.log(chalk.red('Failed to compile.\n'));
     throw new Error(`Failed to compile.\n${error.message || error}`);

packages/cli/src/lib/bundler/config.ts

@@ -74,11 +74,11 @@ export async function createConfig(
   paths: BundlingPaths,
   options: BundlingOptions,
 ): Promise<webpack.Configuration> {
-  const { checksEnabled, isDev } = options;
+  const { checksEnabled, isDev, frontendConfig } = options;
 
   const { plugins, loaders } = transforms(options);
 
-  const baseUrl = options.config.getString('app.baseUrl');
+  const baseUrl = frontendConfig.getString('app.baseUrl');
   const validBaseUrl = new URL(baseUrl);
 
   if (checksEnabled) {
@@ -99,7 +99,7 @@ export async function createConfig(
 
   plugins.push(
     new webpack.EnvironmentPlugin({
-      APP_CONFIG: options.appConfigs,
+      APP_CONFIG: options.frontendAppConfigs,
     }),
   );
 
@@ -109,9 +109,9 @@ export async function createConfig(
       templateParameters: {
         publicPath: validBaseUrl.pathname.replace(/\/$/, ''),
         app: {
-          title: options.config.getString('app.title'),
+          title: frontendConfig.getString('app.title'),
           baseUrl: validBaseUrl.href,
-          googleAnalyticsTrackingId: options.config.getOptionalString(
+          googleAnalyticsTrackingId: frontendConfig.getOptionalString(
             'app.googleAnalyticsTrackingId',
           ),
         },

packages/cli/src/lib/bundler/server.ts

@@ -23,9 +23,14 @@ import { ServeOptions } from './types';
 import { resolveBundlingPaths } from './paths';
 
 export async function serveBundle(options: ServeOptions) {
-  const url = resolveBaseUrl(options.config);
+  const url = resolveBaseUrl(options.frontendConfig);
 
-  const port = Number(url.port) || (url.protocol === 'https:' ? 443 : 80);
+  const host =
+    options.frontendConfig.getOptionalString('app.listen.host') || url.hostname;
+  const port =
+    options.frontendConfig.getOptionalNumber('app.listen.port') ||
+    Number(url.port) ||
+    (url.protocol === 'https:' ? 443 : 80);
 
   const paths = resolveBundlingPaths(options);
   const pkgPath = paths.targetPackageJson;
@@ -50,7 +55,7 @@ export async function serveBundle(options: ServeOptions) {
     clientLogLevel: 'warning',
     stats: 'errors-warnings',
     https: url.protocol === 'https:',
-    host: url.hostname,
+    host,
     port,
     proxy: pkg.proxy,
   });

packages/cli/src/lib/bundler/types.ts

@@ -17,27 +17,29 @@
 import { AppConfig, Config } from '@backstage/config';
 import { BundlingPathsOptions } from './paths';
 import { ParallelOption } from '../parallel';
+import { ConfigSchema } from '@backstage/config-loader';
 
 export type BundlingOptions = {
   checksEnabled: boolean;
   isDev: boolean;
-  config: Config;
-  appConfigs: AppConfig[];
+  frontendConfig: Config;
+  frontendAppConfigs: AppConfig[];
   baseUrl: URL;
   parallel?: ParallelOption;
 };
 
 export type ServeOptions = BundlingPathsOptions & {
   checksEnabled: boolean;
-  config: Config;
-  appConfigs: AppConfig[];
+  frontendConfig: Config;
+  frontendAppConfigs: AppConfig[];
 };
 
 export type BuildOptions = BundlingPathsOptions & {
   statsJsonEnabled: boolean;
   parallel?: ParallelOption;
-  config: Config;
-  appConfigs: AppConfig[];
+  schema?: ConfigSchema;
+  frontendConfig: Config;
+  frontendAppConfigs: AppConfig[];
 };
 
 export type BackendBundlingOptions = {

packages/cli/src/lib/config.ts

@@ -14,18 +14,23 @@
  * limitations under the License.
  */
 
-import { loadConfig } from '@backstage/config-loader';
+import { loadConfig, loadConfigSchema } from '@backstage/config-loader';
 import { ConfigReader } from '@backstage/config';
 import { paths } from './paths';
 
-export async function loadCliConfig(
-  configArgs: string[],
-  shouldReadSecrets: boolean = false,
-) {
+export async function loadCliConfig(configArgs: string[]) {
   const configPaths = configArgs.map(arg => paths.resolveTarget(arg));
 
+  // Consider all packages in the monorepo when loading in config
+  const LernaProject = require('@lerna/project');
+  const project = new LernaProject(paths.targetDir);
+  const packages = await project.getPackages();
+  const localPackageNames = packages.map((p: any) => p.name);
+  const schema = await loadConfigSchema({
+    dependencies: localPackageNames,
+  });
+
   const appConfigs = await loadConfig({
-    shouldReadSecrets,
     env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'production',
     configRoot: paths.targetRoot,
     configPaths,
@@ -35,8 +40,24 @@ export async function loadCliConfig(
     `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`,
   );
 
-  return {
-    appConfigs,
-    config: ConfigReader.fromConfigs(appConfigs),
-  };
+  try {
+    const frontendAppConfigs = schema.process(appConfigs, {
+      visiblity: ['frontend'],
+    });
+    const frontendConfig = ConfigReader.fromConfigs(frontendAppConfigs);
+
+    return {
+      schema,
+      appConfigs,
+      frontendConfig,
+      frontendAppConfigs,
+    };
+  } catch (error) {
+    const maybeSchemaError = error as Error & { messages?: string[] };
+    if (maybeSchemaError.messages) {
+      const messages = maybeSchemaError.messages.join('\n  ');
+      throw new Error(`Configuration does not match schema\n\n  ${messages}`);
+    }
+    throw error;
+  }
 }

packages/cli/src/lib/diff/handlers.ts

@@ -61,7 +61,7 @@ class PackageJsonHandler {
       await this.syncField('main:src');
     }
     await this.syncField('types');
-    await this.syncField('files');
+    await this.syncFiles();
     await this.syncScripts();
     await this.syncPublishConfig();
     await this.syncDependencies('dependencies');
@@ -105,6 +105,15 @@ class PackageJsonHandler {
     }
   }
 
+  private async syncFiles() {
+    if (typeof this.targetPkg.configSchema === 'string') {
+      const files = [...this.pkg.files, this.targetPkg.configSchema];
+      await this.syncField('files', { files });
+    } else {
+      await this.syncField('files');
+    }
+  }
+
   private async syncScripts() {
     const pkgScripts = this.pkg.scripts;
     const targetScripts = (this.targetPkg.scripts =

packages/config-loader/package.json

@@ -30,13 +30,20 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@backstage/cli-common": "^0.1.1",
     "@backstage/config": "^0.1.1",
+    "ajv": "^6.12.5",
     "fs-extra": "^9.0.0",
+    "json-schema": "^0.2.5",
+    "json-schema-merge-allof": "^0.7.0",
+    "typescript-json-schema": "^0.43.0",
     "yaml": "^1.9.2",
     "yup": "^0.29.3"
   },
   "devDependencies": {
     "@types/jest": "^26.0.7",
+    "@types/json-schema": "^7.0.6",
+    "@types/json-schema-merge-allof": "^0.6.0",
     "@types/mock-fs": "^4.10.0",
     "@types/node": "^12.0.0",
     "@types/yup": "^0.29.8",

packages/config-loader/src/index.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
-export { readEnvConfig } from './lib';
+export { readEnvConfig, loadConfigSchema } from './lib';
+export type { ConfigSchema, ConfigVisibility } from './lib';
 export { loadConfig } from './loader';
 export type { LoadConfigOptions } from './loader';

packages/config-loader/src/lib/index.ts

@@ -17,3 +17,4 @@
 export { readConfigFile } from './reader';
 export { readEnvConfig } from './env';
 export { readSecret } from './secrets';
+export * from './schema';

packages/config-loader/src/lib/reader.test.ts

@@ -28,7 +28,6 @@ function memoryFiles(files: { [path: string]: string }) {
 
 const mockContext: ReaderContext = {
   env: {},
-  skip: () => false,
   readFile: jest.fn(),
   readSecret: jest.fn(),
 };
@@ -179,22 +178,4 @@ describe('readConfigFile', () => {
 
     await expect(config).rejects.toThrow('Invalid secret at .app: NOPE');
   });
-
-  it('should omit skipped values', async () => {
-    const readFile = memoryFiles({
-      './app-config.yaml': 'app: { title: skip, name: include }',
-    });
-
-    const config = readConfigFile('./app-config.yaml', {
-      ...mockContext,
-      readFile,
-      skip: (path: string) => path === '.app.title',
-      readSecret: jest.fn() as ReadSecretFunc,
-    });
-
-    await expect(config).resolves.toEqual({
-      context: 'app-config.yaml',
-      data: { app: { name: 'include' } },
-    });
-  });
 });

packages/config-loader/src/lib/reader.ts

@@ -37,10 +37,6 @@ export async function readConfigFile(
     obj: JsonValue,
     path: string,
   ): Promise<JsonValue | undefined> {
-    if (ctx.skip(path)) {
-      return undefined;
-    }
-
     if (typeof obj !== 'object') {
       return obj;
     } else if (obj === null) {

packages/config-loader/src/lib/schema/collect.test.ts

@@ -0,0 +1,229 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import mockFs from 'mock-fs';
+import { collectConfigSchemas } from './collect';
+
+const mockSchema = {
+  type: 'object',
+  properties: {
+    key: {
+      type: 'string',
+      visibility: 'frontend',
+    },
+  },
+};
+
+describe('collectConfigSchemas', () => {
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should not find any schemas without packages', async () => {
+    mockFs({
+      'lerna.json': JSON.stringify({
+        packages: ['packages/*'],
+      }),
+    });
+
+    await expect(collectConfigSchemas([])).resolves.toEqual([]);
+  });
+
+  it('should find schema in a local package', async () => {
+    mockFs({
+      node_modules: {
+        a: {
+          'package.json': JSON.stringify({
+            name: 'a',
+            configSchema: mockSchema,
+          }),
+        },
+      },
+    });
+
+    await expect(collectConfigSchemas(['a'])).resolves.toEqual([
+      {
+        path: 'node_modules/a/package.json',
+        value: mockSchema,
+      },
+    ]);
+  });
+
+  it('should find schema in transitive dependencies', async () => {
+    mockFs({
+      node_modules: {
+        a: {
+          'package.json': JSON.stringify({
+            name: 'a',
+            dependencies: { b: '0.0.0', '@backstage/mock': '0.0.0' },
+          }),
+        },
+        b: {
+          'package.json': JSON.stringify({
+            name: 'b',
+            dependencies: {
+              c1: '0.0.0',
+              c2: '0.0.0',
+              '@backstage/mock': '0.0.0',
+            },
+            configSchema: { ...mockSchema, title: 'b' },
+          }),
+        },
+        c1: {
+          'package.json': JSON.stringify({
+            name: 'c1',
+            dependencies: { d1: '0.0.0' },
+            configSchema: { ...mockSchema, title: 'c1' },
+          }),
+        },
+        c2: {
+          'package.json': JSON.stringify({
+            name: 'c2',
+            dependencies: { d2: '0.0.0' },
+          }),
+        },
+        d1: {
+          'package.json': JSON.stringify({
+            name: 'd1',
+            dependencies: {},
+            configSchema: { ...mockSchema, title: 'd1' },
+          }),
+        },
+        d2: {
+          'package.json': JSON.stringify({
+            name: 'd2',
+            dependencies: {},
+            configSchema: { ...mockSchema, title: 'd2' },
+          }),
+        },
+      },
+    });
+
+    await expect(collectConfigSchemas(['a'])).resolves.toEqual([
+      {
+        path: 'node_modules/b/package.json',
+        value: { ...mockSchema, title: 'b' },
+      },
+      {
+        path: 'node_modules/c1/package.json',
+        value: { ...mockSchema, title: 'c1' },
+      },
+      {
+        path: 'node_modules/d1/package.json',
+        value: { ...mockSchema, title: 'd1' },
+      },
+    ]);
+  });
+
+  it('should schema of different types', async () => {
+    mockFs({
+      node_modules: {
+        a: {
+          'package.json': JSON.stringify({
+            name: 'a',
+            configSchema: { ...mockSchema, title: 'inline' },
+          }),
+        },
+        b: {
+          'package.json': JSON.stringify({
+            name: 'b',
+            configSchema: 'schema.json',
+          }),
+          'schema.json': JSON.stringify({ ...mockSchema, title: 'external' }),
+        },
+        c: {
+          'package.json': JSON.stringify({
+            name: 'c',
+            configSchema: 'schema.d.ts',
+          }),
+          'schema.d.ts': `export interface Config {
+              /** @visibility secret */
+              tsKey: string
+            }`,
+        },
+      },
+      // TypeScript compilation needs to load some real files inside the typescript dir
+      '../../node_modules/typescript': (mockFs as any).load(
+        '../../node_modules/typescript',
+      ),
+    });
+
+    await expect(collectConfigSchemas(['a', 'b', 'c'])).resolves.toEqual([
+      {
+        path: 'node_modules/a/package.json',
+        value: { ...mockSchema, title: 'inline' },
+      },
+      {
+        path: 'node_modules/b/schema.json',
+        value: { ...mockSchema, title: 'external' },
+      },
+      {
+        path: 'node_modules/c/schema.d.ts',
+        value: {
+          $schema: 'http://json-schema.org/draft-07/schema#',
+          type: 'object',
+          properties: {
+            tsKey: {
+              type: 'string',
+              visibility: 'secret',
+            },
+          },
+          required: ['tsKey'],
+        },
+      },
+    ]);
+  });
+
+  it('should not allow unknown schema file types', async () => {
+    mockFs({
+      node_modules: {
+        a: {
+          'package.json': JSON.stringify({
+            name: 'a',
+            configSchema: 'schema.yaml',
+          }),
+          'schema.yaml': mockSchema,
+        },
+      },
+    });
+
+    await expect(collectConfigSchemas(['a'])).rejects.toThrow(
+      'Config schema files must be .json or .d.ts, got schema.yaml',
+    );
+  });
+
+  it('should reject typescript config declaration without a Config type', async () => {
+    mockFs({
+      node_modules: {
+        a: {
+          'package.json': JSON.stringify({
+            name: 'a',
+            configSchema: 'schema.d.ts',
+          }),
+          'schema.d.ts': `export interface NotConfig {}`,
+        },
+      },
+      // TypeScript compilation needs to load some real files inside the typescript dir
+      '../../node_modules/typescript': (mockFs as any).load(
+        '../../node_modules/typescript',
+      ),
+    });
+
+    await expect(collectConfigSchemas(['a'])).rejects.toThrow(
+      'Invalid schema in node_modules/a/schema.d.ts, missing Config export',
+    );
+  });
+});

packages/config-loader/src/lib/schema/collect.ts

@@ -0,0 +1,180 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import {
+  resolve as resolvePath,
+  relative as relativePath,
+  dirname,
+  sep,
+} from 'path';
+import { ConfigSchemaPackageEntry } from './types';
+import { getProgramFromFiles, generateSchema } from 'typescript-json-schema';
+import { JsonObject } from '@backstage/config';
+
+type Item = {
+  name: string;
+  parentPath?: string;
+};
+
+const req =
+  typeof __non_webpack_require__ === 'undefined'
+    ? require
+    : __non_webpack_require__;
+
+/**
+ * This collects all known config schemas across all dependencies of the app.
+ */
+export async function collectConfigSchemas(
+  packageNames: string[],
+): Promise<ConfigSchemaPackageEntry[]> {
+  const visitedPackages = new Set<string>();
+  const schemas = Array<ConfigSchemaPackageEntry>();
+  const tsSchemaPaths = Array<string>();
+  const currentDir = await fs.realpath(process.cwd());
+
+  async function processItem({ name, parentPath }: Item) {
+    // Ensures that we only process each package once. We don't bother with
+    // loading in schemas from duplicates of different versions, as that's not
+    // supported by Backstage right now anyway. We may want to change that in
+    // the future though, if it for example becomes possible to load in two
+    // different versions of e.g. @backstage/core at once.
+    if (visitedPackages.has(name)) {
+      return;
+    }
+    visitedPackages.add(name);
+
+    let pkgPath: string;
+    try {
+      pkgPath = req.resolve(
+        `${name}/package.json`,
+        parentPath && {
+          paths: [parentPath],
+        },
+      );
+    } catch {
+      // We can somewhat safely ignore packages that don't export package.json,
+      // as they are likely not part of the Backstage ecosystem anyway.
+      return;
+    }
+
+    const pkg = await fs.readJson(pkgPath);
+    const depNames = [
+      ...Object.keys(pkg.dependencies ?? {}),
+      ...Object.keys(pkg.peerDependencies ?? {}),
+    ];
+
+    // TODO(Rugvip): Trying this out to avoid having to traverse the full dependency graph,
+    //               since that's pretty slow. We probably need a better way to determine when
+    //               we've left the Backstage ecosystem, but this will do for now.
+    const hasSchema = 'configSchema' in pkg;
+    const hasBackstageDep = depNames.some(_ => _.startsWith('@backstage/'));
+    if (!hasSchema && !hasBackstageDep) {
+      return;
+    }
+    if (hasSchema) {
+      if (typeof pkg.configSchema === 'string') {
+        const isJson = pkg.configSchema.endsWith('.json');
+        const isDts = pkg.configSchema.endsWith('.d.ts');
+        if (!isJson && !isDts) {
+          throw new Error(
+            `Config schema files must be .json or .d.ts, got ${pkg.configSchema}`,
+          );
+        }
+        if (isDts) {
+          tsSchemaPaths.push(
+            relativePath(
+              currentDir,
+              resolvePath(dirname(pkgPath), pkg.configSchema),
+            ),
+          );
+        } else {
+          const path = resolvePath(dirname(pkgPath), pkg.configSchema);
+          const value = await fs.readJson(path);
+          schemas.push({
+            value,
+            path: relativePath(currentDir, path),
+          });
+        }
+      } else {
+        schemas.push({
+          value: pkg.configSchema,
+          path: relativePath(currentDir, pkgPath),
+        });
+      }
+    }
+
+    await Promise.all(
+      depNames.map(name => processItem({ name, parentPath: pkgPath })),
+    );
+  }
+
+  await Promise.all(packageNames.map(name => processItem({ name })));
+
+  const tsSchemas = compileTsSchemas(tsSchemaPaths);
+
+  return schemas.concat(tsSchemas);
+}
+
+// This handles the support of TypeScript .d.ts config schema declarations.
+// We collect all typescript schema definition and compile them all in one go.
+// This is much faster than compiling them separately.
+function compileTsSchemas(paths: string[]) {
+  if (paths.length === 0) {
+    return [];
+  }
+
+  const program = getProgramFromFiles(paths, {
+    incremental: false,
+    isolatedModules: true,
+    lib: ['ES5'], // Skipping most libs speeds processing up a lot, we just need the primitive types anyway
+    noEmit: true,
+    noResolve: true,
+    skipLibCheck: true, // Skipping lib checks speeds things up
+    skipDefaultLibCheck: true,
+    strict: true,
+    typeRoots: [], // Do not include any additional types
+    types: [],
+  });
+
+  const tsSchemas = paths.map(path => {
+    let value;
+    try {
+      value = generateSchema(
+        program,
+        // All schemas should export a `Config` symbol
+        'Config',
+        // This enables usage of @visibility is doc comments
+        {
+          required: true,
+          validationKeywords: ['visibility'],
+        },
+        [path.split(sep).join('/')], // Unix paths are expected for all OSes here
+      ) as JsonObject | null;
+    } catch (error) {
+      if (error.message !== 'type Config not found') {
+        throw error;
+      }
+    }
+
+    if (!value) {
+      throw new Error(`Invalid schema in ${path}, missing Config export`);
+    }
+    return { path, value };
+  });
+
+  return tsSchemas;
+}

packages/config-loader/src/lib/schema/compile.test.ts

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { compileConfigSchemas } from './compile';
+
+describe('compileConfigSchemas', () => {
+  it('should merge schemas', () => {
+    const validate = compileConfigSchemas([
+      {
+        path: 'a',
+        value: { type: 'object', properties: { a: { type: 'string' } } },
+      },
+      {
+        path: 'b',
+        value: { type: 'object', properties: { b: { type: 'number' } } },
+      },
+    ]);
+    expect(validate([{ data: { a: 1 }, context: 'test' }])).toEqual({
+      errors: ['Config should be string { type=string } at .a'],
+      visibilityByPath: new Map(),
+    });
+    expect(validate([{ data: { b: 'b' }, context: 'test' }])).toEqual({
+      errors: ['Config should be number { type=number } at .b'],
+      visibilityByPath: new Map(),
+    });
+  });
+
+  it('should discover visibilities', () => {
+    const validate = compileConfigSchemas([
+      {
+        path: 'a1',
+        value: {
+          type: 'object',
+          properties: {
+            a: { type: 'string', visibility: 'frontend' },
+            b: { type: 'string', visibility: 'backend' },
+            c: { type: 'string' },
+            d: {
+              type: 'array',
+              visibility: 'secret',
+              items: { type: 'string', visibility: 'frontend' },
+            },
+          },
+        },
+      },
+      {
+        path: 'a2',
+        value: {
+          type: 'object',
+          properties: {
+            a: { type: 'string' },
+            b: { type: 'string', visibility: 'secret' },
+            c: { type: 'string', visibility: 'backend' },
+            d: {
+              type: 'array',
+              visibility: 'secret',
+              items: { type: 'string' },
+            },
+          },
+        },
+      },
+    ]);
+    expect(
+      validate([
+        { data: { a: 'a', b: 'b', c: 'c', d: ['d'] }, context: 'test' },
+      ]),
+    ).toEqual({
+      visibilityByPath: new Map(
+        Object.entries({
+          '.a': 'frontend',
+          '.b': 'secret',
+          '.d': 'secret',
+          '.d.0': 'frontend',
+        }),
+      ),
+    });
+  });
+
+  it('should reject visiblity conflicts', () => {
+    expect(() =>
+      compileConfigSchemas([
+        {
+          path: 'a1',
+          value: {
+            type: 'object',
+            properties: { a: { type: 'string', visibility: 'frontend' } },
+          },
+        },
+        {
+          path: 'a2',
+          value: {
+            type: 'object',
+            properties: { a: { type: 'string', visibility: 'secret' } },
+          },
+        },
+      ]),
+    ).toThrow(
+      "Config schema visibility is both 'frontend' and 'secret' for properties/a/visibility",
+    );
+  });
+});

packages/config-loader/src/lib/schema/compile.ts

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ajv from 'ajv';
+import { JSONSchema7 as JSONSchema } from 'json-schema';
+import mergeAllOf, { Resolvers } from 'json-schema-merge-allof';
+import { ConfigReader } from '@backstage/config';
+import {
+  ConfigSchemaPackageEntry,
+  ValidationFunc,
+  CONFIG_VISIBILITIES,
+  ConfigVisibility,
+} from './types';
+
+/**
+ * This takes a collection of Backstage configuration schemas from various
+ * sources and compiles them down into a single schema validation function.
+ *
+ * It also handles the implementation of the custom "visibility" keyword used
+ * to specify the scope of different config paths.
+ */
+export function compileConfigSchemas(
+  schemas: ConfigSchemaPackageEntry[],
+): ValidationFunc {
+  // The ajv instance below is stateful and doesn't really allow for additional
+  // output during validation. We work around this by having this extra piece
+  // of state that we reset before each validation.
+  const visibilityByPath = new Map<string, ConfigVisibility>();
+
+  const ajv = new Ajv({
+    allErrors: true,
+    schemas: {
+      'https://backstage.io/schema/config-v1': true,
+    },
+  }).addKeyword('visibility', {
+    metaSchema: {
+      type: 'string',
+      enum: CONFIG_VISIBILITIES,
+    },
+    compile(visibility: ConfigVisibility) {
+      return (_data, dataPath) => {
+        if (!dataPath) {
+          return false;
+        }
+        if (visibility && visibility !== 'backend') {
+          const normalizedPath = dataPath.replace(
+            /\['?(.*?)'?\]/g,
+            (_, segment) => `.${segment}`,
+          );
+          visibilityByPath.set(normalizedPath, visibility);
+        }
+        return true;
+      };
+    },
+  });
+
+  const merged = mergeAllOf(
+    { allOf: schemas.map(_ => _.value) },
+    {
+      // JSONSchema is typically subtractive, as in it always reduces the set of allowed
+      // inputs through constraints. This changes the object property merging to be additive
+      // rather than subtractive.
+      ignoreAdditionalProperties: true,
+      resolvers: {
+        // This ensures that the visibilities across different schemas are sound, and
+        // selects the most specific visibility for each path.
+        visibility(values: string[], path: string[]) {
+          const hasFrontend = values.some(_ => _ === 'frontend');
+          const hasSecret = values.some(_ => _ === 'secret');
+          if (hasFrontend && hasSecret) {
+            throw new Error(
+              `Config schema visibility is both 'frontend' and 'secret' for ${path.join(
+                '/',
+              )}`,
+            );
+          } else if (hasFrontend) {
+            return 'frontend';
+          } else if (hasSecret) {
+            return 'secret';
+          }
+
+          return 'backend';
+        },
+      } as Partial<Resolvers<JSONSchema>>,
+    },
+  );
+
+  const validate = ajv.compile(merged);
+
+  return configs => {
+    const config = ConfigReader.fromConfigs(configs).get();
+
+    visibilityByPath.clear();
+
+    const valid = validate(config);
+    if (!valid) {
+      const errors = validate.errors ?? [];
+      return {
+        errors: errors.map(({ dataPath, message, params }) => {
+          const paramStr = Object.entries(params)
+            .map(([name, value]) => `${name}=${value}`)
+            .join(' ');
+          return `Config ${message || ''} { ${paramStr} } at ${dataPath}`;
+        }),
+        visibilityByPath: new Map(),
+      };
+    }
+
+    return {
+      visibilityByPath: new Map(visibilityByPath),
+    };
+  };
+}

packages/config-loader/src/lib/schema/filtering.test.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { JsonObject } from '@backstage/config';
+import { ConfigVisibility } from './types';
+import { filterByVisibility } from './filtering';
+
+const data = {
+  arr: ['f', 'b', 's'],
+  objArr: [
+    { f: 1, b: 2, s: 3 },
+    { f: 4, b: 5, s: 6 },
+  ],
+  obj: {
+    f: 'a',
+    b: {
+      s: true,
+    },
+  },
+  arrF: [{ never: 'here' }],
+  arrB: [{ never: 'here' }],
+  arrS: [{ never: 'here' }],
+  objF: { never: 'here' },
+  objB: { never: 'here' },
+  objS: { never: 'here' },
+};
+
+const visiblity = new Map<string, ConfigVisibility>(
+  Object.entries({
+    '.arr.0': 'frontend',
+    '.arr.1': 'backend',
+    '.arr.2': 'secret',
+    '.obj.f': 'frontend',
+    '.obj.b': 'backend',
+    '.obj.b.s': 'secret',
+    '.objArr.0.f': 'frontend',
+    '.objArr.0.b': 'backend',
+    '.objArr.0.s': 'secret',
+    '.objArr.1.f': 'frontend',
+    '.objArr.1.b': 'backend',
+    '.objArr.1.s': 'secret',
+    '.arrF': 'frontend',
+    '.arrB': 'backend',
+    '.arrS': 'secret',
+    '.objF': 'frontend',
+    '.objB': 'backend',
+    '.objS': 'secret',
+  }),
+);
+
+describe('filterByVisibility', () => {
+  test.each<[ConfigVisibility[], JsonObject]>([
+    [[], {}],
+    [
+      ['frontend'],
+      {
+        arr: ['f'],
+        objArr: [{ f: 1 }, { f: 4 }],
+        obj: { f: 'a' },
+        arrF: [],
+        objF: {},
+      },
+    ],
+    [
+      ['backend'],
+      {
+        arr: ['b'],
+        objArr: [{ b: 2 }, { b: 5 }],
+        obj: { b: {} },
+        arrF: [{ never: 'here' }],
+        arrB: [{ never: 'here' }],
+        arrS: [{ never: 'here' }],
+        objF: { never: 'here' },
+        objB: { never: 'here' },
+        objS: { never: 'here' },
+      },
+    ],
+    [
+      ['secret'],
+      {
+        arr: ['s'],
+        objArr: [{ s: 3 }, { s: 6 }],
+        obj: { b: { s: true } },
+        arrS: [],
+        objS: {},
+      },
+    ],
+    [['frontend', 'backend', 'secret'], data],
+  ])('should filter correctly with %p', (filter, expected) => {
+    expect(filterByVisibility(data, filter, visiblity)).toEqual(expected);
+  });
+});

packages/config-loader/src/lib/schema/filtering.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { JsonObject, JsonValue } from '@backstage/config';
+import {
+  ConfigVisibility,
+  DEFAULT_CONFIG_VISIBILITY,
+  TransformFunc,
+} from './types';
+
+/**
+ * This filters data by visibility by discovering the visibility of each
+ * value, and then only keeping the ones that are specified in `includeVisibilities`.
+ */
+export function filterByVisibility(
+  data: JsonObject,
+  includeVisibilities: ConfigVisibility[],
+  visibilityByPath: Map<string, ConfigVisibility>,
+  transformFunc?: TransformFunc<number | string | boolean>,
+): JsonObject {
+  function transform(jsonVal: JsonValue, path: string): JsonValue | undefined {
+    const visibility = visibilityByPath.get(path) ?? DEFAULT_CONFIG_VISIBILITY;
+    const isVisible = includeVisibilities.includes(visibility);
+
+    if (typeof jsonVal !== 'object') {
+      if (isVisible) {
+        if (transformFunc) {
+          return transformFunc(jsonVal, { visibility });
+        }
+        return jsonVal;
+      }
+      return undefined;
+    } else if (jsonVal === null) {
+      return undefined;
+    } else if (Array.isArray(jsonVal)) {
+      const arr = new Array<JsonValue>();
+
+      for (const [index, value] of jsonVal.entries()) {
+        const out = transform(value, `${path}.${index}`);
+        if (out !== undefined) {
+          arr.push(out);
+        }
+      }
+
+      if (arr.length > 0 || isVisible) {
+        return arr;
+      }
+      return undefined;
+    }
+
+    const outObj: JsonObject = {};
+    let hasOutput = false;
+
+    for (const [key, value] of Object.entries(jsonVal)) {
+      if (value === undefined) {
+        continue;
+      }
+      const out = transform(value, `${path}.${key}`);
+      if (out !== undefined) {
+        outObj[key] = out;
+        hasOutput = true;
+      }
+    }
+
+    if (hasOutput || isVisible) {
+      return outObj;
+    }
+    return undefined;
+  }
+
+  return (transform(data, '') as JsonObject) ?? {};
+}

packages/config-loader/src/lib/schema/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { loadConfigSchema } from './load';
+export type { ConfigSchema, ConfigVisibility } from './types';

packages/config-loader/src/lib/schema/load.test.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import mockFs from 'mock-fs';
+import { loadConfigSchema } from './load';
+
+describe('loadConfigSchema', () => {
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should load schema from packages or data', async () => {
+    mockFs({
+      node_modules: {
+        a: {
+          'package.json': JSON.stringify({
+            name: 'a',
+            configSchema: {
+              type: 'object',
+              properties: {
+                key1: { type: 'string', visibility: 'frontend' },
+              },
+            },
+          }),
+        },
+        b: {
+          'package.json': JSON.stringify({
+            name: 'b',
+            configSchema: 'schema.json',
+          }),
+          'schema.json': JSON.stringify({
+            name: 'a',
+            configSchema: {
+              type: 'object',
+              properties: {
+                key2: { type: 'number' },
+              },
+            },
+          }),
+        },
+      },
+    });
+
+    const schema = await loadConfigSchema({
+      dependencies: ['a'],
+    });
+
+    const configs = [{ data: { key1: 'a', key2: 2 }, context: 'test' }];
+
+    expect(schema.process(configs)).toEqual(configs);
+    expect(schema.process(configs, { visiblity: ['frontend'] })).toEqual([
+      { data: { key1: 'a' }, context: 'test' },
+    ]);
+    expect(
+      schema.process(configs, {
+        visiblity: ['frontend'],
+        valueTransform: () => 'X',
+      }),
+    ).toEqual([{ data: { key1: 'X' }, context: 'test' }]);
+    expect(
+      schema.process(configs, {
+        valueTransform: () => 'X',
+      }),
+    ).toEqual([{ data: { key1: 'X', key2: 'X' }, context: 'test' }]);
+
+    const serialized = schema.serialize();
+
+    const schema2 = await loadConfigSchema({ serialized });
+    expect(schema2.process(configs, { visiblity: ['frontend'] })).toEqual([
+      { data: { key1: 'a' }, context: 'test' },
+    ]);
+    expect(() =>
+      schema2.process([...configs, { data: { key1: 3 }, context: 'test2' }]),
+    ).toThrow(
+      'Config validation failed, Config should be string { type=string } at .key1',
+    );
+
+    await expect(
+      loadConfigSchema({
+        serialized: { ...serialized, backstageConfigSchemaVersion: 2 },
+      }),
+    ).rejects.toThrow(
+      'Serialized configuration schema is invalid or has an invalid version number',
+    );
+  });
+});

packages/config-loader/src/lib/schema/load.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AppConfig, JsonObject } from '@backstage/config';
+import { compileConfigSchemas } from './compile';
+import { collectConfigSchemas } from './collect';
+import { filterByVisibility } from './filtering';
+import {
+  ConfigSchema,
+  ConfigSchemaPackageEntry,
+  CONFIG_VISIBILITIES,
+} from './types';
+
+type Options =
+  | {
+      dependencies: string[];
+    }
+  | {
+      serialized: JsonObject;
+    };
+
+/**
+ * Loads config schema for a Backstage instance.
+ */
+export async function loadConfigSchema(
+  options: Options,
+): Promise<ConfigSchema> {
+  let schemas: ConfigSchemaPackageEntry[];
+
+  if ('dependencies' in options) {
+    schemas = await collectConfigSchemas(options.dependencies);
+  } else {
+    const { serialized } = options;
+    if (serialized?.backstageConfigSchemaVersion !== 1) {
+      throw new Error(
+        'Serialized configuration schema is invalid or has an invalid version number',
+      );
+    }
+    schemas = serialized.schemas as ConfigSchemaPackageEntry[];
+  }
+
+  const validate = compileConfigSchemas(schemas);
+
+  return {
+    process(
+      configs: AppConfig[],
+      { visiblity, valueTransform } = {},
+    ): AppConfig[] {
+      const result = validate(configs);
+      if (result.errors) {
+        const error = new Error(
+          `Config validation failed, ${result.errors.join('; ')}`,
+        );
+        (error as any).messages = result.errors;
+        throw error;
+      }
+
+      let processedConfigs = configs;
+
+      if (visiblity) {
+        processedConfigs = processedConfigs.map(({ data, context }) => ({
+          context,
+          data: filterByVisibility(
+            data,
+            visiblity,
+            result.visibilityByPath,
+            valueTransform,
+          ),
+        }));
+      } else if (valueTransform) {
+        processedConfigs = processedConfigs.map(({ data, context }) => ({
+          context,
+          data: filterByVisibility(
+            data,
+            Array.from(CONFIG_VISIBILITIES),
+            result.visibilityByPath,
+            valueTransform,
+          ),
+        }));
+      }
+
+      return processedConfigs;
+    },
+    serialize(): JsonObject {
+      return {
+        schemas,
+        backstageConfigSchemaVersion: 1,
+      };
+    },
+  };
+}

packages/config-loader/src/lib/schema/types.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AppConfig, JsonObject } from '@backstage/config';
+
+/**
+ * An sub-set of configuration schema.
+ */
+export type ConfigSchemaPackageEntry = {
+  /**
+   * The configuration schema itself.
+   */
+  value: JsonObject;
+  /**
+   * The relative path that the configuration schema was discovered at.
+   */
+  path: string;
+};
+
+/**
+ * A list of all possible configuration value visibilities.
+ */
+export const CONFIG_VISIBILITIES = ['frontend', 'backend', 'secret'] as const;
+
+/**
+ * A type representing the possible configuration value visibilities
+ */
+export type ConfigVisibility = typeof CONFIG_VISIBILITIES[number];
+
+/**
+ * The default configuration visibility if no other values is given.
+ */
+export const DEFAULT_CONFIG_VISIBILITY: ConfigVisibility = 'backend';
+
+/**
+ * An explanation of a configuration validation error.
+ */
+type ValidationError = string;
+
+/**
+ * The result of validating configuration data using a schema.
+ */
+type ValidationResult = {
+  /**
+   * Errors that where emitted during validation, if any.
+   */
+  errors?: ValidationError[];
+  /**
+   * The configuration visibilities that were discovered during validation.
+   *
+   * The path in the key uses the form `/<key>/<sub-key>/<array-index>/<leaf-key>`
+   */
+  visibilityByPath: Map<string, ConfigVisibility>;
+};
+
+/**
+ * A function used validate configuration data.
+ */
+export type ValidationFunc = (configs: AppConfig[]) => ValidationResult;
+
+/**
+ * A function used to transform primitive configuration values.
+ */
+export type TransformFunc<T extends number | string | boolean> = (
+  value: T,
+  context: { visibility: ConfigVisibility },
+) => T | undefined;
+
+/**
+ * Options used to process configuration data with a schema.
+ */
+type ConfigProcessingOptions = {
+  /**
+   * The visibilities that should be included in the output data.
+   * If omitted, the data will not be filtered by visibility.
+   */
+  visiblity?: ConfigVisibility[];
+
+  /**
+   * A transform function that can be used to transform primitive configuration values
+   * during validation. The value returned from the transform function will be used
+   * instead of the original value. If the transform returns `undefined`, the value
+   * will be omitted.
+   */
+  valueTransform?: TransformFunc<any>;
+};
+
+/**
+ * A loaded configuration schema that is ready to process configuration data.
+ */
+export type ConfigSchema = {
+  process(
+    appConfigs: AppConfig[],
+    options?: ConfigProcessingOptions,
+  ): AppConfig[];
+
+  serialize(): JsonObject;
+};

packages/config-loader/src/lib/secrets.test.ts

@@ -21,7 +21,6 @@ const ctx: ReaderContext = {
   env: {
     SECRET: 'my-secret',
   },
-  skip: () => false,
   readSecret: jest.fn(),
   async readFile(path) {
     const content = ({

packages/config-loader/src/lib/types.ts

@@ -28,7 +28,6 @@ export type SkipFunc = (path: string) => boolean;
  */
 export type ReaderContext = {
   env: { [name in string]?: string };
-  skip: SkipFunc;
   readFile: ReadFileFunc;
   readSecret: ReadSecretFunc;
 };

packages/config-loader/src/loader.test.ts

@@ -44,47 +44,6 @@ describe('loadConfig', () => {
         configRoot: '/root',
         configPaths: [],
         env: 'production',
-        shouldReadSecrets: false,
-      }),
-    ).resolves.toEqual([
-      {
-        context: 'app-config.yaml',
-        data: {
-          app: {
-            title: 'Example App',
-          },
-        },
-      },
-    ]);
-  });
-
-  it('loads config without secrets', async () => {
-    await expect(
-      loadConfig({
-        configRoot: '/root',
-        configPaths: ['/root/app-config.yaml'],
-        env: 'production',
-        shouldReadSecrets: false,
-      }),
-    ).resolves.toEqual([
-      {
-        context: 'app-config.yaml',
-        data: {
-          app: {
-            title: 'Example App',
-          },
-        },
-      },
-    ]);
-  });
-
-  it('loads config with secrets', async () => {
-    await expect(
-      loadConfig({
-        configRoot: '/root',
-        configPaths: ['/root/app-config.yaml'],
-        env: 'production',
-        shouldReadSecrets: true,
       }),
     ).resolves.toEqual([
       {
@@ -99,16 +58,12 @@ describe('loadConfig', () => {
     ]);
   });
 
-  it('loads development config without secrets', async () => {
+  it('loads config with secrets', async () => {
     await expect(
       loadConfig({
         configRoot: '/root',
-        configPaths: [
-          '/root/app-config.yaml',
-          '/root/app-config.development.yaml',
-        ],
-        env: 'development',
-        shouldReadSecrets: false,
+        configPaths: ['/root/app-config.yaml'],
+        env: 'production',
       }),
     ).resolves.toEqual([
       {
@@ -116,15 +71,10 @@ describe('loadConfig', () => {
         data: {
           app: {
             title: 'Example App',
+            sessionKey: 'abc123',
           },
         },
       },
-      {
-        context: 'app-config.development.yaml',
-        data: {
-          app: {},
-        },
-      },
     ]);
   });
 
@@ -137,7 +87,6 @@ describe('loadConfig', () => {
           '/root/app-config.development.yaml',
         ],
         env: 'development',
-        shouldReadSecrets: true,
       }),
     ).resolves.toEqual([
       {

packages/config-loader/src/loader.ts

@@ -28,18 +28,13 @@ export type LoadConfigOptions = {
 
   // TODO(Rugvip): This will be removed in the future, but for now we use it to warn about possible mistakes.
   env: string;
-
-  // Whether to read secrets or omit them, defaults to false.
-  shouldReadSecrets?: boolean;
 };
 
 class Context {
   constructor(
     private readonly options: {
-      secretPaths: Set<string>;
       env: { [name in string]?: string };
       rootPath: string;
-      shouldReadSecrets: boolean;
     },
   ) {}
 
@@ -47,26 +42,14 @@ class Context {
     return this.options.env;
   }
 
-  skip(path: string): boolean {
-    if (this.options.shouldReadSecrets) {
-      return false;
-    }
-    return this.options.secretPaths.has(path);
-  }
-
   async readFile(path: string): Promise<string> {
     return fs.readFile(resolvePath(this.options.rootPath, path), 'utf8');
   }
 
   async readSecret(
-    path: string,
+    _path: string,
     desc: JsonObject,
   ): Promise<string | undefined> {
-    this.options.secretPaths.add(path);
-    if (!this.options.shouldReadSecrets) {
-      return undefined;
-    }
-
     return readSecret(desc, this);
   }
 }
@@ -100,8 +83,6 @@ export async function loadConfig(
   }
 
   try {
-    const secretPaths = new Set<string>();
-
     for (const configPath of configPaths) {
       if (!isAbsolute(configPath)) {
         throw new Error(`Config load path is not absolute: '${configPath}'`);
@@ -109,10 +90,8 @@ export async function loadConfig(
       const config = await readConfigFile(
         configPath,
         new Context({
-          secretPaths,
           env: process.env,
           rootPath: dirname(configPath),
-          shouldReadSecrets: Boolean(options.shouldReadSecrets),
         }),
       );
 

packages/core/config.d.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface Config {
+  /**
+   * Generic frontend configuration.
+   */
+  app: {
+    /**
+     * The public absolute root URL that the frontend.
+     * @visibility frontend
+     */
+    baseUrl: string;
+
+    /**
+     * The title of the app.
+     * @visibility frontend
+     */
+    title?: string;
+  };
+
+  /**
+   * Generic backend configuration.
+   */
+  backend: {
+    /**
+     * The public absolute root URL that the backend is reachable at.
+     * @visibility frontend
+     */
+    baseUrl: string;
+  };
+
+  /**
+   * Configuration that provides information about the organization that the app is for.
+   */
+  organization?: {
+    /**
+     * The name of the organization that the app belongs to.
+     * @visibility frontend
+     */
+    name?: string;
+  };
+
+  homepage?: {
+    clocks?: {
+      /** @visibility frontend */
+      label: string;
+      /** @visibility frontend */
+      timezone: string;
+    }[];
+  };
+}

packages/core/package.json

@@ -79,6 +79,8 @@
     "@types/zen-observable": "^0.8.0"
   },
   "files": [
-    "dist"
-  ]
+    "dist",
+    "config.d.ts"
+  ],
+  "configSchema": "config.d.ts"
 }

packages/core/src/api-wrappers/createApp.tsx

@@ -61,7 +61,11 @@ export const defaultConfigLoader: AppConfigLoader = async (
   if (runtimeConfigJson !== '__app_injected_runtime_config__'.toUpperCase()) {
     try {
       const data = JSON.parse(runtimeConfigJson) as JsonObject;
-      configs.push({ data, context: 'env' });
+      if (Array.isArray(data)) {
+        configs.push(...data);
+      } else {
+        configs.push({ data, context: 'env' });
+      }
     } catch (error) {
       throw new Error(`Failed to load runtime configuration, ${error}`);
     }

plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx

@@ -26,24 +26,26 @@ import { ApiExplorerPage } from './ApiExplorerPage';
 describe('ApiCatalogPage', () => {
   const catalogApi: Partial<CatalogApi> = {
     getEntities: () =>
-      Promise.resolve([
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'API',
-          metadata: {
-            name: 'Entity1',
+      Promise.resolve({
+        items: [
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'API',
+            metadata: {
+              name: 'Entity1',
+            },
+            spec: { type: 'openapi' },
           },
-          spec: { type: 'openapi' },
-        },
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'API',
-          metadata: {
-            name: 'Entity2',
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'API',
+            metadata: {
+              name: 'Entity2',
+            },
+            spec: { type: 'openapi' },
           },
-          spec: { type: 'openapi' },
-        },
-      ] as Entity[]),
+        ] as Entity[],
+      }),
     getLocationByEntity: () =>
       Promise.resolve({ id: 'id', type: 'github', target: 'url' }),
   };

plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.tsx

@@ -25,8 +25,8 @@ import { ApiExplorerLayout } from './ApiExplorerLayout';
 
 export const ApiExplorerPage = () => {
   const catalogApi = useApi(catalogApiRef);
-  const { loading, error, value: matchingEntities } = useAsync(() => {
-    return catalogApi.getEntities({ kind: 'API' });
+  const { loading, error, value: catalogResponse } = useAsync(() => {
+    return catalogApi.getEntities({ filter: { kind: 'API' } });
   }, [catalogApi]);
 
   return (
@@ -44,7 +44,7 @@ export const ApiExplorerPage = () => {
           <SupportButton>All your APIs</SupportButton>
         </ContentHeader>
         <ApiExplorerTable
-          entities={matchingEntities!}
+          entities={catalogResponse?.items ?? []}
           loading={loading}
           error={error}
         />

plugins/app-backend/package.json

@@ -22,6 +22,7 @@
   "dependencies": {
     "@backstage/backend-common": "^0.2.0",
     "@backstage/config-loader": "^0.2.0",
+    "@backstage/config": "^0.1.1",
     "@types/express": "^4.17.6",
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",

plugins/app-backend/src/lib/config.test.ts

@@ -17,7 +17,7 @@
 import fs from 'fs-extra';
 import { resolve as resolvePath } from 'path';
 import { getVoidLogger } from '@backstage/backend-common';
-import { injectEnvConfig } from './config';
+import { injectConfig } from './config';
 
 jest.mock('fs-extra');
 
@@ -29,12 +29,12 @@ const readFileMock = (fsMock.readFile as unknown) as jest.MockedFunction<
 const MOCK_DIR = 'mock-dir';
 
 const baseOptions = {
-  env: {},
+  appConfigs: [],
   staticDir: MOCK_DIR,
   logger: getVoidLogger(),
 };
 
-describe('injectEnvConfig', () => {
+describe('injectConfig', () => {
   beforeEach(() => {
     fsMock.readdir.mockResolvedValue(['main.js']);
   });
@@ -43,11 +43,23 @@ describe('injectEnvConfig', () => {
     jest.resetAllMocks();
   });
 
-  it('should not inject without config', async () => {
-    await injectEnvConfig(baseOptions);
-    expect(fsMock.readdir).toHaveBeenCalledTimes(0);
-    expect(fsMock.readFile).toHaveBeenCalledTimes(0);
-    expect(fsMock.writeFile).toHaveBeenCalledTimes(0);
+  it('should inject without config', async () => {
+    fsMock.readdir.mockResolvedValue(['main.js']);
+    readFileMock.mockImplementation(
+      async () => '"__APP_INJECTED_RUNTIME_CONFIG__"',
+    );
+    await injectConfig(baseOptions);
+    expect(fsMock.readdir).toHaveBeenCalledTimes(1);
+    expect(fsMock.readFile).toHaveBeenCalledTimes(1);
+    expect(fsMock.writeFile).toHaveBeenCalledTimes(1);
+    expect(fsMock.writeFile).toHaveBeenCalledWith(
+      resolvePath(MOCK_DIR, 'main.js'),
+      '/*__APP_INJECTED_CONFIG_MARKER__*/"[]"/*__INJECTED_END__*/',
+      'utf8',
+    );
+
+    // eslint-disable-next-line no-eval
+    expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual([]);
   });
 
   it('should find the correct file to inject', async () => {
@@ -64,7 +76,10 @@ describe('injectEnvConfig', () => {
       return 'NO_PLACEHOLDER_HERE';
     });
 
-    await injectEnvConfig({ ...baseOptions, env: { APP_CONFIG_x: '0' } });
+    await injectConfig({
+      ...baseOptions,
+      appConfigs: [{ data: { x: 0 }, context: 'test' }],
+    });
     expect(fsMock.readFile).toHaveBeenCalledTimes(2);
     expect(fsMock.readFile).toHaveBeenNthCalledWith(
       1,
@@ -80,14 +95,19 @@ describe('injectEnvConfig', () => {
     expect(fsMock.writeFile).toHaveBeenCalledTimes(1);
     expect(fsMock.writeFile).toHaveBeenCalledWith(
       resolvePath(MOCK_DIR, 'main.js'),
-      '/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/',
+      '/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":0},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/',
       'utf8',
     );
 
     // eslint-disable-next-line no-eval
-    expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual({
-      x: 0,
-    });
+    expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual([
+      {
+        data: {
+          x: 0,
+        },
+        context: 'test',
+      },
+    ]);
   });
 
   it('should re-inject config', async () => {
@@ -96,41 +116,40 @@ describe('injectEnvConfig', () => {
       'JSON.parse("__APP_INJECTED_RUNTIME_CONFIG__")',
     );
 
-    await injectEnvConfig({
+    await injectConfig({
       ...baseOptions,
-      env: {
-        APP_CONFIG_x: '0',
-      },
+      appConfigs: [{ data: { x: 0 }, context: 'test' }],
     });
 
     expect(fsMock.writeFile).toHaveBeenCalledTimes(1);
     expect(fsMock.writeFile).toHaveBeenCalledWith(
       resolvePath(MOCK_DIR, 'main.js'),
-      'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/)',
+      'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":0},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/)',
       'utf8',
     );
 
     // eslint-disable-next-line no-eval
-    expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual({ x: 0 });
+    expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual([
+      { data: { x: 0 }, context: 'test' },
+    ]);
 
     readFileMock.mockResolvedValue(fsMock.writeFile.mock.calls[0][1]);
 
-    await injectEnvConfig({
+    await injectConfig({
       ...baseOptions,
-      env: {
-        APP_CONFIG_x: '1',
-        APP_CONFIG_y: '2',
-      },
+      appConfigs: [{ data: { x: 1, y: 2 }, context: 'test' }],
     });
 
     expect(fsMock.writeFile).toHaveBeenCalledTimes(2);
     expect(fsMock.writeFile).toHaveBeenLastCalledWith(
       resolvePath(MOCK_DIR, 'main.js'),
-      'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":1,\\"y\\":2}"/*__INJECTED_END__*/)',
+      'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"[{\\"data\\":{\\"x\\":1,\\"y\\":2},\\"context\\":\\"test\\"}]"/*__INJECTED_END__*/)',
       'utf8',
     );
 
     // eslint-disable-next-line no-eval
-    expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual({ x: 1, y: 2 });
+    expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual([
+      { data: { x: 1, y: 2 }, context: 'test' },
+    ]);
   });
 });

plugins/app-backend/src/lib/config.ts

@@ -16,34 +16,27 @@
 
 import fs from 'fs-extra';
 import { resolve as resolvePath } from 'path';
-import { readEnvConfig } from '@backstage/config-loader';
 import { Logger } from 'winston';
+import { AppConfig, Config, JsonObject } from '@backstage/config';
+import { loadConfigSchema, readEnvConfig } from '@backstage/config-loader';
 
-type Options = {
-  // Environment to read config from
-  env: { [name: string]: string | undefined };
+type InjectOptions = {
+  appConfigs: AppConfig[];
   // Directory of the static JS files to search for file to inject
   staticDir: string;
   logger: Logger;
 };
 
 /**
- * Injects config from APP_CONFIG_ env vars, replacing existing
- * injected config if it has already been injected.
+ * Injects configs into the app bundle, replacing any existing injected config.
  */
-export async function injectEnvConfig(options: Options) {
-  const { env, staticDir, logger } = options;
-
-  const envConfig = readEnvConfig(env);
-  if (envConfig.length === 0) {
-    return;
-  }
+export async function injectConfig(options: InjectOptions) {
+  const { staticDir, logger, appConfigs } = options;
 
   const files = await fs.readdir(staticDir);
   const jsFiles = files.filter(file => file.endsWith('.js'));
 
-  const [{ data }] = envConfig;
-  const escapedData = JSON.stringify(data).replace(/("|'|\\)/g, '\\$1');
+  const escapedData = JSON.stringify(appConfigs).replace(/("|'|\\)/g, '\\$1');
   const injected = `/*__APP_INJECTED_CONFIG_MARKER__*/"${escapedData}"/*__INJECTED_END__*/`;
 
   for (const jsFile of jsFiles) {
@@ -72,3 +65,33 @@ export async function injectEnvConfig(options: Options) {
   }
   logger.info('Env config not injected');
 }
+
+type ReadOptions = {
+  env: { [name: string]: string | undefined };
+  appDistDir: string;
+  config: Config;
+};
+
+/**
+ * Read config from environment and process the backend config using the
+ * schema that is embedded in the frontend build.
+ */
+export async function readConfigs(options: ReadOptions): Promise<AppConfig[]> {
+  const { env, appDistDir, config } = options;
+
+  const appConfigs = readEnvConfig(env);
+
+  const schemaPath = resolvePath(appDistDir, '.config-schema.json');
+  if (await fs.pathExists(schemaPath)) {
+    const serializedSchema = await fs.readJson(schemaPath);
+    const schema = await loadConfigSchema({ serialized: serializedSchema });
+
+    const frontendConfigs = await schema.process(
+      [{ data: config.get() as JsonObject, context: 'app' }],
+      { visiblity: ['frontend'] },
+    );
+    appConfigs.push(...frontendConfigs);
+  }
+
+  return appConfigs;
+}

plugins/app-backend/src/service/router.test.ts

@@ -16,13 +16,16 @@
 
 import { resolve as resolvePath } from 'path';
 import { getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
 import express from 'express';
 import Router from 'express-promise-router';
 import request from 'supertest';
-
 import { createRouter } from './router';
 
-jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() }));
+jest.mock('../lib/config', () => ({
+  injectConfig: jest.fn(),
+  readConfigs: jest.fn(),
+}));
 
 global.__non_webpack_require__ = {
   /* eslint-disable-next-line no-restricted-syntax */
@@ -35,6 +38,7 @@ describe('createRouter', () => {
   beforeAll(async () => {
     const router = await createRouter({
       logger: getVoidLogger(),
+      config: new ConfigReader({}),
       appPackageName: 'example-app',
     });
     app = express().use(router);
@@ -76,6 +80,7 @@ describe('createRouter with static fallback handler', () => {
 
     const router = await createRouter({
       logger: getVoidLogger(),
+      config: new ConfigReader({}),
       appPackageName: 'example-app',
       staticFallbackHandler,
     });

plugins/app-backend/src/service/router.ts

@@ -15,13 +15,15 @@
  */
 
 import { resolve as resolvePath } from 'path';
-import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common';
 import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
-import { injectEnvConfig } from '../lib/config';
+import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+import { injectConfig, readConfigs } from '../lib/config';
 
 export interface RouterOptions {
+  config: Config;
   logger: Logger;
   appPackageName: string;
   staticFallbackHandler?: express.Handler;
@@ -30,22 +32,27 @@ export interface RouterOptions {
 export async function createRouter(
   options: RouterOptions,
 ): Promise<express.Router> {
-  const appDistDir = resolvePackagePath(options.appPackageName, 'dist');
-  options.logger.info(`Serving static app content from ${appDistDir}`);
+  const { config, logger, appPackageName, staticFallbackHandler } = options;
 
-  await injectEnvConfig({
+  const appDistDir = resolvePackagePath(appPackageName, 'dist');
+  logger.info(`Serving static app content from ${appDistDir}`);
+  const staticDir = resolvePath(appDistDir, 'static');
+
+  const appConfigs = await readConfigs({
+    config,
+    appDistDir,
     env: process.env,
-    logger: options.logger,
-    staticDir: resolvePath(appDistDir, 'static'),
   });
 
+  await injectConfig({ appConfigs, logger, staticDir });
+
   const router = Router();
 
   // Use a separate router for static content so that a fallback can be provided by backend
   const staticRouter = Router();
   staticRouter.use(express.static(resolvePath(appDistDir, 'static')));
-  if (options.staticFallbackHandler) {
-    staticRouter.use(options.staticFallbackHandler);
+  if (staticFallbackHandler) {
+    staticRouter.use(staticFallbackHandler);
   }
   staticRouter.use(notFoundHandler());
 

plugins/app-backend/src/service/standaloneServer.ts

@@ -14,14 +14,16 @@
  * limitations under the License.
  */
 
-import { createServiceBuilder } from '@backstage/backend-common';
 import { Server } from 'http';
 import { Logger } from 'winston';
+import { createServiceBuilder } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
 import { createRouter } from './router';
 
 export interface ServerOptions {
   port: number;
   enableCors: boolean;
+  config: Config;
   logger: Logger;
 }
 
@@ -32,6 +34,7 @@ export async function startStandaloneServer(
   logger.debug('Starting application server...');
   const router = await createRouter({
     logger,
+    config: options.config,
     appPackageName: 'example-app',
   });
 

plugins/catalog/src/components/CatalogFilter/CatalogFilter.test.tsx

@@ -33,30 +33,32 @@ import { ButtonGroup, CatalogFilter } from './CatalogFilter';
 describe('Catalog Filter', () => {
   const catalogApi: Partial<CatalogApi> = {
     getEntities: () =>
-      Promise.resolve([
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity1',
+      Promise.resolve({
+        items: [
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity1',
+            },
+            spec: {
+              owner: 'tools@example.com',
+              type: 'service',
+            },
           },
-          spec: {
-            owner: 'tools@example.com',
-            type: 'service',
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity2',
+            },
+            spec: {
+              owner: 'not-tools@example.com',
+              type: 'service',
+            },
           },
-        },
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity2',
-          },
-          spec: {
-            owner: 'not-tools@example.com',
-            type: 'service',
-          },
-        },
-      ] as Entity[]),
+        ] as Entity[],
+      }),
   };
 
   const identityApi: Partial<IdentityApi> = {

plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx

@@ -34,37 +34,39 @@ import { CatalogPage } from './CatalogPage';
 describe('CatalogPage', () => {
   const catalogApi: Partial<CatalogApi> = {
     getEntities: () =>
-      Promise.resolve([
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity1',
+      Promise.resolve({
+        items: [
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity1',
+            },
+            spec: {
+              owner: 'tools@example.com',
+              type: 'service',
+            },
           },
-          spec: {
-            owner: 'tools@example.com',
-            type: 'service',
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity2',
+            },
+            spec: {
+              owner: 'not-tools@example.com',
+              type: 'service',
+            },
           },
-        },
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity2',
-          },
-          spec: {
-            owner: 'not-tools@example.com',
-            type: 'service',
-          },
-        },
-      ] as Entity[]),
+        ] as Entity[],
+      }),
     getLocationByEntity: () =>
       Promise.resolve({ id: 'id', type: 'github', target: 'url' }),
   };
   const testProfile: Partial<ProfileInfo> = {
     displayName: 'Display Name',
   };
-  const indentityApi: Partial<IdentityApi> = {
+  const identityApi: Partial<IdentityApi> = {
     getUserId: () => 'tools@example.com',
     getProfile: () => testProfile,
   };
@@ -75,7 +77,7 @@ describe('CatalogPage', () => {
         <ApiProvider
           apis={ApiRegistry.from([
             [catalogApiRef, catalogApi],
-            [identityApiRef, indentityApi],
+            [identityApiRef, identityApi],
             [storageApiRef, MockStorageApi.create()],
           ])}
         >

plugins/catalog/src/components/ResultsFilter/ResultsFilter.test.tsx

@@ -33,46 +33,48 @@ import { ResultsFilter } from './ResultsFilter';
 describe('Results Filter', () => {
   const catalogApi: Partial<CatalogApi> = {
     getEntities: () =>
-      Promise.resolve([
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity1',
-            tags: ['java'],
+      Promise.resolve({
+        items: [
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity1',
+              tags: ['java'],
+            },
+            spec: {
+              owner: 'tools@example.com',
+              type: 'service',
+            },
           },
-          spec: {
-            owner: 'tools@example.com',
-            type: 'service',
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity2',
+            },
+            spec: {
+              owner: 'not-tools@example.com',
+              type: 'service',
+            },
           },
-        },
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity2',
+          {
+            apiVersion: 'backstage.io/v1alpha1',
+            kind: 'Component',
+            metadata: {
+              name: 'Entity3',
+              tags: ['java', 'test'],
+            },
+            spec: {
+              owner: 'tools@example.com',
+              type: 'service',
+            },
           },
-          spec: {
-            owner: 'not-tools@example.com',
-            type: 'service',
-          },
-        },
-        {
-          apiVersion: 'backstage.io/v1alpha1',
-          kind: 'Component',
-          metadata: {
-            name: 'Entity3',
-            tags: ['java', 'test'],
-          },
-          spec: {
-            owner: 'tools@example.com',
-            type: 'service',
-          },
-        },
-      ] as Entity[]),
+        ] as Entity[],
+      }),
   };
 
-  const indentityApi: Partial<IdentityApi> = {
+  const identityApi: Partial<IdentityApi> = {
     getUserId: () => 'tools@example.com',
   };
 
@@ -82,7 +84,7 @@ describe('Results Filter', () => {
         <ApiProvider
           apis={ApiRegistry.from([
             [catalogApiRef, catalogApi],
-            [identityApiRef, indentityApi],
+            [identityApiRef, identityApi],
             [storageApiRef, MockStorageApi.create()],
           ])}
         >

plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx

@@ -44,9 +44,13 @@ function useColocatedEntities(entity: Entity): AsyncState<Entity[]> {
   const catalogApi = useApi(catalogApiRef);
   return useAsync(async () => {
     const myLocation = entity.metadata.annotations?.[LOCATION_ANNOTATION];
-    return myLocation
-      ? await catalogApi.getEntities({ [LOCATION_ANNOTATION]: myLocation })
-      : [];
+    if (!myLocation) {
+      return [];
+    }
+    const response = await catalogApi.getEntities({
+      filter: { [LOCATION_ANNOTATION]: myLocation },
+    });
+    return response.items;
   }, [catalogApi, entity]);
 }
 

plugins/catalog/src/filter/EntityFilterGroupsProvider.tsx

@@ -46,9 +46,12 @@ export const EntityFilterGroupsProvider = ({
 // The hook that implements the actual context building
 function useProvideEntityFilters(): FilterGroupsContext {
   const catalogApi = useApi(catalogApiRef);
-  const [{ value: entities, error }, doReload] = useAsyncFn(() =>
-    catalogApi.getEntities({ kind: 'Component' }),
-  );
+  const [{ value: entities, error }, doReload] = useAsyncFn(async () => {
+    const response = await catalogApi.getEntities({
+      filter: { kind: 'Component' },
+    });
+    return response.items;
+  });
 
   const filterGroups = useRef<{
     [filterGroupId: string]: FilterGroup;

plugins/catalog/src/filter/useEntityFilterGroup.test.tsx

@@ -49,7 +49,7 @@ describe('useEntityFilterGroup', () => {
   });
 
   it('works for an empty set of filters', async () => {
-    catalogApi.getEntities.mockResolvedValue([]);
+    catalogApi.getEntities.mockResolvedValue({ items: [] });
     const group: FilterGroup = { filters: {} };
     const { result, waitFor } = renderHook(
       () => useEntityFilterGroup('g1', group),
@@ -60,13 +60,15 @@ describe('useEntityFilterGroup', () => {
   });
 
   it('works for a single group', async () => {
-    catalogApi.getEntities.mockResolvedValue([
-      {
-        apiVersion: 'backstage.io/v1alpha1',
-        kind: 'Component',
-        metadata: { name: 'n' },
-      },
-    ]);
+    catalogApi.getEntities.mockResolvedValue({
+      items: [
+        {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Component',
+          metadata: { name: 'n' },
+        },
+      ],
+    });
     const group: FilterGroup = {
       filters: {
         f1: e => e.metadata.name === 'n',

plugins/cost-insights/config.d.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+interface Config {
+  costInsights: {
+    /**
+     * @visibility frontend
+     */
+    engineerCost: number;
+
+    products: {
+      [kind: string]: {
+        /**
+         * @visibility frontend
+         */
+        name: string;
+
+        /**
+         * @visibility frontend
+         */
+        icon: 'compute' | 'data' | 'database' | 'storage' | 'search' | 'ml';
+      };
+    };
+
+    metrics?: {
+      [kind: string]: {
+        /**
+         * @visibility frontend
+         */
+        name: string;
+
+        /**
+         * @visibility frontend
+         */
+        default?: boolean;
+      };
+    };
+  };
+}

plugins/cost-insights/package.json

@@ -61,6 +61,8 @@
     "msw": "^0.21.2"
   },
   "files": [
-    "dist"
-  ]
+    "dist",
+    "config.d.ts"
+  ],
+  "configSchema": "config.d.ts"
 }

plugins/cost-insights/src/components/BarChart/BarChartTooltip.tsx

@@ -35,7 +35,6 @@ export const BarChartTooltip = ({
   children,
 }: PropsWithChildren<BarChartTooltipProps>) => {
   const classes = useStyles();
-
   return (
     <Box className={classes.tooltip} display="flex" flexDirection="column">
       <Box
@@ -47,14 +46,16 @@ export const BarChartTooltip = ({
         pt={2}
       >
         <Box display="flex" flexDirection="column">
-          <Typography variant="h6">{title}</Typography>
+          <Typography className={classes.truncate} variant="h6">
+            {title}
+          </Typography>
           {subtitle && (
             <Typography className={classes.subtitle} variant="subtitle1">
               {subtitle}
             </Typography>
           )}
         </Box>
-        {topRight}
+        {topRight && <Box ml={1}>{topRight}</Box>}
       </Box>
       {content && (
         <Box px={2} pt={2}>

plugins/cost-insights/src/components/CostInsightsPage/CostInsightsPage.tsx

@@ -16,7 +16,7 @@
 
 import React, { useCallback, useEffect, useState } from 'react';
 import { Box, Container, Divider, Grid, Typography } from '@material-ui/core';
-import { featureFlagsApiRef, Progress, useApi } from '@backstage/core';
+import { Progress, useApi } from '@backstage/core';
 import { default as MaterialAlert } from '@material-ui/lab/Alert';
 import { costInsightsApiRef } from '../../api';
 import { AlertActionCardList } from '../AlertActionCardList';
@@ -49,7 +49,6 @@ import { useSubtleTypographyStyles } from '../../utils/styles';
 
 export const CostInsightsPage = () => {
   const classes = useSubtleTypographyStyles();
-  const featureFlags = useApi(featureFlagsApiRef);
   const client = useApi(costInsightsApiRef);
   const config = useConfig();
   const groups = useGroups();
@@ -193,25 +192,24 @@ export const CostInsightsPage = () => {
       px={3}
       pt={6}
       display="flex"
-      flexDirection="row"
       justifyContent="space-between"
+      alignItems="center"
+      minHeight={40}
     >
-      <Box minHeight={40} width="75%" pt={2}>
+      <Box>
         <Typography variant="h4">Cost Overview</Typography>
         <Typography classes={classes}>
           Billing data as of {lastCompleteBillingDate}
         </Typography>
       </Box>
-      <Box minHeight={40} maxHeight={60} display="flex">
-        {featureFlags.isActive('cost-insights-currencies') && (
-          <Box mr={1}>
-            <CurrencySelect
-              currency={currency}
-              currencies={config.currencies}
-              onSelect={setCurrency}
-            />
-          </Box>
-        )}
+      <Box display="flex">
+        <Box mr={1}>
+          <CurrencySelect
+            currency={currency}
+            currencies={config.currencies}
+            onSelect={setCurrency}
+          />
+        </Box>
         <ProjectSelect
           project={pageFilters.project}
           projects={projects || []}

plugins/cost-insights/src/components/CurrencySelect/CurrencySelect.tsx

@@ -15,7 +15,13 @@
  */
 
 import React from 'react';
-import { MenuItem, Select, SelectProps } from '@material-ui/core';
+import {
+  InputLabel,
+  FormControl,
+  MenuItem,
+  Select,
+  SelectProps,
+} from '@material-ui/core';
 import { Currency, CurrencyType } from '../../types';
 import { findAlways } from '../../utils/assert';
 import { useSelectStyles as useStyles } from '../../utils/styles';
@@ -51,24 +57,28 @@ export const CurrencySelect = ({
   };
 
   return (
-    <Select
-      className={classes.select}
-      variant="outlined"
-      onChange={handleOnChange}
-      value={currency.kind || NULL_VALUE}
-      renderValue={renderValue}
-    >
-      {currencies.map((c: Currency) => (
-        <MenuItem
-          className={classes.menuItem}
-          key={c.kind || NULL_VALUE}
-          value={c.kind || NULL_VALUE}
-        >
-          <span role="img" aria-label={c.label}>
-            {c.label}
-          </span>
-        </MenuItem>
-      ))}
-    </Select>
+    <FormControl variant="outlined">
+      <InputLabel shrink>Convert to:</InputLabel>
+      <Select
+        className={classes.select}
+        variant="outlined"
+        labelWidth={100}
+        onChange={handleOnChange}
+        value={currency.kind || NULL_VALUE}
+        renderValue={renderValue}
+      >
+        {currencies.map((c: Currency) => (
+          <MenuItem
+            className={classes.menuItem}
+            key={c.kind || NULL_VALUE}
+            value={c.kind || NULL_VALUE}
+          >
+            <span role="img" aria-label={c.label}>
+              {c.label}
+            </span>
+          </MenuItem>
+        ))}
+      </Select>
+    </FormControl>
   );
 };

plugins/cost-insights/src/utils/styles.ts

@@ -386,7 +386,7 @@ export const useTooltipStyles = makeStyles<CostInsightsTheme>(
         boxShadow: theme.shadows[1],
         color: theme.palette.tooltip.color,
         fontSize: theme.typography.fontSize,
-        width: 250,
+        maxWidth: 300,
       },
       actions: {
         padding: theme.spacing(2),
@@ -403,6 +403,12 @@ export const useTooltipStyles = makeStyles<CostInsightsTheme>(
       divider: {
         backgroundColor: emphasize(theme.palette.divider, 1),
       },
+      truncate: {
+        maxWidth: 200,
+        whiteSpace: 'nowrap',
+        overflow: 'hidden',
+        textOverflow: 'ellipsis',
+      },
       subtitle: {
         fontStyle: 'italic',
       },

plugins/lighthouse/package.json

@@ -52,5 +52,21 @@
   },
   "files": [
     "dist"
-  ]
+  ],
+  "configSchema": {
+    "$schema": "https://backstage.io/schema/config-v1",
+    "title": "@backstage/lighthouse",
+    "type": "object",
+    "properties": {
+      "lighthouse": {
+        "type": "object",
+        "properties": {
+          "baseUrl": {
+            "type": "string",
+            "visibility": "frontend"
+          }
+        }
+      }
+    }
+  }
 }

plugins/rollbar/package.json

@@ -52,5 +52,21 @@
   },
   "files": [
     "dist"
-  ]
+  ],
+  "configSchema": {
+    "$schema": "https://backstage.io/schema/config-v1",
+    "title": "@backstage/rollbar",
+    "type": "object",
+    "properties": {
+      "rollbar": {
+        "type": "object",
+        "properties": {
+          "organization": {
+            "type": "string",
+            "visibility": "frontend"
+          }
+        }
+      }
+    }
+  }
 }

plugins/rollbar/src/components/RollbarHome/RollbarHome.test.tsx

@@ -52,7 +52,7 @@ describe('RollbarHome component', () => {
               catalogApiRef,
               ({
                 async getEntities() {
-                  return [];
+                  return { items: [] };
                 },
               } as Partial<CatalogApi>) as CatalogApi,
             ],

plugins/rollbar/src/hooks/useRollbarEntities.ts

@@ -28,8 +28,8 @@ export function useRollbarEntities() {
     configApi.getString('organization.name');
 
   const { value, loading, error } = useAsync(async () => {
-    const entities = await catalogApi.getEntities();
-    return entities.filter(entity => {
+    const response = await catalogApi.getEntities();
+    return response.items.filter(entity => {
       return !!entity.metadata.annotations?.[ROLLBAR_ANNOTATION];
     });
   }, [catalogApi]);

plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts

@@ -17,10 +17,13 @@
 jest.mock('nodegit');
 jest.mock('azure-devops-node-api/GitApi');
 jest.mock('azure-devops-node-api/interfaces/GitInterfaces');
+jest.mock('./helpers', () => ({
+  pushToRemoteUserPass: jest.fn(),
+}));
 
 import { AzurePublisher } from './azure';
 import { GitApi } from 'azure-devops-node-api/GitApi';
-import * as NodeGit from 'nodegit';
+import { pushToRemoteUserPass } from './helpers';
 
 const { mockGitApi } = require('azure-devops-node-api/GitApi') as {
   mockGitApi: {
@@ -28,25 +31,6 @@ const { mockGitApi } = require('azure-devops-node-api/GitApi') as {
   };
 };
 
-const {
-  Repository,
-  mockRepo,
-  mockIndex,
-  Signature,
-  Remote,
-  mockRemote,
-  Cred,
-} = require('nodegit') as {
-  Repository: jest.Mocked<{ init: any }>;
-  Signature: jest.Mocked<{ now: any }>;
-  Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
-  Remote: jest.Mocked<{ create: any }>;
-
-  mockIndex: jest.Mocked<NodeGit.Index>;
-  mockRepo: jest.Mocked<NodeGit.Repository>;
-  mockRemote: jest.Mocked<NodeGit.Remote>;
-};
-
 describe('Azure Publisher', () => {
   const publisher = new AzurePublisher(new GitApi('', []), 'fake-token');
 
@@ -60,7 +44,7 @@ describe('Azure Publisher', () => {
         remoteUrl: 'mockclone',
       } as { remoteUrl: string });
 
-      await publisher.publish({
+      const result = await publisher.publish({
         values: {
           storePath: 'project/repo',
           owner: 'bob',
@@ -68,109 +52,16 @@ describe('Azure Publisher', () => {
         directory: '/tmp/test',
       });
 
+      expect(result).toEqual({ remoteUrl: 'mockclone' });
       expect(mockGitApi.createRepository).toHaveBeenCalledWith(
         {
           name: 'repo',
         },
         'project',
       );
-    });
-  });
-
-  describe('publish: createGitDirectory', () => {
-    const values = {
-      isOrg: true,
-      storePath: 'blam/test',
-      owner: 'lols',
-    };
-
-    const mockDir = '/tmp/test/dir';
-
-    mockGitApi.createRepository.mockResolvedValue({
-      remoteUrl: 'mockclone',
-    } as { remoteUrl: string });
-
-    it('should call init on the repo with the directory', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
-    });
-
-    it('should call refresh index on the index and write the new files', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(mockRepo.refreshIndex).toHaveBeenCalled();
-    });
-
-    it('should call add all files and write', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(mockIndex.addAll).toHaveBeenCalled();
-      expect(mockIndex.write).toHaveBeenCalled();
-      expect(mockIndex.writeTree).toHaveBeenCalled();
-    });
-
-    it('should create a commit with on head with the right name and commiter', async () => {
-      const mockSignature = { mockSignature: 'bloblly' };
-      Signature.now.mockReturnValue(mockSignature);
-
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(Signature.now).toHaveBeenCalledTimes(2);
-      expect(Signature.now).toHaveBeenCalledWith(
-        'Scaffolder',
-        'scaffolder@backstage.io',
-      );
-
-      expect(mockRepo.createCommit).toHaveBeenCalledWith(
-        'HEAD',
-        mockSignature,
-        mockSignature,
-        'initial commit',
-        'mockoid',
-        [],
-      );
-    });
-
-    it('creates a remote with the repo and remote', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(Remote.create).toHaveBeenCalledWith(
-        mockRepo,
-        'origin',
+      expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+        '/tmp/test',
         'mockclone',
-      );
-    });
-
-    it('shoud push to the remote repo', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      const [remotes, { callbacks }] = mockRemote.push.mock
-        .calls[0] as NodeGit.PushOptions[];
-
-      expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
-
-      callbacks?.credentials?.();
-
-      expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
         'notempty',
         'fake-token',
       );

plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts

@@ -17,10 +17,9 @@
 import { PublisherBase } from './types';
 import { GitApi } from 'azure-devops-node-api/GitApi';
 import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces';
-
+import { pushToRemoteUserPass } from './helpers';
 import { JsonValue } from '@backstage/config';
 import { RequiredTemplateValues } from '../templater';
-import { Repository, Remote, Signature, Cred } from 'nodegit';
 
 export class AzurePublisher implements PublisherBase {
   private readonly client: GitApi;
@@ -39,7 +38,7 @@ export class AzurePublisher implements PublisherBase {
     directory: string;
   }): Promise<{ remoteUrl: string }> {
     const remoteUrl = await this.createRemote(values);
-    await this.pushToRemote(directory, remoteUrl);
+    await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token);
 
     return { remoteUrl };
   }
@@ -54,29 +53,4 @@ export class AzurePublisher implements PublisherBase {
 
     return repo.remoteUrl || '';
   }
-
-  private async pushToRemote(directory: string, remote: string): Promise<void> {
-    const repo = await Repository.init(directory, 0);
-    const index = await repo.refreshIndex();
-    await index.addAll();
-    await index.write();
-    const oid = await index.writeTree();
-    await repo.createCommit(
-      'HEAD',
-      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
-      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
-      'initial commit',
-      oid,
-      [],
-    );
-
-    const remoteRepo = await Remote.create(repo, 'origin', remote);
-
-    await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
-      callbacks: {
-        // Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
-        credentials: () => Cred.userpassPlaintextNew('notempty', this.token),
-      },
-    });
-  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts

@@ -16,15 +16,18 @@
 
 jest.mock('@octokit/rest');
 jest.mock('nodegit');
+jest.mock('./helpers', () => ({
+  pushToRemoteUserPass: jest.fn(),
+}));
 
 import { Octokit } from '@octokit/rest';
-import * as NodeGit from 'nodegit';
 import {
   OctokitResponse,
   ReposCreateInOrgResponseData,
   UsersGetByUsernameResponseData,
 } from '@octokit/types';
 import { GithubPublisher } from './github';
+import { pushToRemoteUserPass } from './helpers';
 
 const { mockGithubClient } = require('@octokit/rest') as {
   mockGithubClient: {
@@ -34,25 +37,6 @@ const { mockGithubClient } = require('@octokit/rest') as {
   };
 };
 
-const {
-  Repository,
-  mockRepo,
-  mockIndex,
-  Signature,
-  Remote,
-  mockRemote,
-  Cred,
-} = require('nodegit') as {
-  Repository: jest.Mocked<{ init: any }>;
-  Signature: jest.Mocked<{ now: any }>;
-  Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
-  Remote: jest.Mocked<{ create: any }>;
-
-  mockIndex: jest.Mocked<NodeGit.Index>;
-  mockRepo: jest.Mocked<NodeGit.Repository>;
-  mockRemote: jest.Mocked<NodeGit.Remote>;
-};
-
 describe('GitHub Publisher', () => {
   beforeEach(() => {
     jest.clearAllMocks();
@@ -72,8 +56,13 @@ describe('GitHub Publisher', () => {
             clone_url: 'mockclone',
           },
         } as OctokitResponse<ReposCreateInOrgResponseData>);
+        mockGithubClient.users.getByUsername.mockResolvedValue({
+          data: {
+            type: 'Organization',
+          },
+        } as OctokitResponse<UsersGetByUsernameResponseData>);
 
-        await publisher.publish({
+        const result = await publisher.publish({
           values: {
             storePath: 'blam/test',
             owner: 'bob',
@@ -82,6 +71,7 @@ describe('GitHub Publisher', () => {
           directory: '/tmp/test',
         });
 
+        expect(result).toEqual({ remoteUrl: 'mockclone' });
         expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({
           org: 'blam',
           name: 'test',
@@ -97,6 +87,12 @@ describe('GitHub Publisher', () => {
           repo: 'test',
           permission: 'admin',
         });
+        expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+          '/tmp/test',
+          'mockclone',
+          'abc',
+          'x-oauth-basic',
+        );
       });
 
       it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => {
@@ -111,7 +107,7 @@ describe('GitHub Publisher', () => {
           },
         } as OctokitResponse<UsersGetByUsernameResponseData>);
 
-        await publisher.publish({
+        const result = await publisher.publish({
           values: {
             storePath: 'blam/test',
             owner: 'bob',
@@ -120,6 +116,7 @@ describe('GitHub Publisher', () => {
           directory: '/tmp/test',
         });
 
+        expect(result).toEqual({ remoteUrl: 'mockclone' });
         expect(
           mockGithubClient.repos.createForAuthenticatedUser,
         ).toHaveBeenCalledWith({
@@ -127,6 +124,12 @@ describe('GitHub Publisher', () => {
           private: false,
         });
         expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled();
+        expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+          '/tmp/test',
+          'mockclone',
+          'abc',
+          'x-oauth-basic',
+        );
       });
     });
 
@@ -142,7 +145,7 @@ describe('GitHub Publisher', () => {
         },
       } as OctokitResponse<UsersGetByUsernameResponseData>);
 
-      await publisher.publish({
+      const result = await publisher.publish({
         values: {
           storePath: 'blam/test',
           owner: 'bob',
@@ -152,6 +155,7 @@ describe('GitHub Publisher', () => {
         directory: '/tmp/test',
       });
 
+      expect(result).toEqual({ remoteUrl: 'mockclone' });
       expect(
         mockGithubClient.repos.createForAuthenticatedUser,
       ).toHaveBeenCalledWith({
@@ -165,113 +169,12 @@ describe('GitHub Publisher', () => {
         username: 'bob',
         permission: 'admin',
       });
-    });
-
-    describe('publish: createGitDirectory', () => {
-      const values = {
-        storePath: 'blam/test',
-        owner: 'lols',
-        access: 'lols',
-      };
-
-      const mockDir = '/tmp/test/dir';
-
-      mockGithubClient.repos.createInOrg.mockResolvedValue({
-        data: {
-          clone_url: 'mockclone',
-        },
-      } as OctokitResponse<ReposCreateInOrgResponseData>);
-      mockGithubClient.users.getByUsername.mockResolvedValue({
-        data: {
-          type: 'Organization',
-        },
-      } as OctokitResponse<UsersGetByUsernameResponseData>);
-
-      it('should call init on the repo with the directory', async () => {
-        await publisher.publish({
-          values,
-          directory: mockDir,
-        });
-
-        expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
-      });
-
-      it('should call refresh index on the index and write the new files', async () => {
-        await publisher.publish({
-          values,
-          directory: mockDir,
-        });
-
-        expect(mockRepo.refreshIndex).toHaveBeenCalled();
-      });
-
-      it('should call add all files and write', async () => {
-        await publisher.publish({
-          values,
-          directory: mockDir,
-        });
-
-        expect(mockIndex.addAll).toHaveBeenCalled();
-        expect(mockIndex.write).toHaveBeenCalled();
-        expect(mockIndex.writeTree).toHaveBeenCalled();
-      });
-
-      it('should create a commit with on head with the right name and commiter', async () => {
-        const mockSignature = { mockSignature: 'bloblly' };
-        Signature.now.mockReturnValue(mockSignature);
-
-        await publisher.publish({
-          values,
-          directory: mockDir,
-        });
-
-        expect(Signature.now).toHaveBeenCalledTimes(2);
-        expect(Signature.now).toHaveBeenCalledWith(
-          'Scaffolder',
-          'scaffolder@backstage.io',
-        );
-
-        expect(mockRepo.createCommit).toHaveBeenCalledWith(
-          'HEAD',
-          mockSignature,
-          mockSignature,
-          'initial commit',
-          'mockoid',
-          [],
-        );
-      });
-
-      it('creates a remote with the repo and remote', async () => {
-        await publisher.publish({
-          values,
-          directory: mockDir,
-        });
-
-        expect(Remote.create).toHaveBeenCalledWith(
-          mockRepo,
-          'origin',
-          'mockclone',
-        );
-      });
-
-      it('shoud push to the remote repo', async () => {
-        await publisher.publish({
-          values,
-          directory: mockDir,
-        });
-
-        const [remotes, { callbacks }] = mockRemote.push.mock
-          .calls[0] as NodeGit.PushOptions[];
-
-        expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
-
-        callbacks?.credentials?.();
-
-        expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
-          'abc',
-          'x-oauth-basic',
-        );
-      });
+      expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+        '/tmp/test',
+        'mockclone',
+        'abc',
+        'x-oauth-basic',
+      );
     });
   });
 
@@ -294,7 +197,7 @@ describe('GitHub Publisher', () => {
         },
       } as OctokitResponse<UsersGetByUsernameResponseData>);
 
-      await publisher.publish({
+      const result = await publisher.publish({
         values: {
           isOrg: true,
           storePath: 'blam/test',
@@ -303,12 +206,19 @@ describe('GitHub Publisher', () => {
         directory: '/tmp/test',
       });
 
+      expect(result).toEqual({ remoteUrl: 'mockclone' });
       expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({
         org: 'blam',
         name: 'test',
         private: true,
         visibility: 'internal',
       });
+      expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+        '/tmp/test',
+        'mockclone',
+        'abc',
+        'x-oauth-basic',
+      );
     });
   });
 
@@ -331,7 +241,7 @@ describe('GitHub Publisher', () => {
         },
       } as OctokitResponse<UsersGetByUsernameResponseData>);
 
-      await publisher.publish({
+      const result = await publisher.publish({
         values: {
           storePath: 'blam/test',
           owner: 'bob',
@@ -339,12 +249,19 @@ describe('GitHub Publisher', () => {
         directory: '/tmp/test',
       });
 
+      expect(result).toEqual({ remoteUrl: 'mockclone' });
       expect(
         mockGithubClient.repos.createForAuthenticatedUser,
       ).toHaveBeenCalledWith({
         name: 'test',
         private: true,
       });
+      expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+        '/tmp/test',
+        'mockclone',
+        'abc',
+        'x-oauth-basic',
+      );
     });
   });
 });

plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts

@@ -16,10 +16,9 @@
 
 import { PublisherBase } from './types';
 import { Octokit } from '@octokit/rest';
-
+import { pushToRemoteUserPass } from './helpers';
 import { JsonValue } from '@backstage/config';
 import { RequiredTemplateValues } from '../templater';
-import { Repository, Remote, Signature, Cred } from 'nodegit';
 
 export type RepoVisibilityOptions = 'private' | 'internal' | 'public';
 
@@ -52,7 +51,12 @@ export class GithubPublisher implements PublisherBase {
     directory: string;
   }): Promise<{ remoteUrl: string }> {
     const remoteUrl = await this.createRemote(values);
-    await this.pushToRemote(directory, remoteUrl);
+    await pushToRemoteUserPass(
+      directory,
+      remoteUrl,
+      this.token,
+      'x-oauth-basic',
+    );
 
     return { remoteUrl };
   }
@@ -104,29 +108,4 @@ export class GithubPublisher implements PublisherBase {
 
     return data?.clone_url;
   }
-
-  private async pushToRemote(directory: string, remote: string): Promise<void> {
-    const repo = await Repository.init(directory, 0);
-    const index = await repo.refreshIndex();
-    await index.addAll();
-    await index.write();
-    const oid = await index.writeTree();
-    await repo.createCommit(
-      'HEAD',
-      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
-      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
-      'initial commit',
-      oid,
-      [],
-    );
-
-    const remoteRepo = await Remote.create(repo, 'origin', remote);
-    await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
-      callbacks: {
-        credentials: () => {
-          return Cred.userpassPlaintextNew(this.token, 'x-oauth-basic');
-        },
-      },
-    });
-  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts

@@ -16,11 +16,14 @@
 
 jest.mock('nodegit');
 jest.mock('@gitbeaker/node');
+jest.mock('./helpers', () => ({
+  pushToRemoteUserPass: jest.fn(),
+}));
 
 import { GitlabPublisher } from './gitlab';
 import { Gitlab as GitlabAPI } from '@gitbeaker/core';
 import { Gitlab } from '@gitbeaker/node';
-import * as NodeGit from 'nodegit';
+import { pushToRemoteUserPass } from './helpers';
 
 const { mockGitlabClient } = require('@gitbeaker/node') as {
   mockGitlabClient: {
@@ -30,25 +33,6 @@ const { mockGitlabClient } = require('@gitbeaker/node') as {
   };
 };
 
-const {
-  Repository,
-  mockRepo,
-  mockIndex,
-  Signature,
-  Remote,
-  mockRemote,
-  Cred,
-} = require('nodegit') as {
-  Repository: jest.Mocked<{ init: any }>;
-  Signature: jest.Mocked<{ now: any }>;
-  Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
-  Remote: jest.Mocked<{ create: any }>;
-
-  mockIndex: jest.Mocked<NodeGit.Index>;
-  mockRepo: jest.Mocked<NodeGit.Repository>;
-  mockRemote: jest.Mocked<NodeGit.Remote>;
-};
-
 describe('GitLab Publisher', () => {
   const publisher = new GitlabPublisher(new Gitlab({}), 'fake-token');
 
@@ -61,8 +45,11 @@ describe('GitLab Publisher', () => {
       mockGitlabClient.Namespaces.show.mockResolvedValue({
         id: 42,
       } as { id: number });
+      mockGitlabClient.Projects.create.mockResolvedValue({
+        http_url_to_repo: 'mockclone',
+      } as { http_url_to_repo: string });
 
-      await publisher.publish({
+      const result = await publisher.publish({
         values: {
           isOrg: true,
           storePath: 'blam/test',
@@ -71,10 +58,17 @@ describe('GitLab Publisher', () => {
         directory: '/tmp/test',
       });
 
+      expect(result).toEqual({ remoteUrl: 'mockclone' });
       expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({
         namespace_id: 42,
         name: 'test',
       });
+      expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+        '/tmp/test',
+        'mockclone',
+        'oauth2',
+        'fake-token',
+      );
     });
 
     it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => {
@@ -86,7 +80,7 @@ describe('GitLab Publisher', () => {
         http_url_to_repo: 'mockclone',
       } as { http_url_to_repo: string });
 
-      await publisher.publish({
+      const result = await publisher.publish({
         values: {
           storePath: 'blam/test',
           owner: 'bob',
@@ -94,109 +88,15 @@ describe('GitLab Publisher', () => {
         directory: '/tmp/test',
       });
 
+      expect(result).toEqual({ remoteUrl: 'mockclone' });
       expect(mockGitlabClient.Users.current).toHaveBeenCalled();
-
       expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({
         namespace_id: 21,
         name: 'test',
       });
-    });
-  });
-
-  describe('publish: createGitDirectory', () => {
-    const values = {
-      isOrg: true,
-      storePath: 'blam/test',
-      owner: 'lols',
-    };
-
-    const mockDir = '/tmp/test/dir';
-
-    mockGitlabClient.Projects.create.mockResolvedValue({
-      http_url_to_repo: 'mockclone',
-    } as { http_url_to_repo: string });
-
-    it('should call init on the repo with the directory', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
-    });
-
-    it('should call refresh index on the index and write the new files', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(mockRepo.refreshIndex).toHaveBeenCalled();
-    });
-
-    it('should call add all files and write', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(mockIndex.addAll).toHaveBeenCalled();
-      expect(mockIndex.write).toHaveBeenCalled();
-      expect(mockIndex.writeTree).toHaveBeenCalled();
-    });
-
-    it('should create a commit with on head with the right name and commiter', async () => {
-      const mockSignature = { mockSignature: 'bloblly' };
-      Signature.now.mockReturnValue(mockSignature);
-
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(Signature.now).toHaveBeenCalledTimes(2);
-      expect(Signature.now).toHaveBeenCalledWith(
-        'Scaffolder',
-        'scaffolder@backstage.io',
-      );
-
-      expect(mockRepo.createCommit).toHaveBeenCalledWith(
-        'HEAD',
-        mockSignature,
-        mockSignature,
-        'initial commit',
-        'mockoid',
-        [],
-      );
-    });
-
-    it('creates a remote with the repo and remote', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      expect(Remote.create).toHaveBeenCalledWith(
-        mockRepo,
-        'origin',
+      expect(pushToRemoteUserPass).toHaveBeenCalledWith(
+        '/tmp/test',
         'mockclone',
-      );
-    });
-
-    it('shoud push to the remote repo', async () => {
-      await publisher.publish({
-        values,
-        directory: mockDir,
-      });
-
-      const [remotes, { callbacks }] = mockRemote.push.mock
-        .calls[0] as NodeGit.PushOptions[];
-
-      expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
-
-      callbacks?.credentials?.();
-
-      expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
         'oauth2',
         'fake-token',
       );

plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts

@@ -16,10 +16,9 @@
 
 import { PublisherBase } from './types';
 import { Gitlab } from '@gitbeaker/core';
-
+import { pushToRemoteUserPass } from './helpers';
 import { JsonValue } from '@backstage/config';
 import { RequiredTemplateValues } from '../templater';
-import { Repository, Remote, Signature, Cred } from 'nodegit';
 
 export class GitlabPublisher implements PublisherBase {
   private readonly client: Gitlab;
@@ -38,7 +37,7 @@ export class GitlabPublisher implements PublisherBase {
     directory: string;
   }): Promise<{ remoteUrl: string }> {
     const remoteUrl = await this.createRemote(values);
-    await this.pushToRemote(directory, remoteUrl);
+    await pushToRemoteUserPass(directory, remoteUrl, 'oauth2', this.token);
 
     return { remoteUrl };
   }
@@ -63,28 +62,4 @@ export class GitlabPublisher implements PublisherBase {
 
     return project?.http_url_to_repo;
   }
-
-  private async pushToRemote(directory: string, remote: string): Promise<void> {
-    const repo = await Repository.init(directory, 0);
-    const index = await repo.refreshIndex();
-    await index.addAll();
-    await index.write();
-    const oid = await index.writeTree();
-    await repo.createCommit(
-      'HEAD',
-      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
-      Signature.now('Scaffolder', 'scaffolder@backstage.io'),
-      'initial commit',
-      oid,
-      [],
-    );
-
-    const remoteRepo = await Remote.create(repo, 'origin', remote);
-
-    await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
-      callbacks: {
-        credentials: () => Cred.userpassPlaintextNew('oauth2', this.token),
-      },
-    });
-  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+jest.mock('nodegit');
+import * as NodeGit from 'nodegit';
+import { pushToRemoteCred } from './helpers';
+
+const {
+  Repository,
+  mockRepo,
+  mockIndex,
+  Signature,
+  Remote,
+  mockRemote,
+  Cred,
+} = require('nodegit') as {
+  Repository: jest.Mocked<{ init: any }>;
+  Signature: jest.Mocked<{ now: any }>;
+  Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
+  Remote: jest.Mocked<{ create: any }>;
+
+  mockIndex: jest.Mocked<NodeGit.Index>;
+  mockRepo: jest.Mocked<NodeGit.Repository>;
+  mockRemote: jest.Mocked<NodeGit.Remote>;
+};
+
+describe('pushToRemoteCred', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  const directory = '/tmp/test/dir';
+  const remote = 'mockclone';
+  const credentialsProvider = () =>
+    NodeGit.Cred.userpassPlaintextNew('username', 'password');
+
+  it('should call init on the repo with the directory', async () => {
+    await pushToRemoteCred(directory, remote, credentialsProvider);
+
+    expect(Repository.init).toHaveBeenCalledWith(directory, 0);
+  });
+
+  it('should call refresh index on the index and write the new files', async () => {
+    await pushToRemoteCred(directory, remote, credentialsProvider);
+
+    expect(mockRepo.refreshIndex).toHaveBeenCalled();
+  });
+
+  it('should call add all files and write', async () => {
+    await pushToRemoteCred(directory, remote, credentialsProvider);
+
+    expect(mockIndex.addAll).toHaveBeenCalled();
+    expect(mockIndex.write).toHaveBeenCalled();
+    expect(mockIndex.writeTree).toHaveBeenCalled();
+  });
+
+  it('should create a commit with on head with the right name and commiter', async () => {
+    const mockSignature = { mockSignature: 'bloblly' };
+    Signature.now.mockReturnValue(mockSignature);
+
+    await pushToRemoteCred(directory, remote, credentialsProvider);
+
+    expect(Signature.now).toHaveBeenCalledTimes(2);
+    expect(Signature.now).toHaveBeenCalledWith(
+      'Scaffolder',
+      'scaffolder@backstage.io',
+    );
+
+    expect(mockRepo.createCommit).toHaveBeenCalledWith(
+      'HEAD',
+      mockSignature,
+      mockSignature,
+      'initial commit',
+      'mockoid',
+      [],
+    );
+  });
+
+  it('creates a remote with the repo and remote', async () => {
+    await pushToRemoteCred(directory, remote, credentialsProvider);
+
+    expect(Remote.create).toHaveBeenCalledWith(mockRepo, 'origin', 'mockclone');
+  });
+
+  it('shoud push to the remote repo', async () => {
+    await pushToRemoteCred(directory, remote, credentialsProvider);
+
+    const [remotes, { callbacks }] = mockRemote.push.mock
+      .calls[0] as NodeGit.PushOptions[];
+
+    expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
+
+    callbacks?.credentials?.();
+
+    expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
+      'username',
+      'password',
+    );
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Repository, Remote, Signature, Cred } from 'nodegit';
+
+export async function pushToRemoteCred(
+  directory: string,
+  remote: string,
+  credentialsProvider: () => Cred,
+): Promise<void> {
+  const repo = await Repository.init(directory, 0);
+  const index = await repo.refreshIndex();
+  await index.addAll();
+  await index.write();
+  const oid = await index.writeTree();
+  await repo.createCommit(
+    'HEAD',
+    Signature.now('Scaffolder', 'scaffolder@backstage.io'),
+    Signature.now('Scaffolder', 'scaffolder@backstage.io'),
+    'initial commit',
+    oid,
+    [],
+  );
+
+  const remoteRepo = await Remote.create(repo, 'origin', remote);
+
+  await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
+    callbacks: {
+      credentials: credentialsProvider,
+    },
+  });
+}
+
+export async function pushToRemoteUserPass(
+  directory: string,
+  remote: string,
+  username: string,
+  password: string,
+): Promise<void> {
+  return pushToRemoteCred(directory, remote, () =>
+    Cred.userpassPlaintextNew(username, password),
+  );
+}

plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx

@@ -53,10 +53,12 @@ export const ScaffolderPage = () => {
 
   const { data: templates, isValidating, error } = useStaleWhileRevalidate(
     'templates/all',
-    async () =>
-      catalogApi.getEntities({ kind: 'Template' }) as Promise<
-        TemplateEntityV1alpha1[]
-      >,
+    async () => {
+      const response = await catalogApi.getEntities({
+        filter: { kind: 'Template' },
+      });
+      return response.items as TemplateEntityV1alpha1[];
+    },
   );
 
   useEffect(() => {

plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx

@@ -13,18 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import React from 'react';
-import { TemplatePage } from './TemplatePage';
-import { wrapInTestApp, renderWithEffects } from '@backstage/test-utils';
-import { ApiRegistry, errorApiRef, ApiProvider } from '@backstage/core';
-import { scaffolderApiRef, ScaffolderApi } from '../../api';
-import { catalogApiRef, CatalogApi } from '@backstage/plugin-catalog';
-import { mutate } from 'swr';
-import { act } from 'react-dom/test-utils';
-import { Route, MemoryRouter } from 'react-router';
-import { rootRoute } from '../../routes';
-import { ThemeProvider } from '@material-ui/core';
+import { ApiProvider, ApiRegistry, errorApiRef } from '@backstage/core';
+import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog';
+import { renderInTestApp, renderWithEffects } from '@backstage/test-utils';
 import { lightTheme } from '@backstage/theme';
+import { ThemeProvider } from '@material-ui/core';
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { MemoryRouter, Route } from 'react-router';
+import { ScaffolderApi, scaffolderApiRef } from '../../api';
+import { rootRoute } from '../../routes';
+import { TemplatePage } from './TemplatePage';
 
 const templateMock = {
   apiVersion: 'backstage.io/v1alpha1',
@@ -90,48 +89,43 @@ const apis = ApiRegistry.from([
 ]);
 
 describe('TemplatePage', () => {
-  afterEach(async () => {
-    // Cleaning up swr's cache
-    await act(async () => {
-      await mutate('templates/test');
-    });
-  });
+  beforeEach(() => jest.resetAllMocks());
+
   it('renders correctly', async () => {
-    catalogApiMock.getEntities.mockResolvedValueOnce([templateMock]);
-    const rendered = await renderWithEffects(
-      wrapInTestApp(
-        <ApiProvider apis={apis}>
-          <TemplatePage />
-        </ApiProvider>,
-      ),
+    catalogApiMock.getEntities.mockResolvedValueOnce({ items: [templateMock] });
+    const rendered = await renderInTestApp(
+      <ApiProvider apis={apis}>
+        <TemplatePage />
+      </ApiProvider>,
     );
 
     expect(rendered.queryByText('Create a new component')).toBeInTheDocument();
     expect(rendered.queryByText('React SSR Template')).toBeInTheDocument();
     // await act(async () => await mutate('templates/test'));
   });
+
   it('renders spinner while loading', async () => {
     let resolve: Function;
     const promise = new Promise<any>(res => {
       resolve = res;
     });
-    catalogApiMock.getEntities.mockResolvedValueOnce(promise);
-    const rendered = await renderWithEffects(
-      wrapInTestApp(
-        <ApiProvider apis={apis}>
-          <TemplatePage />
-        </ApiProvider>,
-      ),
+    catalogApiMock.getEntities.mockReturnValueOnce(promise);
+    const rendered = await renderInTestApp(
+      <ApiProvider apis={apis}>
+        <TemplatePage />
+      </ApiProvider>,
     );
 
     expect(rendered.queryByText('Create a new component')).toBeInTheDocument();
     expect(rendered.queryByTestId('loading-progress')).toBeInTheDocument();
     // Need to cleanup the promise or will timeout
-    resolve!();
+    act(() => {
+      resolve!({ items: [] });
+    });
   });
 
   it('navigates away if no template was loaded', async () => {
-    catalogApiMock.getEntities.mockResolvedValueOnce([]);
+    catalogApiMock.getEntities.mockResolvedValueOnce({ items: [] });
 
     const rendered = await renderWithEffects(
       <ApiProvider apis={apis}>

plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx

@@ -27,28 +27,26 @@ import { catalogApiRef } from '@backstage/plugin-catalog';
 import { LinearProgress } from '@material-ui/core';
 import { IChangeEvent } from '@rjsf/core';
 import React, { useState } from 'react';
-import { useParams } from 'react-router-dom';
-import useStaleWhileRevalidate from 'swr';
-import { scaffolderApiRef } from '../../api';
-import { JobStatusModal } from '../JobStatusModal';
-import { Job } from '../../types';
-import { MultistepJsonForm } from '../MultistepJsonForm';
 import { Navigate } from 'react-router';
+import { useParams } from 'react-router-dom';
+import { useAsync } from 'react-use';
+import { scaffolderApiRef } from '../../api';
 import { rootRoute } from '../../routes';
+import { Job } from '../../types';
+import { JobStatusModal } from '../JobStatusModal';
+import { MultistepJsonForm } from '../MultistepJsonForm';
 
 const useTemplate = (
   templateName: string,
   catalogApi: typeof catalogApiRef.T,
 ) => {
-  const { data, error } = useStaleWhileRevalidate(
-    `templates/${templateName}`,
-    async () =>
-      catalogApi.getEntities({
-        kind: 'Template',
-        'metadata.name': templateName,
-      }) as Promise<TemplateEntityV1alpha1[]>,
-  );
-  return { template: data?.[0], loading: !error && !data, error };
+  const { value, loading, error } = useAsync(async () => {
+    const response = await catalogApi.getEntities({
+      filter: { kind: 'Template', 'metadata.name': templateName },
+    });
+    return response.items as TemplateEntityV1alpha1[];
+  });
+  return { template: value?.[0], loading, error };
 };
 
 const OWNER_REPO_SCHEMA = {
@@ -110,7 +108,7 @@ export const TemplatePage = () => {
   const handleCreateComplete = async (job: Job) => {
     const target = job.metadata.remoteUrl?.replace(
       /\.git$/,
-      // TODO(Rugvip): This is not the location we want. As part of scaffodler v2 we
+      // TODO(Rugvip): This is not the location we want. As part of scaffolder v2 we
       //               want this to be more flexible, but before that we might want
       //               to update all templates to use catalog-info.yaml instead.
       '/blob/master/component-info.yaml',

plugins/search/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint')],
+};

plugins/search/README.md

@@ -0,0 +1,9 @@
+# Backstage Search
+
+**This plugin is still under development.**
+
+You can follow the progress under the Global search in Backstage [milestone](https://github.com/backstage/backstage/milestone/21) or reach out to us in the #search Discord channel.
+
+## Getting started
+
+Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search)to check out the plugin.