Merge branch 'master' of github.com:spotify/backstage into shmidt-i/load-scaffolder-templates-from-api

docs/FAQ.md

@@ -11,12 +11,16 @@ brand.
 
 ### Is Backstage a monitoring platform?
 
-No, but it can be! Backstage is designed to be a developer portal for all your infrastructure tooling, services, and documentation. So, it's not a monitoring platform — but that doesn't mean you can't integrate a monitoring tool into Backstage by writing [a plugin](https://github.com/spotify/backstage/blob/master/docs/FAQ.md#what-is-a-plugin-in-backstage).
+No, but it can be! Backstage is designed to be a developer portal for all your
+infrastructure tooling, services, and documentation. So, it's not a monitoring
+platform — but that doesn't mean you can't integrate a monitoring tool into
+Backstage by writing
+[a plugin](https://github.com/spotify/backstage/blob/master/docs/FAQ.md#what-is-a-plugin-in-backstage).
 
 ### How is Backstage licensed?
 
-Backstage was released as open sourced software by Spotify and is licensed
-under [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0).
+Backstage was released as open sourced software by Spotify and is licensed under
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0).
 
 ### Why did we open source Backstage?
 
@@ -95,7 +99,15 @@ plugins. ​
 
 ### What is a "plugin" in Backstage?
 
-By far, our most-used plugin is our TechDocs plugin, which we use for creating technical documentation. Our philosophy at Spotify is to treat "docs like code", where you write documentation using the same workflow as you write your code. This makes it easier to create, find, and update documentation. We hope to release [the open source version](https://github.com/spotify/backstage/issues/687) in the future. (See also: "[Will Spotify's internal plugins be open sourced, too?](https://github.com/spotify/backstage/blob/master/docs/FAQ.md#what-is-a-plugin-in-backstage)" above)
+By far, our most-used plugin is our TechDocs plugin, which we use for creating
+technical documentation. Our philosophy at Spotify is to treat "docs like code",
+where you write documentation using the same workflow as you write your code.
+This makes it easier to create, find, and update documentation. We hope to
+release
+[the open source version](https://github.com/spotify/backstage/issues/687) in
+the future. (See also:
+"[Will Spotify's internal plugins be open sourced, too?](https://github.com/spotify/backstage/blob/master/docs/FAQ.md#what-is-a-plugin-in-backstage)"
+above)
 
 ### Do I have to write plugins in TypeScript?
 
@@ -144,10 +156,12 @@ Integrators also configure closed source plugins locally from the monorepo. ​
 ​ We chose GitHub because it is the tool that we are most familiar with, so that
 will naturally lead to integrations for GitHub being developed at an early
 stage. ​ Hosting this project on GitHub does not exclude integrations with
-alternatives, such as GitLab or Bitbucket. We believe that in time there will be
-plugins that will provide functionality for these tools as well. Hopefully,
-contributed by the community! ​ Also note, implementations of Backstage can be
-hosted wherever you feel suits your needs best. ​
+alternatives, such as
+[GitLab](https://github.com/spotify/backstage/issues?q=is%3Aissue+is%3Aopen+GitLab)
+or Bitbucket. We believe that in time there will be plugins that will provide
+functionality for these tools as well. Hopefully, contributed by the community!
+​ Also note, implementations of Backstage can be hosted wherever you feel suits
+your needs best. ​
 
 ### Who maintains Backstage?
 

packages/catalog-model/package.json

@@ -1,9 +1,7 @@
 {
   "name": "@backstage/catalog-model",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.cjs.js",
-  "module": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,
@@ -22,8 +20,10 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@backstage/config": "^0.1.1-alpha.9",
     "@types/yup": "^0.28.2",
     "lodash": "^4.17.15",
+    "uuid": "^8.0.0",
     "yup": "^0.28.5"
   },
   "devDependencies": {

packages/catalog-model/src/entity/Entity.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { JsonObject } from '@backstage/config';
+
 /**
  * The format envelope that's common to all versions/kinds of entity.
  *
@@ -39,7 +41,7 @@ export type Entity = {
   /**
    * The specification data describing the entity itself.
    */
-  spec?: object;
+  spec?: JsonObject;
 };
 
 /**
@@ -48,7 +50,7 @@ export type Entity = {
  * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
  * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
  */
-export type EntityMeta = {
+export type EntityMeta = JsonObject & {
   /**
    * A globally unique ID for the entity.
    *
@@ -112,3 +114,8 @@ export type EntityMeta = {
    */
   annotations?: Record<string, string>;
 };
+
+/**
+ * The keys of EntityMeta that are auto-generated.
+ */
+export const entityMetaGeneratedFields = ['uid', 'etag', 'generation'] as const;

packages/catalog-model/src/entity/index.ts

@@ -14,5 +14,12 @@
  * limitations under the License.
  */
 
+export { entityMetaGeneratedFields } from './Entity';
 export type { Entity, EntityMeta } from './Entity';
 export * from './policies';
+export {
+  entityHasChanges,
+  generateEntityEtag,
+  generateEntityUid,
+  generateUpdatedEntity,
+} from './util';

packages/catalog-model/src/entity/util.test.ts

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import lodash from 'lodash';
+import {
+  generateEntityEtag,
+  generateEntityUid,
+  entityHasChanges,
+  generateUpdatedEntity,
+} from './util';
+import { Entity } from './Entity';
+
+describe('util', () => {
+  describe('generateEntityUid', () => {
+    it('generates randomness', () => {
+      expect(generateEntityUid()).not.toEqual('');
+      expect(generateEntityUid()).not.toEqual(generateEntityUid());
+    });
+  });
+
+  describe('generateEntityEtag', () => {
+    it('generates randomness', () => {
+      expect(generateEntityEtag()).not.toEqual('');
+      expect(generateEntityEtag()).not.toEqual(generateEntityEtag());
+    });
+  });
+
+  describe('entityHasChanges', () => {
+    let a: Entity;
+    beforeEach(() => {
+      a = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          name: 'name',
+          custom: 'custom',
+          labels: {
+            labelKey: 'labelValue',
+          },
+          annotations: {
+            annotationKey: 'annotationValue',
+          },
+        },
+        spec: {
+          a: 'a',
+        },
+      };
+    });
+
+    it('happy path: clone has no changes', () => {
+      const b = lodash.cloneDeep(a);
+      expect(entityHasChanges(a, b)).toBe(false);
+    });
+
+    it('detects root field changes', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.apiVersion += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.apiVersion;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.kind += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.kind;
+      expect(entityHasChanges(a, b)).toBe(true);
+    });
+
+    it('detects metadata changes', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.metadata.name += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.metadata.custom;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.metadata.custom;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.metadata.labels.n = 'n';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.metadata.labels.labelKey += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+    });
+
+    it('detects annotation changes, but not removals', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.metadata.annotations.annotationKey += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.metadata.annotations.n = 'n';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.metadata.annotations.annotationKey;
+      expect(entityHasChanges(a, b)).toBe(false);
+    });
+
+    it('detects spec changes', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.spec.a += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.spec.a;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.spec.n = 'n';
+      expect(entityHasChanges(a, b)).toBe(true);
+    });
+  });
+
+  describe('generateUpdatedEntity', () => {
+    let a: Entity;
+    let b: any;
+    beforeEach(() => {
+      a = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          uid: 'da921f56-f655-4e6e-9b8b-bb19a57818d8',
+          etag: 'NzY5NDA5NzQtYmEwNC00MDY0LWFiYmItNTYxYzQxM2JhZDcx',
+          generation: 2,
+          name: 'name',
+          custom: 'custom',
+          labels: {
+            labelKey: 'labelValue',
+          },
+          annotations: {
+            annotationKey: 'annotationValue',
+          },
+        },
+        spec: {
+          a: 'a',
+        },
+      };
+      b = lodash.cloneDeep(a);
+      delete b.metadata.uid;
+      delete b.metadata.etag;
+      delete b.metadata.generation;
+    });
+
+    it('happy path: running on itself leaves it unchanged', () => {
+      const result = generateUpdatedEntity(a, b);
+      expect(result).toEqual(a);
+    });
+
+    it('bumps etag and generation when spec is changed', () => {
+      b.spec.a += 'a';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation! + 1);
+      expect(result.spec).toEqual({ a: 'aa' });
+    });
+
+    it('bumps only etag when other things than spec are changed', () => {
+      b.metadata.n = 'n';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation);
+      expect(result.metadata.n).toEqual('n');
+    });
+
+    it('retains new annotations', () => {
+      b.metadata.annotations.annotationKey = 'changedValue';
+      b.metadata.annotations.newKey = 'newValue';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation);
+      expect(result.metadata.annotations).toEqual({
+        annotationKey: 'changedValue',
+        newKey: 'newValue',
+      });
+    });
+
+    it('retains old annotations', () => {
+      b.metadata.annotations.newKey = 'newValue';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation);
+      expect(result.metadata.annotations).toEqual({
+        annotationKey: 'annotationValue',
+        newKey: 'newValue',
+      });
+    });
+  });
+});

packages/catalog-model/src/entity/util.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import lodash from 'lodash';
+import { v4 as uuidv4 } from 'uuid';
+import { Entity } from './Entity';
+
+/**
+ * Generates a new random UID for an entity.
+ *
+ * @returns A string with enough randomness to uniquely identify an entity
+ */
+export function generateEntityUid(): string {
+  return uuidv4();
+}
+
+/**
+ * Generates a new random Etag for an entity.
+ *
+ * @returns A string with enough randomness to uniquely identify an entity
+ *          revision
+ */
+export function generateEntityEtag(): string {
+  return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, '');
+}
+
+/**
+ * Checks whether there are any significant changes going from the previous to
+ * the next version of this entity.
+ *
+ * Significance, in this case, means that we do not compare generated fields
+ * such as uid, etag and generation, and we only check that no new annotations
+ * are added or existing annotations were changed (since they are effectively
+ * merged when doing updates).
+ *
+ * @param previous The old state of the entity
+ * @param next The new state of the entity
+ */
+export function entityHasChanges(previous: Entity, next: Entity): boolean {
+  if (entityHasAnnotationChanges(previous, next)) {
+    return true;
+  }
+
+  const e1 = lodash.cloneDeep(previous);
+  const e2 = lodash.cloneDeep(next);
+
+  if (!e1.metadata.labels) {
+    e1.metadata.labels = {};
+  }
+  if (!e2.metadata.labels) {
+    e2.metadata.labels = {};
+  }
+
+  // Remove generated fields
+  delete e1.metadata.uid;
+  delete e1.metadata.etag;
+  delete e1.metadata.generation;
+  delete e2.metadata.uid;
+  delete e2.metadata.etag;
+  delete e2.metadata.generation;
+
+  // Remove already compared things
+  delete e1.metadata.annotations;
+  delete e2.metadata.annotations;
+
+  return !lodash.isEqual(e1, e2);
+}
+
+/**
+ * Takes an old revision of an entity and a new desired state, and merges
+ * them into a complete new state.
+ *
+ * The previous revision is expected to be a complete model loaded from the
+ * catalog, including the uid, etag and generation fields.
+ *
+ * @param previous The old state of the entity
+ * @param next The new state of the entity
+ * @returns An entity with the merged state of both
+ */
+export function generateUpdatedEntity(previous: Entity, next: Entity): Entity {
+  const { uid, etag, generation } = previous.metadata;
+  if (!uid || !etag || !generation) {
+    throw new Error('Previous entity must have uid, etag and generation');
+  }
+
+  const result = lodash.cloneDeep(next);
+
+  // Annotations are merged, with the new ones taking precedence
+  if (previous.metadata.annotations) {
+    next.metadata.annotations = {
+      ...previous.metadata.annotations,
+      ...next.metadata.annotations,
+    };
+  }
+
+  // Generated fields are copied and updated
+  const bumpEtag = entityHasChanges(previous, result);
+  const bumpGeneration = !lodash.isEqual(previous.spec, result.spec);
+  result.metadata.uid = uid;
+  result.metadata.etag = bumpEtag ? generateEntityEtag() : etag;
+  result.metadata.generation = bumpGeneration ? generation + 1 : generation;
+
+  return result;
+}
+
+function entityHasAnnotationChanges(previous: Entity, next: Entity): boolean {
+  // Since the next annotations get merged into the previous, extract only
+  // the overlapping keys and check if their values match.
+  if (next.metadata.annotations) {
+    if (!previous.metadata.annotations) {
+      return true;
+    }
+    if (
+      !lodash.isEqual(
+        next.metadata.annotations,
+        lodash.pick(
+          previous.metadata.annotations,
+          Object.keys(next.metadata.annotations),
+        ),
+      )
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}

packages/cli/config/jest.js

@@ -25,32 +25,13 @@ async function getConfig() {
     return require(path.resolve('jest.config.ts'));
   }
 
-  const moduleNameMapper = {
-    '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'),
-  };
-
-  // Only point to src/ if we're not in CI, there we just build packages first anyway
-  if (!process.env.CI) {
-    const LernaProject = require('@lerna/project');
-    const project = new LernaProject(path.resolve('.'));
-    const packages = await project.getPackages();
-
-    // To avoid having to build all deps inside the monorepo before running tests,
-    // we point directory to src/ where applicable.
-    // For example, @backstage/core = <repo-root>/packages/core/src/index.ts is added to moduleNameMapper
-    for (const pkg of packages) {
-      const mainSrc = pkg.get('main:src');
-      if (mainSrc) {
-        moduleNameMapper[`^${pkg.name}$`] = path.resolve(pkg.location, mainSrc);
-      }
-    }
-  }
-
   const options = {
     rootDir: path.resolve('src'),
     coverageDirectory: path.resolve('coverage'),
     collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'],
-    moduleNameMapper,
+    moduleNameMapper: {
+      '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'),
+    },
 
     // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed
     // TODO: jest is working on module support, it's possible that we can remove this in the future

packages/cli/src/lib/bundler/config.ts

@@ -33,9 +33,6 @@ import { BundlingOptions, BackendBundlingOptions } from './types';
 
 export function resolveBaseUrl(config: Config): URL {
   const baseUrl = config.getString('app.baseUrl');
-  if (!baseUrl) {
-    throw new Error('app.baseUrl must be set in config');
-  }
   try {
     return new URL(baseUrl, 'http://localhost:3000');
   } catch (error) {
@@ -52,9 +49,6 @@ export function createConfig(
   const { plugins, loaders } = transforms(options);
 
   const baseUrl = options.config.getString('app.baseUrl');
-  if (!baseUrl) {
-    throw new Error('app.baseUrl must be set in config');
-  }
   const validBaseUrl = new URL(baseUrl, 'https://backstage-app.dev');
 
   if (checksEnabled) {
@@ -115,7 +109,7 @@ export function createConfig(
     entry: [require.resolve('react-hot-loader/patch'), paths.targetEntry],
     resolve: {
       extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx'],
-      mainFields: ['main:src', 'browser', 'module', 'main'],
+      mainFields: ['browser', 'module', 'main'],
       plugins: [
         new ModuleScopePlugin(
           [paths.targetSrc, paths.targetDev],
@@ -189,7 +183,7 @@ export function createBackendConfig(
     ],
     resolve: {
       extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx'],
-      mainFields: ['main:src', 'browser', 'module', 'main'],
+      mainFields: ['browser', 'module', 'main'],
       modules: [paths.targetNodeModules, paths.rootNodeModules],
       plugins: [
         new ModuleScopePlugin(

packages/cli/src/lib/tasks.ts

@@ -171,9 +171,7 @@ export async function installWithLocalDeps(dir: string) {
   });
 
   // This takes care of pointing all the installed packages from this repo to
-  // dist instead of the local src.
-  // For example node_modules/@backstage/core/packages.json is rewritten to point
-  // types to dist/index.d.ts and the main:src field is removed.
+  // dist instead of the local src, using the field overrides in publishConfig.
   // Without this we get type checking errors in the e2e test
   if (process.env.BACKSTAGE_E2E_CLI_TEST) {
     Task.section('Patching local dependencies for e2e tests');
@@ -192,7 +190,6 @@ export async function installWithLocalDeps(dir: string) {
           const depJson = await fs.readJson(depJsonPath);
 
           // We want dist to be used for e2e tests
-          delete depJson['main:src'];
           for (const key of Object.keys(depJson.publishConfig)) {
             if (key !== 'access') {
               depJson[key] = depJson.publishConfig[key];

packages/cli/templates/default-app/plugins/welcome/package.json.hbs

@@ -1,8 +1,7 @@
 {
   "name": "plugin-welcome",
   "version": "0.0.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "private": true,
   "publishConfig": {

packages/cli/templates/default-plugin/package.json.hbs

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-{{id}}",
   "version": "{{version}}",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": true,

packages/config-loader/src/lib/env.test.ts

@@ -45,9 +45,12 @@ describe('readEnv', () => {
       }),
     ).toEqual([
       {
-        foo: 'bar',
-        numbers: { a: 1, b: 2, c: false },
-        very: { deep: { nested: { config: { object: {} } } } },
+        data: {
+          foo: 'bar',
+          numbers: { a: 1, b: 2, c: false },
+          very: { deep: { nested: { config: { object: {} } } } },
+        },
+        context: 'env',
       },
     ]);
   });

packages/config-loader/src/lib/env.ts

@@ -42,7 +42,7 @@ const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
 export function readEnv(env: {
   [name: string]: string | undefined;
 }): AppConfig[] {
-  let config: JsonObject | undefined = undefined;
+  let data: JsonObject | undefined = undefined;
 
   for (const [name, value] of Object.entries(env)) {
     if (!value) {
@@ -52,7 +52,7 @@ export function readEnv(env: {
       const key = name.replace(ENV_PREFIX, '');
       const keyParts = key.split('_');
 
-      let obj = (config = config ?? {});
+      let obj = (data = data ?? {});
       for (const [index, part] of keyParts.entries()) {
         if (!CONFIG_KEY_PART_PATTERN.test(part)) {
           throw new TypeError(`Invalid env config key '${key}'`);
@@ -87,5 +87,5 @@ export function readEnv(env: {
     }
   }
 
-  return config ? [config] : [];
+  return data ? [{ data, context: 'env' }] : [];
 }

packages/config-loader/src/lib/reader.test.ts

@@ -38,11 +38,14 @@ describe('readConfigFile', () => {
     } as ReaderContext);
 
     await expect(config).resolves.toEqual({
-      app: {
-        title: 'Test',
-        x: 1,
-        y: [true],
+      data: {
+        app: {
+          title: 'Test',
+          x: 1,
+          y: [true],
+        },
       },
+      context: 'app-config.yaml',
     });
   });
 
@@ -83,7 +86,10 @@ describe('readConfigFile', () => {
     });
 
     await expect(config).resolves.toEqual({
-      app: 'secret',
+      data: {
+        app: 'secret',
+      },
+      context: 'app-config.yaml',
     });
     expect(readSecret).toHaveBeenCalledWith({ file: './my-secret' });
   });

packages/config-loader/src/lib/reader.ts

@@ -14,16 +14,20 @@
  * limitations under the License.
  */
 
+import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
+import { basename } from 'path';
 import yaml from 'yaml';
-import { isObject } from './utils';
-import { JsonValue, JsonObject } from '@backstage/config';
 import { ReaderContext } from './types';
+import { isObject } from './utils';
 
 /**
  * Reads and parses, and validates, and transforms a single config file.
  * The transformation rewrites any special values, like the $secret key.
  */
-export async function readConfigFile(filePath: string, ctx: ReaderContext) {
+export async function readConfigFile(
+  filePath: string,
+  ctx: ReaderContext,
+): Promise<AppConfig> {
   const configYaml = await ctx.readFile(filePath);
   const config = yaml.parse(configYaml);
 
@@ -63,9 +67,12 @@ export async function readConfigFile(filePath: string, ctx: ReaderContext) {
     const out: JsonObject = {};
 
     for (const [key, value] of Object.entries(obj)) {
-      const result = await transform(value, `${path}.${key}`);
-      if (result !== undefined) {
-        out[key] = result;
+      // undefined covers optional fields
+      if (value !== undefined) {
+        const result = await transform(value, `${path}.${key}`);
+        if (result !== undefined) {
+          out[key] = result;
+        }
       }
     }
 
@@ -76,5 +83,5 @@ export async function readConfigFile(filePath: string, ctx: ReaderContext) {
   if (!isObject(finalConfig)) {
     throw new TypeError('Expected object at config root');
   }
-  return finalConfig;
+  return { data: finalConfig, context: basename(filePath) };
 }

packages/config-loader/src/lib/secrets.ts

@@ -122,7 +122,7 @@ export async function readSecret(
     const { path } = secret;
     const parts = typeof path === 'string' ? path.split('.') : path;
 
-    let value: JsonValue = await parser(content);
+    let value: JsonValue | undefined = await parser(content);
     for (const [index, part] of parts.entries()) {
       if (!isObject(value)) {
         const errPath = parts.slice(0, index).join('.');
@@ -132,6 +132,7 @@ export async function readSecret(
       }
       value = value[part];
     }
+
     return String(value);
   }
 

packages/config/package.json

@@ -29,6 +29,9 @@
     "postpack": "backstage-cli postpack",
     "clean": "backstage-cli clean"
   },
+  "dependencies": {
+    "lodash": "^4.17.15"
+  },
   "devDependencies": {
     "@types/jest": "^25.2.2",
     "@types/node": "^12.0.0"

packages/config/src/reader.test.ts

@@ -37,72 +37,112 @@ const DATA = {
 };
 
 function expectValidValues(config: ConfigReader) {
+  expect(config.keys()).toEqual(Object.keys(DATA));
+  expect(config.get('zero')).toBe(0);
   expect(config.getNumber('zero')).toBe(0);
   expect(config.getNumber('one')).toBe(1);
+  expect(config.getOptional('true')).toBe(true);
   expect(config.getBoolean('true')).toBe(true);
   expect(config.getBoolean('false')).toBe(false);
   expect(config.getString('string')).toBe('string');
+  expect(config.get('strings')).toEqual(['string1', 'string2']);
   expect(config.getStringArray('strings')).toEqual(['string1', 'string2']);
   expect(config.getConfig('nested').getNumber('one')).toBe(1);
+  expect(config.get('nested')).toEqual({
+    one: 1,
+    string: 'string',
+    strings: ['string1', 'string2'],
+  });
   expect(config.getConfig('nested').getString('string')).toBe('string');
-  expect(config.getConfig('nested').getStringArray('strings')).toEqual([
-    'string1',
-    'string2',
-  ]);
+  expect(
+    config.getOptionalConfig('nested')!.getStringArray('strings'),
+  ).toEqual(['string1', 'string2']);
+  expect(config.getOptional('missing')).toBe(undefined);
+  expect(config.getOptionalConfig('missing')).toBe(undefined);
+  expect(config.getOptionalConfigArray('missing')).toBe(undefined);
+  expect(config.getNumber('zero')).toBe(0);
+  expect(config.getBoolean('true')).toBe(true);
+  expect(config.getString('string')).toBe('string');
+  expect(config.getStringArray('strings')).toEqual(['string1', 'string2']);
 
   const [config1, config2, config3] = config.getConfigArray('nestlings');
   expect(config1.getBoolean('boolean')).toBe(true);
   expect(config2.getString('string')).toBe('string');
   expect(config3.getNumber('number')).toBe(42);
+  expect(
+    config.getOptionalConfigArray('nestlings')![0].getBoolean('boolean'),
+  ).toBe(true);
 }
 
 function expectInvalidValues(config: ConfigReader) {
+  expect(() => config.getBoolean('string')).toThrow(
+    "Invalid type in config for key 'string' in 'ctx', got string, wanted boolean",
+  );
   expect(() => config.getNumber('string')).toThrow(
-    'Invalid type in config for key string, got string, wanted number',
+    "Invalid type in config for key 'string' in 'ctx', got string, wanted number",
   );
   expect(() => config.getString('one')).toThrow(
-    'Invalid type in config for key one, got number, wanted string',
+    "Invalid type in config for key 'one' in 'ctx', got number, wanted string",
   );
   expect(() => config.getNumber('true')).toThrow(
-    'Invalid type in config for key true, got boolean, wanted number',
+    "Invalid type in config for key 'true' in 'ctx', got boolean, wanted number",
   );
   expect(() => config.getStringArray('null')).toThrow(
-    'Invalid type in config for key null, got null, wanted string-array',
+    "Invalid type in config for key 'null' in 'ctx', got null, wanted string-array",
   );
   expect(() => config.getString('emptyString')).toThrow(
-    'Invalid type in config for key emptyString, got empty-string, wanted string',
+    "Invalid type in config for key 'emptyString' in 'ctx', got empty-string, wanted string",
   );
   expect(() => config.getStringArray('badStrings')).toThrow(
-    'Invalid type in config for key badStrings[1], got empty-string, wanted string',
+    "Invalid type in config for key 'badStrings[1]' in 'ctx', got empty-string, wanted string",
   );
   expect(() => config.getStringArray('worseStrings')).toThrow(
-    'Invalid type in config for key worseStrings[1], got number, wanted string',
+    "Invalid type in config for key 'worseStrings[1]' in 'ctx', got number, wanted string",
   );
   expect(() => config.getStringArray('worstStrings')).toThrow(
-    'Invalid type in config for key worstStrings[2], got object, wanted string',
+    "Invalid type in config for key 'worstStrings[2]' in 'ctx', got object, wanted string",
   );
   expect(() => config.getConfig('one')).toThrow(
-    'Invalid type in config for key one, got number, wanted object',
+    "Invalid type in config for key 'one' in 'ctx', got number, wanted object",
   );
   expect(() => config.getConfigArray('one')).toThrow(
-    'Invalid type in config for key one, got number, wanted object-array',
+    "Invalid type in config for key 'one' in 'ctx', got number, wanted object-array",
+  );
+  expect(() => config.getBoolean('missing')).toThrow(
+    "Missing required config value at 'missing'",
+  );
+  expect(() => config.getNumber('missing')).toThrow(
+    "Missing required config value at 'missing'",
+  );
+  expect(() => config.getString('missing')).toThrow(
+    "Missing required config value at 'missing'",
+  );
+  expect(() => config.getStringArray('missing')).toThrow(
+    "Missing required config value at 'missing'",
   );
 }
 
+const CTX = 'ctx';
+
 describe('ConfigReader', () => {
   it('should read empty config with valid keys', () => {
-    const config = new ConfigReader({});
-    expect(config.getString('x')).toBeUndefined();
-    expect(config.getString('x_x')).toBeUndefined();
-    expect(config.getString('x-X')).toBeUndefined();
-    expect(config.getString('x0')).toBeUndefined();
-    expect(config.getString('X-x2')).toBeUndefined();
-    expect(config.getString('x0_x0')).toBeUndefined();
-    expect(config.getString('x_x-x_x')).toBeUndefined();
+    const config = new ConfigReader({}, CTX);
+    expect(config.keys()).toEqual([]);
+    expect(config.getOptionalString('x')).toBeUndefined();
+    expect(config.getOptionalString('x_x')).toBeUndefined();
+    expect(config.getOptionalString('x-X')).toBeUndefined();
+    expect(config.getOptionalString('x0')).toBeUndefined();
+    expect(config.getOptionalString('X-x2')).toBeUndefined();
+    expect(config.getOptionalString('x0_x0')).toBeUndefined();
+    expect(config.getOptionalString('x_x-x_x')).toBeUndefined();
+
+    expect(
+      new ConfigReader(undefined, CTX).getOptionalString('x'),
+    ).toBeUndefined();
   });
 
   it('should throw on invalid keys', () => {
-    const config = new ConfigReader({});
+    const config = new ConfigReader({}, CTX);
 
     expect(() => config.getString('.')).toThrow(/^Invalid config key/);
     expect(() => config.getString('0')).toThrow(/^Invalid config key/);
@@ -119,35 +159,39 @@ describe('ConfigReader', () => {
     expect(() => config.getString('a.a.a.a.')).toThrow(/^Invalid config key/);
     expect(() => config.getString('a._')).toThrow(/^Invalid config key/);
     expect(() => config.getString('a.-.a')).toThrow(/^Invalid config key/);
+
+    expect(() => new ConfigReader(undefined, CTX).getString('.')).toThrow(
+      /^Invalid config key/,
+    );
   });
 
   it('should read valid values', () => {
-    const config = new ConfigReader(DATA);
+    const config = new ConfigReader(DATA, CTX);
     expectValidValues(config);
   });
 
   it('should fail to read invalid values', () => {
-    const config = new ConfigReader(DATA);
+    const config = new ConfigReader(DATA, CTX);
     expectInvalidValues(config);
   });
 });
 
 describe('ConfigReader with fallback', () => {
   it('should behave as if without fallback', () => {
-    const config = new ConfigReader({}, new ConfigReader(DATA));
-    expect(config.getString('x')).toBeUndefined();
+    const config = new ConfigReader({}, CTX, new ConfigReader(DATA, CTX));
+    expect(config.getOptionalString('x')).toBeUndefined();
     expect(() => config.getString('.')).toThrow(/^Invalid config key/);
     expect(() => config.getString('a.')).toThrow(/^Invalid config key/);
   });
 
   it('should read values from itself', () => {
-    const config = new ConfigReader(DATA, new ConfigReader({}));
+    const config = new ConfigReader(DATA, CTX, new ConfigReader({}, CTX));
     expectValidValues(config);
     expectInvalidValues(config);
   });
 
   it('should read values from a fallback', () => {
-    const config = new ConfigReader({}, new ConfigReader(DATA));
+    const config = new ConfigReader({}, CTX, new ConfigReader(DATA, CTX));
     expectValidValues(config);
     expectInvalidValues(config);
   });
@@ -155,12 +199,92 @@ describe('ConfigReader with fallback', () => {
   it('should read values from multiple levels of fallbacks', () => {
     const config = new ConfigReader(
       {},
-      new ConfigReader({}, new ConfigReader({}, new ConfigReader(DATA))),
+      CTX,
+      new ConfigReader(
+        {},
+        CTX,
+        new ConfigReader({}, CTX, new ConfigReader(DATA, CTX)),
+      ),
     );
     expectValidValues(config);
     expectInvalidValues(config);
   });
 
+  it('should show error with correct context', () => {
+    const config = ConfigReader.fromConfigs([
+      {
+        data: {
+          c: true,
+        },
+        context: 'x',
+      },
+      {
+        data: {
+          b: true,
+          c: true,
+          nested1: {
+            a: true,
+          },
+          badBefore: true,
+          badAfter: {
+            a: true,
+          },
+        },
+        context: 'y',
+      },
+      {
+        data: {
+          a: true,
+          b: true,
+          c: true,
+          nested1: {
+            a: true,
+            b: true,
+          },
+          badBefore: {
+            a: true,
+          },
+          badAfter: true,
+        },
+        context: 'z',
+      },
+    ]);
+
+    expect(() => config.getNumber('a')).toThrow(
+      "Invalid type in config for key 'a' in 'z', got boolean, wanted number",
+    );
+    expect(() => config.getNumber('b')).toThrow(
+      "Invalid type in config for key 'b' in 'y', got boolean, wanted number",
+    );
+    expect(() => config.getNumber('c')).toThrow(
+      "Invalid type in config for key 'c' in 'x', got boolean, wanted number",
+    );
+    expect(() => config.getNumber('nested1.a')).toThrow(
+      "Invalid type in config for key 'nested1.a' in 'y', got boolean, wanted number",
+    );
+    expect(() => config.getNumber('nested1.b')).toThrow(
+      "Invalid type in config for key 'nested1.b' in 'z', got boolean, wanted number",
+    );
+    expect(() => config.getConfig('nested1').getNumber('a')).toThrow(
+      "Invalid type in config for key 'nested1.a' in 'y', got boolean, wanted number",
+    );
+    expect(() => config.getConfig('nested1').getNumber('b')).toThrow(
+      "Invalid type in config for key 'nested1.b' in 'z', got boolean, wanted number",
+    );
+    expect(() => config.getNumber('badBefore.a')).toThrow(
+      "Invalid type in config for key 'badBefore' in 'y', got boolean, wanted object",
+    );
+    expect(() => config.getNumber('badBefore.b')).toThrow(
+      "Invalid type in config for key 'badBefore' in 'y', got boolean, wanted object",
+    );
+    expect(() => config.getNumber('badAfter.a')).toThrow(
+      "Invalid type in config for key 'badAfter.a' in 'y', got boolean, wanted number",
+    );
+    expect(() => config.getNumber('badAfter.b')).toThrow(
+      "Invalid type in config for key 'badAfter' in 'z', got boolean, wanted object",
+    );
+  });
+
   it('should read merged objects', () => {
     const a = {
       merged: {
@@ -181,7 +305,18 @@ describe('ConfigReader with fallback', () => {
       },
     };
 
-    const config = new ConfigReader(a, new ConfigReader(b));
+    const config = new ConfigReader(a, CTX, new ConfigReader(b, CTX));
+
+    expect(config.keys()).toEqual(['merged']);
+    expect(config.getConfig('merged').keys()).toEqual([
+      'x',
+      'z',
+      'arr',
+      'config',
+      'configs',
+      'y',
+    ]);
+    expect(config.getConfig('merged.config').keys()).toEqual(['d', 'e']);
 
     expect(config.getString('merged.x')).toBe('x');
     expect(config.getString('merged.y')).toBe('y');
@@ -206,12 +341,19 @@ describe('ConfigReader with fallback', () => {
       'a',
       'b',
     ]);
+    expect(() => config.getConfig('merged').getStringArray('x')).toThrow(
+      "Invalid type in config for key 'merged.x' in 'ctx', got string, wanted string-array",
+    );
 
     // Config arrays aren't merged either
     expect(config.getConfigArray('merged.configs').length).toBe(1);
     expect(config.getConfigArray('merged.configs')[0].getString('a')).toBe('a');
+    expect(config.getConfigArray('merged.configs')[0].getString('a')).toBe('a');
+    expect(() =>
+      config.getConfigArray('merged.configs')[0].getString('missing'),
+    ).toThrow("Missing required config value at 'merged.configs[0].missing'");
     expect(
-      config.getConfigArray('merged.configs')[0].getString('b'),
+      config.getConfigArray('merged.configs')[0].getOptionalString('b'),
     ).toBeUndefined();
 
     // Config arrays aren't merged either
@@ -220,7 +362,173 @@ describe('ConfigReader with fallback', () => {
       config.getConfig('merged').getConfigArray('configs')[0].getString('a'),
     ).toBe('a');
     expect(
-      config.getConfig('merged').getConfigArray('configs')[0].getString('b'),
+      config
+        .getConfig('merged')
+        .getConfigArray('configs')[0]
+        .getOptionalString('b'),
     ).toBeUndefined();
   });
 });
+
+describe('ConfigReader.get()', () => {
+  const config1 = {
+    a: {
+      x: 'x1',
+      y: ['y11', 'y12', 'y13'],
+      z: false,
+    },
+    b: {
+      x: 'x1',
+      y: ['y11'],
+    },
+  };
+  const config2 = {
+    b: {
+      x: 'x2',
+      y: ['y21', 'y22'],
+      z: 'z2',
+    },
+    c: {
+      c1: {
+        c2: 'c2',
+      },
+    },
+  };
+  const config3 = {
+    c: {
+      c1: 'c1',
+    },
+  };
+  const configs = [
+    {
+      data: config1,
+      context: '1',
+    },
+    {
+      data: config2,
+      context: '2',
+    },
+    {
+      data: config3,
+      context: '3',
+    },
+  ];
+
+  it('should be able to select sub-configs', () => {
+    expect(new ConfigReader(config1).get('a')).toEqual(config1.a);
+    expect(new ConfigReader(config1).get('b')).toEqual(config1.b);
+    expect(new ConfigReader(config2).get('b')).toEqual(config2.b);
+    expect(new ConfigReader(config2).get('c')).toEqual(config2.c);
+    expect(new ConfigReader(config3).get('c')).toEqual(config3.c);
+    expect(new ConfigReader(config2).get('c.c1')).toEqual(config2.c.c1);
+    expect(new ConfigReader(config2).getConfig('c').get('c1')).toEqual(
+      config2.c.c1,
+    );
+  });
+
+  it('should merge in fallback configs', () => {
+    expect(ConfigReader.fromConfigs([configs[0], configs[1]]).get('a')).toEqual(
+      {
+        x: 'x1',
+        y: ['y11', 'y12', 'y13'],
+        z: false,
+      },
+    );
+    expect(ConfigReader.fromConfigs([configs[0], configs[1]]).get('b')).toEqual(
+      {
+        x: 'x1',
+        y: ['y11'],
+        z: 'z2',
+      },
+    );
+    expect(ConfigReader.fromConfigs([configs[0], configs[1]]).get('c')).toEqual(
+      {
+        c1: {
+          c2: 'c2',
+        },
+      },
+    );
+    expect(ConfigReader.fromConfigs([configs[0], configs[1]]).get('a')).toEqual(
+      {
+        x: 'x1',
+        y: ['y11', 'y12', 'y13'],
+        z: false,
+      },
+    );
+    expect(ConfigReader.fromConfigs([configs[0], configs[1]]).get('b')).toEqual(
+      {
+        x: 'x1',
+        y: ['y11'],
+        z: 'z2',
+      },
+    );
+    expect(ConfigReader.fromConfigs([configs[0], configs[1]]).get('c')).toEqual(
+      {
+        c1: {
+          c2: 'c2',
+        },
+      },
+    );
+
+    expect(
+      ConfigReader.fromConfigs([configs[2], configs[1]]).getOptional('b'),
+    ).toEqual({
+      x: 'x2',
+      y: ['y21', 'y22'],
+      z: 'z2',
+    });
+    expect(
+      ConfigReader.fromConfigs([configs[2], configs[1]]).getOptional('c'),
+    ).toEqual({
+      c1: 'c1',
+    });
+  });
+
+  it('should not merge non-objects', () => {
+    const config = ConfigReader.fromConfigs([
+      {
+        data: {
+          a: ['1', '2'],
+          c: [],
+          d: {
+            x: 'x',
+          },
+          e: ['3'],
+          f: 'foo',
+          g: { z: 'z' },
+          h: {
+            a: 'a1',
+            c: 'c1',
+          },
+        },
+        context: '1',
+      },
+      {
+        data: {
+          a: ['x', 'y', 'z'],
+          b: ['1'],
+          c: ['1'],
+          d: ['2'],
+          e: {
+            y: 'y',
+          },
+          f: { x: 'x' },
+          g: 'bar',
+          h: {
+            a: 'a2',
+            b: 'b2',
+          },
+        },
+        context: '2',
+      },
+    ]);
+    expect(config.get('a')).toEqual(['1', '2']);
+    expect(config.get('b')).toEqual(['1']);
+    expect(config.get('c')).toEqual([]);
+    expect(config.get('d')).toEqual({ x: 'x' });
+    expect(config.get('e')).toEqual(['3']);
+    expect(config.get('f')).toEqual('foo');
+    expect(config.get('g')).toEqual({ z: 'z' });
+    expect(config.get('h')).toEqual({ a: 'a1', b: 'b2', c: 'c1' });
+  });
+});

packages/config/src/reader.ts

@@ -15,6 +15,8 @@
  */
 
 import { AppConfig, Config, JsonValue, JsonObject } from './types';
+import cloneDeep from 'lodash/cloneDeep';
+import mergeWith from 'lodash/mergeWith';
 
 // Update the same pattern in config-loader package if this is changed
 const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i;
@@ -39,43 +41,106 @@ function typeOf(value: JsonValue | undefined): string {
   return type;
 }
 
-export class ConfigReader implements Config {
-  private static readonly nullReader = new ConfigReader({});
+// Separate out a couple of common error messages to reduce bundle size.
+const errors = {
+  type(key: string, context: string, typeName: string, expected: string) {
+    return `Invalid type in config for key '${key}' in '${context}', got ${typeName}, wanted ${expected}`;
+  },
+  missing(key: string) {
+    return `Missing required config value at '${key}'`;
+  },
+};
 
+export class ConfigReader implements Config {
   static fromConfigs(configs: AppConfig[]): ConfigReader {
     if (configs.length === 0) {
-      return new ConfigReader({});
+      return new ConfigReader(undefined);
     }
 
     // Merge together all configs info a single config with recursive fallback
     // readers, giving the first config object in the array the highest priority.
-    return configs.reduceRight<ConfigReader>((previousReader, nextConfig) => {
-      return new ConfigReader(nextConfig, previousReader);
-    }, undefined!);
+    return configs.reduceRight<ConfigReader>(
+      (previousReader, { data, context }) => {
+        return new ConfigReader(data, context, previousReader);
+      },
+      undefined!,
+    );
   }
 
   constructor(
-    private readonly data: JsonObject,
+    private readonly data: JsonObject | undefined,
+    private readonly context: string = 'empty-config',
     private readonly fallback?: ConfigReader,
+    private readonly prefix: string = '',
   ) {}
 
-  getConfig(key: string): ConfigReader {
+  keys(): string[] {
+    const localKeys = this.data ? Object.keys(this.data) : [];
+    const fallbackKeys = this.fallback?.keys() ?? [];
+    return [...new Set([...localKeys, ...fallbackKeys])];
+  }
+
+  get(key: string): JsonValue {
+    const value = this.getOptional(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptional(key: string): JsonValue | undefined {
     const value = this.readValue(key);
-    const fallbackConfig = this.fallback?.getConfig(key);
+    const fallbackValue = this.fallback?.getOptional(key);
+
+    if (value === undefined) {
+      return fallbackValue;
+    } else if (fallbackValue === undefined) {
+      return value;
+    }
+
+    // Avoid merging arrays and primitive values, since that's how merging works for other
+    // methods for reading config.
+    return mergeWith(
+      {},
+      { value: cloneDeep(fallbackValue) },
+      { value },
+      (into, from) => (!isObject(from) || !isObject(into) ? from : undefined),
+    ).value;
+  }
+
+  getConfig(key: string): ConfigReader {
+    const value = this.getOptionalConfig(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptionalConfig(key: string): ConfigReader | undefined {
+    const value = this.readValue(key);
+    const fallbackConfig = this.fallback?.getOptionalConfig(key);
+    const prefix = this.fullKey(key);
+
     if (isObject(value)) {
-      return new ConfigReader(value, fallbackConfig);
+      return new ConfigReader(value, this.context, fallbackConfig, prefix);
     }
     if (value !== undefined) {
       throw new TypeError(
-        `Invalid type in config for key ${key}, got ${typeOf(
-          value,
-        )}, wanted object`,
+        errors.type(this.fullKey(key), this.context, typeOf(value), 'object'),
       );
     }
-    return fallbackConfig ?? ConfigReader.nullReader;
+    return fallbackConfig;
   }
 
   getConfigArray(key: string): ConfigReader[] {
+    const value = this.getOptionalConfigArray(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptionalConfigArray(key: string): ConfigReader[] | undefined {
     const configs = this.readConfigValue<JsonObject[]>(key, values => {
       if (!Array.isArray(values)) {
         return { expected: 'object-array' };
@@ -89,24 +154,60 @@ export class ConfigReader implements Config {
       return true;
     });
 
-    return (configs ?? []).map(obj => new ConfigReader(obj));
+    if (!configs) {
+      return undefined;
+    }
+
+    return configs.map(
+      (obj, index) =>
+        new ConfigReader(
+          obj,
+          this.context,
+          undefined,
+          this.fullKey(`${key}[${index}]`),
+        ),
+    );
   }
 
-  getNumber(key: string): number | undefined {
+  getNumber(key: string): number {
+    const value = this.getOptionalNumber(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptionalNumber(key: string): number | undefined {
     return this.readConfigValue(
       key,
       value => typeof value === 'number' || { expected: 'number' },
     );
   }
 
-  getBoolean(key: string): boolean | undefined {
+  getBoolean(key: string): boolean {
+    const value = this.getOptionalBoolean(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptionalBoolean(key: string): boolean | undefined {
     return this.readConfigValue(
       key,
       value => typeof value === 'boolean' || { expected: 'boolean' },
     );
   }
 
-  getString(key: string): string | undefined {
+  getString(key: string): string {
+    const value = this.getOptionalString(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptionalString(key: string): string | undefined {
     return this.readConfigValue(
       key,
       value =>
@@ -114,7 +215,15 @@ export class ConfigReader implements Config {
     );
   }
 
-  getStringArray(key: string): string[] | undefined {
+  getStringArray(key: string): string[] {
+    const value = this.getOptionalStringArray(key);
+    if (value === undefined) {
+      throw new Error(errors.missing(this.fullKey(key)));
+    }
+    return value;
+  }
+
+  getOptionalStringArray(key: string): string[] | undefined {
     return this.readConfigValue(key, values => {
       if (!Array.isArray(values)) {
         return { expected: 'string-array' };
@@ -128,6 +237,10 @@ export class ConfigReader implements Config {
     });
   }
 
+  private fullKey(key: string): string {
+    return `${this.prefix}${this.prefix ? '.' : ''}${key}`;
+  }
+
   private readConfigValue<T extends JsonValue>(
     key: string,
     validate: (
@@ -147,9 +260,13 @@ export class ConfigReader implements Config {
           value: theValue = value,
           expected,
         } = result;
-        const typeName = typeOf(theValue);
         throw new TypeError(
-          `Invalid type in config for key ${keyName}, got ${typeName}, wanted ${expected}`,
+          errors.type(
+            this.fullKey(keyName),
+            this.context,
+            typeOf(theValue),
+            expected,
+          ),
         );
       }
     }
@@ -159,16 +276,25 @@ export class ConfigReader implements Config {
 
   private readValue(key: string): JsonValue | undefined {
     const parts = key.split('.');
-
-    let value: JsonValue | undefined = this.data;
     for (const part of parts) {
       if (!CONFIG_KEY_PART_PATTERN.test(part)) {
         throw new TypeError(`Invalid config key '${key}'`);
       }
+    }
+
+    if (this.data === undefined) {
+      return undefined;
+    }
+
+    let value: JsonValue | undefined = this.data;
+    for (const [index, part] of parts.entries()) {
       if (isObject(value)) {
         value = value[part];
-      } else {
-        value = undefined;
+      } else if (value !== undefined) {
+        const badKey = this.fullKey(parts.slice(0, index).join('.'));
+        throw new TypeError(
+          errors.type(badKey, this.context, typeOf(value), 'object'),
+        );
       }
     }
 

packages/config/src/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-export type JsonObject = { [key in string]: JsonValue };
+export type JsonObject = { [key in string]?: JsonValue };
 export type JsonArray = JsonValue[];
 export type JsonValue =
   | JsonObject
@@ -24,18 +24,32 @@ export type JsonValue =
   | boolean
   | null;
 
-export type AppConfig = JsonObject;
+export type AppConfig = {
+  context: string;
+  data: JsonObject;
+};
 
 export type Config = {
+  keys(): string[];
+
+  get(key: string): JsonValue;
+  getOptional(key: string): JsonValue | undefined;
+
   getConfig(key: string): Config;
+  getOptionalConfig(key: string): Config | undefined;
 
   getConfigArray(key: string): Config[];
+  getOptionalConfigArray(key: string): Config[] | undefined;
 
-  getNumber(key: string): number | undefined;
+  getNumber(key: string): number;
+  getOptionalNumber(key: string): number | undefined;
 
-  getBoolean(key: string): boolean | undefined;
+  getBoolean(key: string): boolean;
+  getOptionalBoolean(key: string): boolean | undefined;
 
-  getString(key: string): string | undefined;
+  getString(key: string): string;
+  getOptionalString(key: string): string | undefined;
 
-  getStringArray(key: string): string[] | undefined;
+  getStringArray(key: string): string[];
+  getOptionalStringArray(key: string): string[] | undefined;
 };

packages/core-api/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli build --outputs types,esm",

packages/core-api/src/apis/definitions/ConfigApi.ts

@@ -14,20 +14,7 @@
  * limitations under the License.
  */
 import { createApiRef } from '../ApiRef';
-
-export type Config = {
-  getConfig(key: string): Config;
-
-  getConfigArray(key: string): Config[];
-
-  getNumber(key: string): number | undefined;
-
-  getBoolean(key: string): boolean | undefined;
-
-  getString(key: string): string | undefined;
-
-  getStringArray(key: string): string[] | undefined;
-};
+import { Config } from '@backstage/config';
 
 // Using interface to make the ConfigApi name show up in docs
 export interface ConfigApi extends Config {}

packages/core-api/src/apis/definitions/auth.ts

@@ -157,25 +157,47 @@ export type ProfileInfoOptions = {
   optional?: boolean;
 };
 
+/**
+ * This API provides access to profile information of the user from an auth provider.
+ */
 export type ProfileInfoApi = {
   getProfile(options?: ProfileInfoOptions): Promise<ProfileInfo | undefined>;
 };
 
+/**
+ * Profile information of the user from an auth provider.
+ */
 export type ProfileInfo = {
-  provider: string;
+  /**
+   * Email ID.
+   */
   email: string;
+  /**
+   * Display name that can be presented to the user.
+   */
   name?: string;
+  /**
+   * URL to an avatar image of the user.
+   */
   picture?: string;
 };
 
+/**
+ * Session state values passed to subscribers of the SessionStateApi.
+ */
 export enum SessionState {
   SignedIn = 'SignedIn',
   SignedOut = 'SignedOut',
 }
 
+/**
+ * This API provides access to an sessionState$ observable which provides an update when the
+ * user performs a sign in or sign out from an auth provider.
+ */
 export type SessionStateApi = {
   sessionState$(): Observable<SessionState>;
 };
+
 /**
  * Provides authentication towards Google APIs and identities.
  *

packages/core-api/src/app/App.tsx

@@ -282,7 +282,7 @@ export class PrivateAppImpl implements BackstageApp {
       const configApi = useApi(configApiRef);
 
       let { pathname } = new URL(
-        configApi.getString('app.baseUrl') ?? '/',
+        configApi.getOptionalString('app.baseUrl') ?? '/',
         'http://dummy.dev', // baseUrl can be specified as just a path
       );
       if (pathname.endsWith('/')) {

packages/core/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli build --outputs types,esm",
@@ -46,7 +45,7 @@
     "rc-progress": "^3.0.0",
     "react": "^16.12.0",
     "react-dom": "^16.12.0",
-    "react-helmet": "6.0.0",
+    "react-helmet": "6.1.0",
     "react-hook-form": "^5.7.2",
     "react-router": "6.0.0-alpha.5",
     "react-router-dom": "6.0.0-alpha.5",

packages/core/src/api-wrappers/createApp.test.tsx

@@ -15,6 +15,7 @@
  */
 
 import { defaultConfigLoader } from './createApp';
+import { AppConfig } from '@backstage/config';
 
 describe('defaultConfigLoader', () => {
   afterEach(() => {
@@ -24,24 +25,33 @@ describe('defaultConfigLoader', () => {
   it('loads static config', async () => {
     Object.defineProperty(process.env, 'APP_CONFIG', {
       configurable: true,
-      value: [{ my: 'config' }, { my: 'override-config' }] as any,
+      value: [
+        { data: { my: 'config' }, context: 'a' },
+        { data: { my: 'override-config' }, context: 'b' },
+      ] as AppConfig[],
     });
     const configs = await defaultConfigLoader();
-    expect(configs).toEqual([{ my: 'config' }, { my: 'override-config' }]);
+    expect(configs).toEqual([
+      { data: { my: 'config' }, context: 'a' },
+      { data: { my: 'override-config' }, context: 'b' },
+    ]);
   });
 
   it('loads runtime config', async () => {
     Object.defineProperty(process.env, 'APP_CONFIG', {
       configurable: true,
-      value: [{ my: 'override-config' }, { my: 'config' }] as any,
+      value: [
+        { data: { my: 'override-config' }, context: 'a' },
+        { data: { my: 'config' }, context: 'b' },
+      ] as AppConfig[],
     });
     const configs = await (defaultConfigLoader as any)(
       '{"my":"runtime-config"}',
     );
     expect(configs).toEqual([
-      { my: 'runtime-config' },
-      { my: 'override-config' },
-      { my: 'config' },
+      { data: { my: 'runtime-config' }, context: 'env' },
+      { data: { my: 'override-config' }, context: 'a' },
+      { data: { my: 'config' }, context: 'b' },
     ]);
   });
 
@@ -64,7 +74,7 @@ describe('defaultConfigLoader', () => {
   it('fails to load bad runtime config', async () => {
     Object.defineProperty(process.env, 'APP_CONFIG', {
       configurable: true,
-      value: [{ my: 'config' }] as any,
+      value: [{ data: { my: 'config' }, context: 'a' }] as AppConfig[],
     });
 
     await expect((defaultConfigLoader as any)('}')).rejects.toThrow(

packages/core/src/api-wrappers/createApp.tsx

@@ -27,7 +27,7 @@ import { BrowserRouter, MemoryRouter } from 'react-router-dom';
 import { ErrorPage } from '../layout/ErrorPage';
 import Progress from '../components/Progress';
 import { lightTheme, darkTheme } from '@backstage/theme';
-import { AppConfig } from '@backstage/config';
+import { AppConfig, JsonObject } from '@backstage/config';
 
 const { PrivateAppImpl } = privateExports;
 
@@ -59,7 +59,8 @@ export const defaultConfigLoader: AppConfigLoader = async (
   // Avoiding this string also being replaced at runtime
   if (runtimeConfigJson !== '__app_injected_runtime_config__'.toUpperCase()) {
     try {
-      configs.unshift(JSON.parse(runtimeConfigJson));
+      const data = JSON.parse(runtimeConfigJson) as JsonObject;
+      configs.unshift({ data, context: 'env' });
     } catch (error) {
       throw new Error(`Failed to load runtime configuration, ${error}`);
     }

packages/core/src/layout/Sidebar/Items.tsx

@@ -233,5 +233,5 @@ export const SidebarDivider = styled('hr')({
   width: '100%',
   background: '#383838',
   border: 'none',
-  margin: 0,
+  margin: '12px 0px',
 });

packages/core/src/layout/Sidebar/Settings/UserProfile.tsx

@@ -104,7 +104,7 @@ export const UserProfile: FC<{ open: boolean; setOpen: Function }> = ({
         onClick={handleClick}
         icon={avatar || AccountCircleIcon}
       >
-        {open ? <ExpandLess /> : <ExpandMore />}
+        {open ? <ExpandMore /> : <ExpandLess />}
       </SidebarItem>
     </>
   );

packages/core/src/layout/SignInPage/SignInPage.tsx

@@ -39,7 +39,7 @@ export const SignInPage: FC<Props> = ({ onResult, providers }) => {
 
   return (
     <Page>
-      <Header title={configApi.getString('app.title') ?? 'Backstage'} />
+      <Header title={configApi.getString('app.title')} />
       <Content>
         <ContentHeader title="Select a sign-in method" />
         <Grid container>{providerElements}</Grid>

packages/dev-utils/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli build --outputs types,esm",

packages/storybook/.storybook/main.js

@@ -16,7 +16,7 @@ module.exports = {
     const coreSrc = path.resolve(__dirname, '../../core/src');
 
     // Mirror config in packages/cli/src/lib/bundler
-    config.resolve.mainFields = ['main:src', 'browser', 'module', 'main'];
+    config.resolve.mainFields = ['browser', 'module', 'main'];
 
     // Remove the default babel-loader for js files, we're using sucrase instead
     const [jsLoader] = config.module.rules.splice(0, 1);

packages/test-utils-core/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli build --outputs types,esm",

packages/test-utils/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli build --outputs types,esm",

packages/theme/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli build --outputs types,esm",

plugins/auth-backend/src/lib/EnvironmentHandler.ts

@@ -57,6 +57,8 @@ export class EnvironmentHandler implements AuthProviderRouteHandlers {
 
   async logout(req: express.Request, res: express.Response): Promise<void> {
     const provider = this.getProviderForEnv(req);
-    await provider.logout(req, res);
+    if (provider.logout) {
+      await provider.logout(req, res);
+    }
   }
 }

plugins/auth-backend/src/lib/PassportStrategyHelper.ts

@@ -21,13 +21,14 @@ import {
   RedirectInfo,
   RefreshTokenResponse,
   ProfileInfo,
+  ProviderStrategy,
 } from '../providers/types';
 
 export const makeProfileInfo = (
   profile: passport.Profile,
   params: any,
 ): ProfileInfo => {
-  const { provider, displayName: name } = profile;
+  const { displayName: name } = profile;
 
   let email = '';
   if (profile.emails) {
@@ -51,7 +52,6 @@ export const makeProfileInfo = (
   }
 
   return {
-    provider,
     name,
     email,
     picture,
@@ -100,12 +100,12 @@ export const executeFrameHandlerStrategy = async (
 };
 
 export const executeRefreshTokenStrategy = async (
-  providerstrategy: passport.Strategy,
+  providerStrategy: passport.Strategy,
   refreshToken: string,
   scope: string,
 ): Promise<RefreshTokenResponse> => {
   return new Promise((resolve, reject) => {
-    const anyStrategy = providerstrategy as any;
+    const anyStrategy = providerStrategy as any;
     const OAuth2 = anyStrategy._oauth2.constructor;
     const oauth2 = new OAuth2(
       anyStrategy._oauth2._clientId,
@@ -149,12 +149,12 @@ export const executeRefreshTokenStrategy = async (
 };
 
 export const executeFetchUserProfileStrategy = async (
-  providerstrategy: passport.Strategy,
+  providerStrategy: passport.Strategy,
   accessToken: string,
   params: any,
 ): Promise<ProfileInfo> => {
   return new Promise((resolve, reject) => {
-    const anyStrategy = providerstrategy as any;
+    const anyStrategy = (providerStrategy as unknown) as ProviderStrategy;
     anyStrategy.userProfile(
       accessToken,
       (error: Error, passportProfile: passport.Profile) => {

plugins/auth-backend/src/providers/factories.ts

@@ -44,7 +44,9 @@ export const createAuthProviderRouter = (
   router.get('/start', provider.start.bind(provider));
   router.get('/handler/frame', provider.frameHandler.bind(provider));
   router.post('/handler/frame', provider.frameHandler.bind(provider));
-  router.post('/logout', provider.logout.bind(provider));
+  if (provider.logout) {
+    router.post('/logout', provider.logout.bind(provider));
+  }
   if (provider.refresh) {
     router.get('/refresh', provider.refresh.bind(provider));
   }

plugins/auth-backend/src/providers/types.ts

@@ -18,48 +18,163 @@ import express from 'express';
 import { Logger } from 'winston';
 
 export type OAuthProviderOptions = {
+  /**
+   * Client ID of the auth provider.
+   */
   clientID: string;
+  /**
+   * Client Secret of the auth provider.
+   */
   clientSecret: string;
+  /**
+   * Callback URL to be passed to the auth provider to redirect to after the user signs in.
+   */
   callbackURL: string;
 };
 
-export type SAMLProviderConfig = {
-  entryPoint: string;
-  issuer: string;
-};
-
-export type EnvironmentProviderConfig = {
-  [key: string]: OAuthProviderConfig | SAMLProviderConfig;
-};
-
-export type AuthProviderConfig = {
-  baseUrl: string;
-};
-
 export type OAuthProviderConfig = {
+  /**
+   * Cookies can be marked with a secure flag to send cookies only when the request
+   * is over an encrypted channel (HTTPS).
+   *
+   * For development environment we don't mark the cookie as secure since we serve
+   * localhost over HTTP.
+   */
   secure: boolean;
-  appOrigin: string; // http://localhost:3000
+  /**
+   * The protocol://domain[:port] where the app (frontend) is hosted. This is used to post messages back
+   * to the window that initiates an auth request.
+   */
+  appOrigin: string;
+  /**
+   * Client ID of the auth provider.
+   */
   clientId: string;
+  /**
+   * Client Secret of the auth provider.
+   */
   clientSecret: string;
 };
 
+export type EnvironmentProviderConfig = {
+  /**
+   * key, values are environment names and OAuthProviderConfigs
+   *
+   * For e.g
+   * {
+   *   development: DevelopmentOAuthProviderConfig
+   *   production: ProductionOAuthProviderConfig
+   * }
+   */
+  [key: string]: OAuthProviderConfig;
+};
+
+export type AuthProviderConfig = {
+  /**
+   * The protocol://domain[:port] where the app is hosted. This is used to construct the
+   * callbackURL to redirect to once the user signs in to the auth provider.
+   */
+  baseUrl: string;
+};
+
+/**
+ * Any OAuth provider needs to implement this interface which has provider specific
+ * handlers for different methods to perform authentication, get access tokens,
+ * refresh tokens and perform sign out.
+ */
 export interface OAuthProviderHandlers {
+  /**
+   * This method initiates a sign in request with an auth provider.
+   * @param {express.Request} req
+   * @param options
+   */
   start(req: express.Request, options: any): Promise<any>;
+
+  /**
+   * Handles the redirect from the auth provider when the user has signed in.
+   * @param {express.Request} req
+   */
   handler(req: express.Request): Promise<any>;
+
+  /**
+   * (Optional) Given a refresh token and scope fetches a new access token from the auth provider.
+   * @param {string} refreshToken
+   * @param {string} scope
+   */
   refresh?(refreshToken: string, scope: string): Promise<any>;
+
+  /**
+   * (Optional) Sign out of the auth provider.
+   */
   logout?(): Promise<any>;
 }
 
+/**
+ * Any Auth provider needs to implement this interface which handles the routes in the
+ * auth backend. Any auth API requests from the frontend reaches these methods.
+ *
+ * The routes in the auth backend API are tied to these methods like below
+ *
+ * /auth/[provider]/start -> start
+ * /auth/[provider]/handler/frame -> frameHandler
+ * /auth/[provider]/refresh -> refresh
+ * /auth/[provider]/logout -> logout
+ */
 export interface AuthProviderRouteHandlers {
+  /**
+   * Handles the start route of the API. This initiates a sign in request with an auth provider.
+   *
+   * Request
+   * - scopes for the auth request (Optional)
+   * Response
+   * - redirect to the auth provider for the user to sign in or consent.
+   * - sets a nonce cookie and also pass the nonce as 'state' query parameter in the redirect request
+   *
+   * @param {express.Request} req
+   * @param {express.Response} res
+   */
   start(req: express.Request, res: express.Response): Promise<any>;
-  frameHandler(req: express.Request, res: express.Response): Promise<any>;
-  refresh?(req: express.Request, res: express.Response): Promise<any>;
-  logout(req: express.Request, res: express.Response): Promise<any>;
-}
 
-export type SAMLEnvironmentProviderConfig = {
-  [key: string]: SAMLProviderConfig;
-};
+  /**
+   * Once the user signs in or consents in the OAuth screen, the auth provider redirects to the
+   * callbackURL which is handled by this method.
+   *
+   * Request
+   * - to contain a nonce cookie and a 'state' query parameter
+   * Response
+   * - postMessage to the window with a payload that contains accessToken, expiryInSeconds?, idToken? and scope.
+   * - sets a refresh token cookie if the auth provider supports refresh tokens
+   *
+   * @param {express.Request} req
+   * @param {express.Response} res
+   */
+  frameHandler(req: express.Request, res: express.Response): Promise<any>;
+
+  /**
+   * (Optional) If the auth provider supports refresh tokens then this method handles
+   * requests to get a new access token.
+   *
+   * Request
+   * - to contain a refresh token cookie and scope (Optional) query parameter.
+   * Response
+   * - payload with accessToken, expiryInSeconds?, idToken?, scope and user profile information.
+   *
+   * @param {express.Request} req
+   * @param {express.Response} res
+   */
+  refresh?(req: express.Request, res: express.Response): Promise<any>;
+
+  /**
+   * (Optional) Handles sign out requests
+   *
+   * Response
+   * - removes the refresh token cookie
+   *
+   * @param {express.Request} req
+   * @param {express.Response} res
+   */
+  logout?(req: express.Request, res: express.Response): Promise<any>;
+}
 
 export type AuthProviderFactory = (
   globalConfig: AuthProviderConfig,
@@ -68,27 +183,42 @@ export type AuthProviderFactory = (
 ) => AuthProviderRouteHandlers;
 
 export type AuthInfoBase = {
+  /**
+   * An access token issued for the signed in user.
+   */
   accessToken: string;
+  /**
+   * (Optional) Id token issued for the signed in user.
+   */
   idToken?: string;
+  /**
+   * Expiry of the access token in seconds.
+   */
   expiresInSeconds?: number;
+  /**
+   * Scopes granted for the access token.
+   */
   scope: string;
 };
 
 export type AuthInfoWithProfile = AuthInfoBase & {
-  profile:
-    | {
-        provider: string;
-        email: string;
-        name?: string;
-        picture?: string;
-      }
-    | undefined;
+  /**
+   * Profile information of the signed in user.
+   */
+  profile: ProfileInfo | undefined;
 };
 
 export type AuthInfoPrivate = {
+  /**
+   * A refresh token issued for the signed in user.
+   */
   refreshToken: string;
 };
 
+/**
+ * Payload sent as a post message after the auth request is complete.
+ * If successful then has a valid payload with Auth information else contains an error.
+ */
 export type AuthResponse =
   | {
       type: 'auth-result';
@@ -100,18 +230,49 @@ export type AuthResponse =
     };
 
 export type RedirectInfo = {
+  /**
+   * URL to redirect to
+   */
   url: string;
+  /**
+   * Status code to use for the redirect
+   */
   status?: number;
 };
 
 export type ProfileInfo = {
-  provider: string;
+  /**
+   * Email ID of the signed in user.
+   */
   email: string;
+  /**
+   * Display name that can be presented to the signed in user.
+   */
   name: string;
+  /**
+   * URL to an image that can be used as the display image or avatar of the
+   * signed in user.
+   */
   picture: string;
 };
 
 export type RefreshTokenResponse = {
+  /**
+   * An access token issued for the signed in user.
+   */
   accessToken: string;
   params: any;
 };
+
+export type ProviderStrategy = {
+  userProfile(accessToken: string, callback: Function): void;
+};
+
+export type SAMLProviderConfig = {
+  entryPoint: string;
+  issuer: string;
+};
+
+export type SAMLEnvironmentProviderConfig = {
+  [key: string]: SAMLProviderConfig;
+};

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts

@@ -62,6 +62,11 @@ describe('DatabaseEntitiesCatalog', () => {
       const result = await catalog.addOrUpdateEntity(entity);
 
       expect(db.entities).toHaveBeenCalledTimes(1);
+      expect(db.entities).toHaveBeenCalledWith(expect.anything(), [
+        { key: 'kind', values: ['b'] },
+        { key: 'name', values: ['c'] },
+        { key: 'namespace', values: ['d'] },
+      ]);
       expect(db.addEntity).toHaveBeenCalledTimes(1);
       expect(result).toBe(entity);
     });
@@ -71,20 +76,52 @@ describe('DatabaseEntitiesCatalog', () => {
         apiVersion: 'a',
         kind: 'b',
         metadata: {
-          uid: 'uuuu',
+          uid: 'u',
           name: 'c',
           namespace: 'd',
         },
       };
 
-      db.entities.mockResolvedValue([]);
+      db.entityByUid.mockResolvedValue({
+        entity: {
+          apiVersion: 'a',
+          kind: 'b',
+          metadata: {
+            uid: 'u',
+            etag: 'e',
+            generation: 1,
+            name: 'c',
+            namespace: 'd',
+          },
+        },
+      });
       db.updateEntity.mockResolvedValue({ entity });
 
       const catalog = new DatabaseEntitiesCatalog(db);
       const result = await catalog.addOrUpdateEntity(entity);
 
       expect(db.entities).toHaveBeenCalledTimes(0);
+      expect(db.entityByUid).toHaveBeenCalledTimes(1);
+      expect(db.entityByUid).toHaveBeenCalledWith(expect.anything(), 'u');
       expect(db.updateEntity).toHaveBeenCalledTimes(1);
+      expect(db.updateEntity).toHaveBeenCalledWith(
+        expect.anything(),
+        {
+          entity: {
+            apiVersion: 'a',
+            kind: 'b',
+            metadata: {
+              uid: 'u',
+              etag: 'e',
+              generation: 1,
+              name: 'c',
+              namespace: 'd',
+            },
+          },
+        },
+        'e',
+        1,
+      );
       expect(result).toBe(entity);
     });
 
@@ -101,19 +138,45 @@ describe('DatabaseEntitiesCatalog', () => {
         apiVersion: 'a',
         kind: 'b',
         metadata: {
+          uid: 'u',
+          etag: 'e',
+          generation: 1,
           name: 'c',
           namespace: 'd',
         },
       };
 
       db.entities.mockResolvedValue([{ entity: existing }]);
-      db.updateEntity.mockResolvedValue({ entity: added });
+      db.updateEntity.mockResolvedValue({ entity: existing });
 
       const catalog = new DatabaseEntitiesCatalog(db);
       const result = await catalog.addOrUpdateEntity(added);
 
       expect(db.entities).toHaveBeenCalledTimes(1);
+      expect(db.entities).toHaveBeenCalledWith(expect.anything(), [
+        { key: 'kind', values: ['b'] },
+        { key: 'name', values: ['c'] },
+        { key: 'namespace', values: ['d'] },
+      ]);
       expect(db.updateEntity).toHaveBeenCalledTimes(1);
+      expect(db.updateEntity).toHaveBeenCalledWith(
+        expect.anything(),
+        {
+          entity: {
+            apiVersion: 'a',
+            kind: 'b',
+            metadata: {
+              uid: 'u',
+              etag: 'e',
+              generation: 1,
+              name: 'c',
+              namespace: 'd',
+            },
+          },
+        },
+        'e',
+        1,
+      );
       expect(result).toEqual(existing);
     });
   });

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -15,7 +15,10 @@
  */
 
 import { NotFoundError } from '@backstage/backend-common';
-import { LOCATION_ANNOTATION } from '@backstage/catalog-model';
+import {
+  generateUpdatedEntity,
+  LOCATION_ANNOTATION,
+} from '@backstage/catalog-model';
 import type { Entity } from '@backstage/catalog-model';
 import type { Database, DbEntityResponse, EntityFilters } from '../database';
 import type { EntitiesCatalog } from './types';
@@ -43,9 +46,10 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
     namespace: string | undefined,
     name: string,
   ): Promise<Entity | undefined> {
-    return await this.database.transaction(tx =>
+    const response = await this.database.transaction(tx =>
       this.entityByNameInternal(tx, kind, name, namespace),
     );
+    return response?.entity;
   }
 
   async addOrUpdateEntity(
@@ -53,25 +57,30 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
     locationId?: string,
   ): Promise<Entity> {
     return await this.database.transaction(async tx => {
-      let response: DbEntityResponse;
+      // Find a matching (by uid, or by compound name, depending on the given
+      // entity) existing entity, to know whether to update or add
+      const existing = entity.metadata.uid
+        ? await this.database.entityByUid(tx, entity.metadata.uid)
+        : await this.entityByNameInternal(
+            tx,
+            entity.kind,
+            entity.metadata.name,
+            entity.metadata.namespace,
+          );
 
-      if (entity.metadata.uid) {
-        response = await this.database.updateEntity(tx, { locationId, entity });
-      } else {
-        const existing = await this.entityByNameInternal(
+      // If it's an update, run the algorithm for annotation merging, updating
+      // etag/generation, etc.
+      let response: DbEntityResponse;
+      if (existing) {
+        const updated = generateUpdatedEntity(existing.entity, entity);
+        response = await this.database.updateEntity(
           tx,
-          entity.kind,
-          entity.metadata.name,
-          entity.metadata.namespace,
+          { locationId, entity: updated },
+          existing.entity.metadata.etag,
+          existing.entity.metadata.generation,
         );
-        if (existing) {
-          response = await this.database.updateEntity(tx, {
-            locationId,
-            entity,
-          });
-        } else {
-          response = await this.database.addEntity(tx, { locationId, entity });
-        }
+      } else {
+        response = await this.database.addEntity(tx, { locationId, entity });
       }
 
       return response.entity;
@@ -110,7 +119,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
     kind: string,
     name: string,
     namespace: string | undefined,
-  ): Promise<Entity | undefined> {
+  ): Promise<DbEntityResponse | undefined> {
     const matches = await this.database.entities(tx, [
       { key: 'kind', values: [kind] },
       { key: 'name', values: [name] },
@@ -123,6 +132,6 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
       },
     ]);
 
-    return matches.length ? matches[0].entity : undefined;
+    return matches.length ? matches[0] : undefined;
   }
 }

plugins/catalog-backend/src/database/CommonDatabase.test.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { ConflictError, NotFoundError } from '@backstage/backend-common';
+import { ConflictError } from '@backstage/backend-common';
 import type { Entity, Location } from '@backstage/catalog-model';
 import { DatabaseManager } from './DatabaseManager';
 import { Database, DatabaseLocationUpdateLogStatus } from './types';
@@ -199,9 +199,7 @@ describe('CommonDatabase', () => {
       );
       expect(updated.entity.apiVersion).toEqual(added.entity.apiVersion);
       expect(updated.entity.kind).toEqual(added.entity.kind);
-      expect(updated.entity.metadata.etag).not.toEqual(
-        added.entity.metadata.etag,
-      );
+      expect(updated.entity.metadata.etag).toEqual(added.entity.metadata.etag);
       expect(updated.entity.metadata.generation).toEqual(
         added.entity.metadata.generation,
       );
@@ -220,41 +218,21 @@ describe('CommonDatabase', () => {
       expect(updated.entity.metadata.name).toEqual('new!');
     });
 
-    it('can update fields if kind, name, and namespace match', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
-      added.entity.apiVersion = 'something.new';
-      delete added.entity.metadata.uid;
-      delete added.entity.metadata.generation;
-      const updated = await db.transaction(tx =>
-        db.updateEntity(tx, { entity: added.entity }),
-      );
-      expect(updated.entity.apiVersion).toEqual('something.new');
-    });
-
-    it('rejects if kind, name, but not namespace match', async () => {
-      const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
-      added.entity.apiVersion = 'something.new';
-      delete added.entity.metadata.uid;
-      delete added.entity.metadata.generation;
-      added.entity.metadata.namespace = 'something.wrong';
-      await expect(
-        db.transaction(tx => db.updateEntity(tx, { entity: added.entity })),
-      ).rejects.toThrow(NotFoundError);
-    });
-
     it('fails to update an entity if etag does not match', async () => {
       const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
-      added.entity.metadata.etag = 'garbage';
       await expect(
-        db.transaction(tx => db.updateEntity(tx, { entity: added.entity })),
+        db.transaction(tx =>
+          db.updateEntity(tx, { entity: added.entity }, 'garbage'),
+        ),
       ).rejects.toThrow(ConflictError);
     });
 
     it('fails to update an entity if generation does not match', async () => {
       const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
-      added.entity.metadata.generation! += 100;
       await expect(
-        db.transaction(tx => db.updateEntity(tx, { entity: added.entity })),
+        db.transaction(tx =>
+          db.updateEntity(tx, { entity: added.entity }, undefined, 1e20),
+        ),
       ).rejects.toThrow(ConflictError);
     });
   });

plugins/catalog-backend/src/database/CommonDatabase.ts

@@ -19,7 +19,14 @@ import {
   InputError,
   NotFoundError,
 } from '@backstage/backend-common';
-import { Entity, EntityMeta, Location } from '@backstage/catalog-model';
+import {
+  Entity,
+  EntityMeta,
+  entityMetaGeneratedFields,
+  generateEntityEtag,
+  generateEntityUid,
+  Location,
+} from '@backstage/catalog-model';
 import Knex from 'knex';
 import lodash from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
@@ -38,88 +45,9 @@ import type {
   EntityFilters,
 } from './types';
 
-function getStrippedMetadata(metadata: EntityMeta): EntityMeta {
-  const output = lodash.cloneDeep(metadata);
-  delete output.uid;
-  delete output.etag;
-  delete output.generation;
-  return output;
-}
-
-function serializeMetadata(metadata: EntityMeta): string {
-  return JSON.stringify(getStrippedMetadata(metadata));
-}
-
-function serializeSpec(spec: Entity['spec']): DbEntitiesRow['spec'] {
-  if (!spec) {
-    return null;
-  }
-
-  return JSON.stringify(spec);
-}
-
-function toEntityRow(
-  locationId: string | undefined,
-  entity: Entity,
-): DbEntitiesRow {
-  return {
-    id: entity.metadata.uid!,
-    location_id: locationId || null,
-    etag: entity.metadata.etag!,
-    generation: entity.metadata.generation!,
-    api_version: entity.apiVersion,
-    kind: entity.kind,
-    name: entity.metadata.name,
-    namespace: entity.metadata.namespace || null,
-    metadata: serializeMetadata(entity.metadata),
-    spec: serializeSpec(entity.spec),
-  };
-}
-
-function toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
-  const entity: Entity = {
-    apiVersion: row.api_version,
-    kind: row.kind,
-    metadata: {
-      ...(JSON.parse(row.metadata) as Entity['metadata']),
-      uid: row.id,
-      etag: row.etag,
-      generation: Number(row.generation), // cast because of sqlite
-    },
-  };
-
-  if (row.spec) {
-    const spec = JSON.parse(row.spec);
-    entity.spec = spec;
-  }
-
-  return {
-    locationId: row.location_id || undefined,
-    entity,
-  };
-}
-
-function specsAreEqual(
-  first: string | null,
-  second: object | undefined,
-): boolean {
-  if (!first && !second) {
-    return true;
-  } else if (!first || !second) {
-    return false;
-  }
-
-  return lodash.isEqual(JSON.parse(first), second);
-}
-
-function generateUid(): string {
-  return uuidv4();
-}
-
-function generateEtag(): string {
-  return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, '');
-}
-
+/**
+ * The core database implementation.
+ */
 export class CommonDatabase implements Database {
   constructor(
     private readonly database: Knex,
@@ -163,12 +91,12 @@ export class CommonDatabase implements Database {
     const newEntity = lodash.cloneDeep(request.entity);
     newEntity.metadata = {
       ...newEntity.metadata,
-      uid: generateUid(),
-      etag: generateEtag(),
+      uid: generateEntityUid(),
+      etag: generateEntityEtag(),
       generation: 1,
     };
 
-    const newRow = toEntityRow(request.locationId, newEntity);
+    const newRow = this.toEntityRow(request.locationId, newEntity);
     await tx<DbEntitiesRow>('entities').insert(newRow);
     await this.updateEntitiesSearch(tx, newRow.id, newEntity);
 
@@ -178,35 +106,20 @@ export class CommonDatabase implements Database {
   async updateEntity(
     txOpaque: unknown,
     request: DbEntityRequest,
+    matchingEtag?: string,
+    matchingGeneration?: number,
   ): Promise<DbEntityResponse> {
     const tx = txOpaque as Knex.Transaction<any, any>;
 
-    const { kind } = request.entity;
-    const {
-      uid,
-      etag: expectedOldEtag,
-      generation: expectedOldGeneration,
-      name,
-      namespace,
-    } = request.entity.metadata ?? {};
+    const { uid } = request.entity.metadata;
 
-    // Find existing entities that match the given metadata
-    let entitySelector: Partial<DbEntitiesRow>;
-    if (uid) {
-      entitySelector = { id: uid };
-    } else if (kind && name) {
-      entitySelector = {
-        kind,
-        name: name,
-        namespace: namespace || null,
-      };
-    } else {
-      throw new InputError(
-        'Must specify either uid, or kind + name + namespace to be able to identify an entity',
-      );
+    if (uid === undefined) {
+      throw new InputError('Must specify uid when updating entities');
     }
+
+    // Find existing entity
     const oldRows = await tx<DbEntitiesRow>('entities')
-      .where(entitySelector)
+      .where({ id: uid })
       .select();
     if (oldRows.length !== 1) {
       throw new NotFoundError('No matching entity found');
@@ -217,51 +130,26 @@ export class CommonDatabase implements Database {
     // The Number cast is here because sqlite reads it as a string, no matter
     // what the table actually says
     oldRow.generation = Number(oldRow.generation);
-    if (expectedOldEtag) {
-      if (expectedOldEtag !== oldRow.etag) {
+    if (matchingEtag) {
+      if (matchingEtag !== oldRow.etag) {
         throw new ConflictError(
-          `Etag mismatch, expected="${expectedOldEtag}" found="${oldRow.etag}"`,
+          `Etag mismatch, expected="${matchingEtag}" found="${oldRow.etag}"`,
         );
       }
     }
-    if (expectedOldGeneration) {
-      if (expectedOldGeneration !== oldRow.generation) {
+    if (matchingGeneration) {
+      if (matchingGeneration !== oldRow.generation) {
         throw new ConflictError(
-          `Generation mismatch, expected="${expectedOldGeneration}" found="${oldRow.generation}"`,
+          `Generation mismatch, expected="${matchingGeneration}" found="${oldRow.generation}"`,
         );
       }
     }
 
-    // Build the new shape of the entity
-    const newEtag = generateEtag();
-    const newGeneration = specsAreEqual(oldRow.spec, request.entity.spec)
-      ? oldRow.generation
-      : oldRow.generation + 1;
-    const newEntity = lodash.cloneDeep(request.entity);
-    newEntity.metadata = {
-      ...newEntity.metadata,
-      uid: oldRow.id,
-      etag: newEtag,
-      generation: newGeneration,
-    };
-
-    // Preserve annotations that were set on the old version of the entity,
-    // unless the new version overwrites them
-    if (oldRow.metadata) {
-      const oldMetadata = JSON.parse(oldRow.metadata) as EntityMeta;
-      if (oldMetadata.annotations) {
-        newEntity.metadata.annotations = {
-          ...oldMetadata.annotations,
-          ...newEntity.metadata.annotations,
-        };
-      }
-    }
-
-    await this.ensureNoSimilarNames(tx, newEntity);
+    await this.ensureNoSimilarNames(tx, request.entity);
 
     // Store the updated entity; select on the old etag to ensure that we do
     // not lose to another writer
-    const newRow = toEntityRow(request.locationId, newEntity);
+    const newRow = this.toEntityRow(request.locationId, request.entity);
     const updatedRows = await tx<DbEntitiesRow>('entities')
       .where({ id: oldRow.id, etag: oldRow.etag })
       .update(newRow);
@@ -271,8 +159,9 @@ export class CommonDatabase implements Database {
       throw new ConflictError(`Failed to update entity`);
     }
 
-    await this.updateEntitiesSearch(tx, oldRow.id, newEntity);
-    return { locationId: request.locationId, entity: newEntity };
+    await this.updateEntitiesSearch(tx, oldRow.id, request.entity);
+
+    return request;
   }
 
   async entities(
@@ -328,7 +217,7 @@ export class CommonDatabase implements Database {
       .orderBy('name', 'asc')
       .groupBy('id');
 
-    return rows.map(row => toEntityResponse(row));
+    return rows.map(row => this.toEntityResponse(row));
   }
 
   async entity(
@@ -347,7 +236,7 @@ export class CommonDatabase implements Database {
       return undefined;
     }
 
-    return toEntityResponse(rows[0]);
+    return this.toEntityResponse(rows[0]);
   }
 
   async entityByUid(
@@ -362,7 +251,7 @@ export class CommonDatabase implements Database {
       return undefined;
     }
 
-    return toEntityResponse(rows[0]);
+    return this.toEntityResponse(rows[0]);
   }
 
   async removeEntity(txOpaque: unknown, uid: string): Promise<void> {
@@ -524,4 +413,47 @@ export class CommonDatabase implements Database {
       }
     }
   }
+
+  private toEntityRow(
+    locationId: string | undefined,
+    entity: Entity,
+  ): DbEntitiesRow {
+    return {
+      id: entity.metadata.uid!,
+      location_id: locationId || null,
+      etag: entity.metadata.etag!,
+      generation: entity.metadata.generation!,
+      api_version: entity.apiVersion,
+      kind: entity.kind,
+      name: entity.metadata.name,
+      namespace: entity.metadata.namespace || null,
+      metadata: JSON.stringify(
+        lodash.omit(entity.metadata, ...entityMetaGeneratedFields),
+      ),
+      spec: entity.spec ? JSON.stringify(entity.spec) : null,
+    };
+  }
+
+  private toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
+    const entity: Entity = {
+      apiVersion: row.api_version,
+      kind: row.kind,
+      metadata: {
+        ...(JSON.parse(row.metadata) as EntityMeta),
+        uid: row.id,
+        etag: row.etag,
+        generation: Number(row.generation), // cast because of sqlite
+      },
+    };
+
+    if (row.spec) {
+      const spec = JSON.parse(row.spec);
+      entity.spec = spec;
+    }
+
+    return {
+      locationId: row.location_id || undefined,
+      entity,
+    };
+  }
 }

plugins/catalog-backend/src/database/types.ts

@@ -104,21 +104,27 @@ export type Database = {
   /**
    * Updates an existing entity in the catalog.
    *
-   * The given entity must contain enough information to identify an already
-   * stored entity in the catalog - either by uid, or by kind + namespace +
-   * name. If no matching entity is found, the operation fails.
+   * The given entity must contain an uid to identify an already stored entity
+   * in the catalog. If it is missing or if no matching entity is found, the
+   * operation fails.
    *
-   * If etag or generation are given, they are taken into account. Attempts to
-   * update a matching entity, but where the etag and/or generation are not
-   * equal to the passed values, will fail.
+   * If matchingEtag or matchingGeneration are given, they are taken into
+   * account. Attempts to update a matching entity, but where the etag and/or
+   * generation are not equal to the passed values, will fail.
    *
    * @param tx An ongoing transaction
    * @param request The entity being updated
+   * @param matchingEtag If specified, reject with ConflictError if not
+   *                     matching the entry in the database
+   * @param matchingGeneration If specified, reject with ConflictError if not
+   *                           matching the entry in the database
    * @returns The updated entity
    */
   updateEntity(
     tx: unknown,
     request: DbEntityRequest,
+    matchingEtag?: string,
+    matchingGeneration?: number,
   ): Promise<DbEntityResponse>;
 
   entities(tx: unknown, filters?: EntityFilters): Promise<DbEntityResponse[]>;

plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts

@@ -15,8 +15,12 @@
  */
 
 import { InputError } from '@backstage/backend-common';
-import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
-import lodash from 'lodash';
+import {
+  Entity,
+  entityHasChanges,
+  Location,
+  LocationSpec,
+} from '@backstage/catalog-model';
 import { v4 as uuidv4 } from 'uuid';
 import { Logger } from 'winston';
 import { EntitiesCatalog, LocationsCatalog } from '../catalog';
@@ -173,7 +177,7 @@ export class HigherOrderOperations implements HigherOrderOperation {
         if (!previous) {
           this.logger.debug(`No such entity found, adding`);
           await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
-        } else if (!this.entitiesAreEqual(previous, entity)) {
+        } else if (entityHasChanges(previous, entity)) {
           this.logger.debug(`Different from existing entity, updating`);
           await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
         } else {
@@ -199,60 +203,4 @@ export class HigherOrderOperations implements HigherOrderOperation {
       }
     }
   }
-
-  // Compares entities, ignoring generated and irrelevant data
-  private entitiesAreEqual(previous: Entity, next: Entity): boolean {
-    if (
-      previous.apiVersion !== next.apiVersion ||
-      previous.kind !== next.kind ||
-      !lodash.isEqual(previous.spec, next.spec) // Accept that {} !== undefined
-    ) {
-      return false;
-    }
-
-    // Since the next annotations get merged into the previous, extract only
-    // the overlapping keys and check if their values match.
-    if (next.metadata.annotations) {
-      if (!previous.metadata.annotations) {
-        return false;
-      }
-      if (
-        !lodash.isEqual(
-          next.metadata.annotations,
-          lodash.pick(
-            previous.metadata.annotations,
-            Object.keys(next.metadata.annotations),
-          ),
-        )
-      ) {
-        return false;
-      }
-    }
-
-    const e1 = lodash.cloneDeep(previous);
-    const e2 = lodash.cloneDeep(next);
-
-    if (!e1.metadata.labels) {
-      e1.metadata.labels = {};
-    }
-    if (!e2.metadata.labels) {
-      e2.metadata.labels = {};
-    }
-
-    // Remove generated fields
-    delete e1.metadata.uid;
-    delete e1.metadata.etag;
-    delete e1.metadata.generation;
-    delete e2.metadata.uid;
-    delete e2.metadata.etag;
-    delete e2.metadata.generation;
-
-    // Remove already compared things
-    delete e1.metadata.annotations;
-    delete e1.spec;
-    delete e2.metadata.annotations;
-    delete e2.spec;
-
-    return lodash.isEqual(e1, e2);
-  }
 }

plugins/catalog/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-catalog",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/catalog/src/components/CatalogPage/CatalogPage.tsx

@@ -25,7 +25,13 @@ import {
 } from '@backstage/core';
 import CatalogLayout from './CatalogLayout';
 import { rootRoute as scaffolderRootRoute } from '@backstage/plugin-scaffolder';
-import { Button, Link, makeStyles, Typography } from '@material-ui/core';
+import {
+  Button,
+  Link,
+  makeStyles,
+  Typography,
+  withStyles,
+} from '@material-ui/core';
 import Edit from '@material-ui/icons/Edit';
 import GitHub from '@material-ui/icons/GitHub';
 import Star from '@material-ui/icons/Star';
@@ -113,6 +119,12 @@ export const CatalogPage: FC<{}> = () => {
 
   const styles = useStyles();
 
+  const YellowStar = withStyles({
+    root: {
+      color: '#f3ba37',
+    },
+  })(Star);
+
   const actions = [
     (rowData: Entity) => {
       const location = findLocationForEntityMeta(rowData.metadata);
@@ -152,7 +164,7 @@ export const CatalogPage: FC<{}> = () => {
     (rowData: Entity) => {
       const isStarred = isStarredEntity(rowData);
       return {
-        icon: isStarred ? Star : StarOutline,
+        icon: isStarred ? YellowStar : StarOutline,
         tooltip: isStarred ? 'Remove from favorites' : 'Add to favorites',
         onClick: () => toggleStarredEntity(rowData),
       };

plugins/catalog/src/components/EntityMetadataCard/EntityMetadataCard.tsx

@@ -23,7 +23,7 @@ type Props = {
 };
 
 export const EntityMetadataCard: FC<Props> = ({ entity }) => (
-  <InfoCard title="Metadata">
+  <InfoCard title="Information">
     <StructuredMetadataTable metadata={entity.metadata} />
   </InfoCard>
 );

plugins/catalog/src/components/EntityPage/EntityPage.tsx

@@ -149,11 +149,11 @@ export const EntityPage: FC<{}> = () => {
           <HeaderTabs tabs={tabs} />
 
           <Content>
-            <Grid container spacing={3} direction="column">
-              <Grid item>
+            <Grid container spacing={3}>
+              <Grid item sm={4}>
                 <EntityMetadataCard entity={entity} />
               </Grid>
-              <Grid item>
+              <Grid item sm={8}>
                 <SentryIssuesWidget
                   sentryProjectId="sample-sentry-project-id"
                   statsFor="24h"

plugins/circleci/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-circleci",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/explore/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-explore",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/gitops-profiles/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-gitops-profiles",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/graphiql/package.json

@@ -18,8 +18,7 @@
     "backstage"
   ],
   "license": "Apache-2.0",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "scripts": {
     "build": "backstage-cli plugin:build",

plugins/lighthouse/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-lighthouse",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/register-component/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-register-component",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/register-component/src/components/RegisterComponentResultDialog/RegisterComponentResultDialog.test.tsx

@@ -14,13 +14,12 @@
  * limitations under the License.
  */
 
-import React, { ComponentProps } from 'react';
-import { render, cleanup } from '@testing-library/react';
-import { RegisterComponentResultDialog } from './RegisterComponentResultDialog';
-import { ThemeProvider } from '@material-ui/core';
 import { lightTheme } from '@backstage/theme';
+import { ThemeProvider } from '@material-ui/core';
+import { cleanup, render } from '@testing-library/react';
+import React, { ComponentProps } from 'react';
 import { MemoryRouter } from 'react-router-dom';
-import { Entity } from '@backstage/catalog-model';
+import { RegisterComponentResultDialog } from './RegisterComponentResultDialog';
 
 const setup = (
   props?: Partial<ComponentProps<typeof RegisterComponentResultDialog>>,
@@ -52,6 +51,7 @@ it('should show a list of components if success', async () => {
   const { rendered } = setup({
     entities: [
       {
+        apiVersion: 'backstage.io/v1alpha1',
         kind: 'Component',
         metadata: {
           name: 'Component1',
@@ -61,6 +61,7 @@ it('should show a list of components if success', async () => {
         },
       },
       {
+        apiVersion: 'backstage.io/v1alpha1',
         kind: 'Component',
         metadata: {
           name: 'Component2',
@@ -69,7 +70,7 @@ it('should show a list of components if success', async () => {
           type: 'service',
         },
       },
-    ] as Entity[],
+    ],
   });
 
   expect(

plugins/scaffolder/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-scaffolder",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/scaffolder/src/components/ScaffolderPage/index.tsx

@@ -20,6 +20,7 @@ import {
   Content,
   ContentHeader,
   Header,
+  SupportButton,
   Page,
   pageTheme,
   useApi,
@@ -60,7 +61,7 @@ const ScaffolderPage: React.FC<{}> = () => {
         pageTitleOverride="Create a new component"
         title={
           <>
-            Create a new component <Lifecycle alpha shorthand />{' '}
+            Create a new component <Lifecycle alpha shorthand />
           </>
         }
         subtitle="Create new software components using standard templates"
@@ -75,6 +76,11 @@ const ScaffolderPage: React.FC<{}> = () => {
           >
             Register existing component
           </Button>
+          <SupportButton>
+            Create new software components using standard templates. Different
+            templates create different kinds of components (services, websites,
+            documentation, ...).
+          </SupportButton>
         </ContentHeader>
         <Typography variant="body2" paragraph style={{ fontStyle: 'italic' }}>
           <strong>NOTE!</strong> This feature is WIP. You can follow progress{' '}

plugins/sentry/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-sentry",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/tech-radar/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-tech-radar",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
   "private": false,

plugins/welcome/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@backstage/plugin-welcome",
   "version": "0.1.1-alpha.9",
-  "main": "dist/index.esm.js",
-  "main:src": "src/index.ts",
+  "main": "src/index.ts",
   "types": "src/index.ts",
   "private": false,
   "license": "Apache-2.0",

plugins/welcome/src/components/WelcomePage/WelcomePage.tsx

@@ -39,7 +39,8 @@ import {
 } from '@backstage/core';
 
 const WelcomePage: FC<{}> = () => {
-  const appTitle = useApi(configApiRef).getString('app.title') ?? 'Backstage';
+  const appTitle =
+    useApi(configApiRef).getOptionalString('app.title') ?? 'Backstage';
   const profile = { givenName: '' };
 
   return (

yarn.lock

@@ -2452,16 +2452,7 @@
     is-module "^1.0.0"
     resolve "^1.14.2"
 
-"@rollup/pluginutils@^3.0.8":
-  version "3.0.10"
-  resolved "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.0.10.tgz#a659b9025920378494cd8f8c59fbf9b3a50d5f12"
-  integrity sha512-d44M7t+PjmMrASHbhgpSbVgtL6EFyX7J4mYxwQ/c5eoaE6N2VgCgEcWVzNnwycIloti+/MpwFr8qfw+nRw00sw==
-  dependencies:
-    "@types/estree" "0.0.39"
-    estree-walker "^1.0.1"
-    picomatch "^2.2.2"
-
-"@rollup/pluginutils@^3.1.0":
+"@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0":
   version "3.1.0"
   resolved "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b"
   integrity sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==
@@ -3949,9 +3940,9 @@
     "@types/react" "*"
 
 "@types/react@*", "@types/react@^16.9":
-  version "16.9.25"
-  resolved "https://registry.npmjs.org/@types/react/-/react-16.9.25.tgz#6ae2159b40138c792058a23c3c04fd3db49e929e"
-  integrity sha512-Dlj2V72cfYLPNscIG3/SMUOzhzj7GK3bpSrfefwt2YT9GLynvLCCZjbhyF6VsT0q0+aRACRX03TDJGb7cA0cqg==
+  version "16.9.37"
+  resolved "https://registry.npmjs.org/@types/react/-/react-16.9.37.tgz#8fb93e7dbd5b1d3796f69aa979a7fe0439bc7bea"
+  integrity sha512-ZqnAXallQiZ08LTSqMfWMNvAfJEzRLOxdlbbbCIJlYGjU98BEU6bE2uBpKPGeWn+v3hIgCraHKtqUcKZBzMP/Q==
   dependencies:
     "@types/prop-types" "*"
     csstype "^2.2.0"
@@ -8121,10 +8112,10 @@ es6-shim@^0.35.5:
   resolved "https://registry.npmjs.org/es6-shim/-/es6-shim-0.35.5.tgz#46f59dc0a84a1c5029e8ff1166ca0a902077a9ab"
   integrity sha512-E9kK/bjtCQRpN1K28Xh4BlmP8egvZBGJJ+9GtnzOwt7mdqtrjHFuVGr7QJfdjBIKqrlU5duPf3pCBoDrkjVYFg==
 
-esbuild@^0.4.11:
-  version "0.4.14"
-  resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.4.14.tgz#19c3ec44fe3bb6c637dd5287d870a87d36352dcc"
-  integrity sha512-8lx+KpHMQM6t3JFutzCWLckcaVQyv5qvdCzWHQdXGGh16SXdv5nfo/+izWcF367PlMkCMfV7iWW7J5I8Skx7ZQ==
+esbuild@^0.5.3:
+  version "0.5.3"
+  resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.5.3.tgz#18f5bdb618220c6f14bcb1cf5af528d02d4734c9"
+  integrity sha512-RVzTK62svYjnh+agJRh+NWfZX74iKwFNUX52cF7Mo4QPS6bKxP1o+8GacPUMND2QnodVp2D3nKJs8gLspSfZzA==
 
 escape-goat@^2.0.0:
   version "2.1.1"
@@ -12177,9 +12168,9 @@ linkify-it@^2.0.0:
     uc.micro "^1.0.1"
 
 lint-staged@^10.1.0:
-  version "10.2.9"
-  resolved "https://registry.npmjs.org/lint-staged/-/lint-staged-10.2.9.tgz#6013ecfa80829cd422446b545fd30a96bca3098c"
-  integrity sha512-ziRAuXEqvJLSXg43ezBpHxRW8FOJCXISaXU//BWrxRrp5cBdRkIx7g5IsB3OI45xYGE0S6cOacfekSjDyDKF2g==
+  version "10.2.11"
+  resolved "https://registry.npmjs.org/lint-staged/-/lint-staged-10.2.11.tgz#713c80877f2dc8b609b05bc59020234e766c9720"
+  integrity sha512-LRRrSogzbixYaZItE2APaS4l2eJMjjf5MbclRZpLJtcQJShcvUzKXsNeZgsLIZ0H0+fg2tL4B59fU9wHIHtFIA==
   dependencies:
     chalk "^4.0.0"
     cli-truncate "2.1.0"
@@ -13178,9 +13169,9 @@ ms@^2.0.0, ms@^2.1.1:
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
 msw@^0.19.0:
-  version "0.19.3"
-  resolved "https://registry.npmjs.org/msw/-/msw-0.19.3.tgz#88f39edbd37313bff15a0e7cd00c71406cbdbbae"
-  integrity sha512-HYLnyrCDDPP72GG/CeHPhBjHsZFYkz36rJLXDWccZWNA24gYjgrcp9iVqqitk2cI6NAJwsupRml9GkfpJBB74w==
+  version "0.19.4"
+  resolved "https://registry.npmjs.org/msw/-/msw-0.19.4.tgz#059026de0cc3303847c27d3aa0e98705a1033df6"
+  integrity sha512-rNfGgIuO0MyfN2F+FOHRqNd2LJIESaaMOJNNiNMcv/Be7Kdzz/ZmghfS/5zFy2SpHWladccqDYgo74JN+YlEyg==
   dependencies:
     "@open-draft/until" "^1.0.0"
     "@types/cookie" "^0.3.3"
@@ -15485,6 +15476,11 @@ react-fast-compare@^2.0.4:
   resolved "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-2.0.4.tgz#e84b4d455b0fec113e0402c329352715196f81f9"
   integrity sha512-suNP+J1VU1MWFKcyt7RtjiSWUjvidmQSlqu+eHslq+342xCbGTYmC0mEhPCOHxlW0CywylOC1u2DFAT+bv4dBw==
 
+react-fast-compare@^3.1.1:
+  version "3.2.0"
+  resolved "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb"
+  integrity sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA==
+
 react-focus-lock@^2.1.0:
   version "2.2.1"
   resolved "https://registry.npmjs.org/react-focus-lock/-/react-focus-lock-2.2.1.tgz#1d12887416925dc53481914b7cedd39494a3b24a"
@@ -15508,14 +15504,14 @@ react-helmet-async@^1.0.2:
     react-fast-compare "^2.0.4"
     shallowequal "^1.1.0"
 
-react-helmet@6.0.0:
-  version "6.0.0"
-  resolved "https://registry.npmjs.org/react-helmet/-/react-helmet-6.0.0.tgz#fcb93ebaca3ba562a686eb2f1f9d46093d83b5f8"
-  integrity sha512-My6S4sa0uHN/IuVUn0HFmasW5xj9clTkB9qmMngscVycQ5vVG51Qp44BEvLJ4lixupTwDlU9qX1/sCrMN4AEPg==
+react-helmet@6.1.0:
+  version "6.1.0"
+  resolved "https://registry.npmjs.org/react-helmet/-/react-helmet-6.1.0.tgz#a750d5165cb13cf213e44747502652e794468726"
+  integrity sha512-4uMzEY9nlDlgxr61NL3XbKRy1hEkXmKNXhjbAIOVw5vcFrsdYbH2FEwcNyWvWinl103nXgzYNlns9ca+8kFiWw==
   dependencies:
     object-assign "^4.1.1"
     prop-types "^15.7.2"
-    react-fast-compare "^2.0.4"
+    react-fast-compare "^3.1.1"
     react-side-effect "^2.1.0"
 
 react-hook-form@^5.7.2:
@@ -16408,12 +16404,12 @@ rollup-plugin-dts@^1.4.6:
     "@babel/code-frame" "^7.8.3"
 
 rollup-plugin-esbuild@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.npmjs.org/rollup-plugin-esbuild/-/rollup-plugin-esbuild-2.0.0.tgz#ec6a779f5680410801ad47be9fa9447d02cef5f0"
-  integrity sha512-a5jeHL9Ay1xc8RUULcqkHQ6poMMYCbcTYmfFlYavLg3ALXeqhlmueWAZMIMvr8YFit0Ru75/uKKpBRmN395gEA==
+  version "2.1.0"
+  resolved "https://registry.npmjs.org/rollup-plugin-esbuild/-/rollup-plugin-esbuild-2.1.0.tgz#8e12337c63a5b1144e0c5e8adf2f1568ad4d7d69"
+  integrity sha512-XYqmwk4X0SPEExgilARbre/PplhLtE3q6wiZtfgIbwxJOVGXWec1Bkcux7TFTHGX3TQozzqEASTsRJCt7py/5Q==
   dependencies:
     "@rollup/pluginutils" "^3.1.0"
-    esbuild "^0.4.11"
+    esbuild "^0.5.3"
 
 rollup-plugin-image-files@^1.4.2:
   version "1.4.2"