Merge branch 'master' of github.com:spotify/backstage into mob/prepare-from-catalog

* 'master' of github.com:spotify/backstage:
  address comments
  chore(catalog-model): move shared entity model logic here

packages/catalog-model/package.json

@@ -20,8 +20,10 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@backstage/config": "^0.1.1-alpha.9",
     "@types/yup": "^0.28.2",
     "lodash": "^4.17.15",
+    "uuid": "^8.0.0",
     "yup": "^0.28.5"
   },
   "devDependencies": {

packages/catalog-model/src/entity/Entity.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import { JsonObject } from '@backstage/config';
+
 /**
  * The format envelope that's common to all versions/kinds of entity.
  *
@@ -39,7 +41,7 @@ export type Entity = {
   /**
    * The specification data describing the entity itself.
    */
-  spec?: object;
+  spec?: JsonObject;
 };
 
 /**
@@ -48,7 +50,7 @@ export type Entity = {
  * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
  * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
  */
-export type EntityMeta = Record<string, any> & {
+export type EntityMeta = JsonObject & {
   /**
    * A globally unique ID for the entity.
    *
@@ -112,3 +114,8 @@ export type EntityMeta = Record<string, any> & {
    */
   annotations?: Record<string, string>;
 };
+
+/**
+ * The keys of EntityMeta that are auto-generated.
+ */
+export const entityMetaGeneratedFields = ['uid', 'etag', 'generation'] as const;

packages/catalog-model/src/entity/index.ts

@@ -14,5 +14,12 @@
  * limitations under the License.
  */
 
+export { entityMetaGeneratedFields } from './Entity';
 export type { Entity, EntityMeta } from './Entity';
 export * from './policies';
+export {
+  entityHasChanges,
+  generateEntityEtag,
+  generateEntityUid,
+  generateUpdatedEntity,
+} from './util';

packages/catalog-model/src/entity/util.test.ts

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import lodash from 'lodash';
+import {
+  generateEntityEtag,
+  generateEntityUid,
+  entityHasChanges,
+  generateUpdatedEntity,
+} from './util';
+import { Entity } from './Entity';
+
+describe('util', () => {
+  describe('generateEntityUid', () => {
+    it('generates randomness', () => {
+      expect(generateEntityUid()).not.toEqual('');
+      expect(generateEntityUid()).not.toEqual(generateEntityUid());
+    });
+  });
+
+  describe('generateEntityEtag', () => {
+    it('generates randomness', () => {
+      expect(generateEntityEtag()).not.toEqual('');
+      expect(generateEntityEtag()).not.toEqual(generateEntityEtag());
+    });
+  });
+
+  describe('entityHasChanges', () => {
+    let a: Entity;
+    beforeEach(() => {
+      a = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          name: 'name',
+          custom: 'custom',
+          labels: {
+            labelKey: 'labelValue',
+          },
+          annotations: {
+            annotationKey: 'annotationValue',
+          },
+        },
+        spec: {
+          a: 'a',
+        },
+      };
+    });
+
+    it('happy path: clone has no changes', () => {
+      const b = lodash.cloneDeep(a);
+      expect(entityHasChanges(a, b)).toBe(false);
+    });
+
+    it('detects root field changes', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.apiVersion += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.apiVersion;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.kind += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.kind;
+      expect(entityHasChanges(a, b)).toBe(true);
+    });
+
+    it('detects metadata changes', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.metadata.name += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.metadata.custom;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.metadata.custom;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.metadata.labels.n = 'n';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.metadata.labels.labelKey += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+    });
+
+    it('detects annotation changes, but not removals', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.metadata.annotations.annotationKey += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.metadata.annotations.n = 'n';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.metadata.annotations.annotationKey;
+      expect(entityHasChanges(a, b)).toBe(false);
+    });
+
+    it('detects spec changes', () => {
+      let b: any = lodash.cloneDeep(a);
+      b.spec.a += 'a';
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      delete b.spec.a;
+      expect(entityHasChanges(a, b)).toBe(true);
+      b = lodash.cloneDeep(a);
+      b.spec.n = 'n';
+      expect(entityHasChanges(a, b)).toBe(true);
+    });
+  });
+
+  describe('generateUpdatedEntity', () => {
+    let a: Entity;
+    let b: any;
+    beforeEach(() => {
+      a = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Component',
+        metadata: {
+          uid: 'da921f56-f655-4e6e-9b8b-bb19a57818d8',
+          etag: 'NzY5NDA5NzQtYmEwNC00MDY0LWFiYmItNTYxYzQxM2JhZDcx',
+          generation: 2,
+          name: 'name',
+          custom: 'custom',
+          labels: {
+            labelKey: 'labelValue',
+          },
+          annotations: {
+            annotationKey: 'annotationValue',
+          },
+        },
+        spec: {
+          a: 'a',
+        },
+      };
+      b = lodash.cloneDeep(a);
+      delete b.metadata.uid;
+      delete b.metadata.etag;
+      delete b.metadata.generation;
+    });
+
+    it('happy path: running on itself leaves it unchanged', () => {
+      const result = generateUpdatedEntity(a, b);
+      expect(result).toEqual(a);
+    });
+
+    it('bumps etag and generation when spec is changed', () => {
+      b.spec.a += 'a';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation! + 1);
+      expect(result.spec).toEqual({ a: 'aa' });
+    });
+
+    it('bumps only etag when other things than spec are changed', () => {
+      b.metadata.n = 'n';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation);
+      expect(result.metadata.n).toEqual('n');
+    });
+
+    it('retains new annotations', () => {
+      b.metadata.annotations.annotationKey = 'changedValue';
+      b.metadata.annotations.newKey = 'newValue';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation);
+      expect(result.metadata.annotations).toEqual({
+        annotationKey: 'changedValue',
+        newKey: 'newValue',
+      });
+    });
+
+    it('retains old annotations', () => {
+      b.metadata.annotations.newKey = 'newValue';
+      const result = generateUpdatedEntity(a, b);
+      expect(result.metadata.uid).toEqual(a.metadata.uid);
+      expect(result.metadata.etag).not.toEqual(a.metadata.etag);
+      expect(result.metadata.generation).toEqual(a.metadata.generation);
+      expect(result.metadata.annotations).toEqual({
+        annotationKey: 'annotationValue',
+        newKey: 'newValue',
+      });
+    });
+  });
+});

packages/catalog-model/src/entity/util.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import lodash from 'lodash';
+import { v4 as uuidv4 } from 'uuid';
+import { Entity } from './Entity';
+
+/**
+ * Generates a new random UID for an entity.
+ *
+ * @returns A string with enough randomness to uniquely identify an entity
+ */
+export function generateEntityUid(): string {
+  return uuidv4();
+}
+
+/**
+ * Generates a new random Etag for an entity.
+ *
+ * @returns A string with enough randomness to uniquely identify an entity
+ *          revision
+ */
+export function generateEntityEtag(): string {
+  return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, '');
+}
+
+/**
+ * Checks whether there are any significant changes going from the previous to
+ * the next version of this entity.
+ *
+ * Significance, in this case, means that we do not compare generated fields
+ * such as uid, etag and generation, and we only check that no new annotations
+ * are added or existing annotations were changed (since they are effectively
+ * merged when doing updates).
+ *
+ * @param previous The old state of the entity
+ * @param next The new state of the entity
+ */
+export function entityHasChanges(previous: Entity, next: Entity): boolean {
+  if (entityHasAnnotationChanges(previous, next)) {
+    return true;
+  }
+
+  const e1 = lodash.cloneDeep(previous);
+  const e2 = lodash.cloneDeep(next);
+
+  if (!e1.metadata.labels) {
+    e1.metadata.labels = {};
+  }
+  if (!e2.metadata.labels) {
+    e2.metadata.labels = {};
+  }
+
+  // Remove generated fields
+  delete e1.metadata.uid;
+  delete e1.metadata.etag;
+  delete e1.metadata.generation;
+  delete e2.metadata.uid;
+  delete e2.metadata.etag;
+  delete e2.metadata.generation;
+
+  // Remove already compared things
+  delete e1.metadata.annotations;
+  delete e2.metadata.annotations;
+
+  return !lodash.isEqual(e1, e2);
+}
+
+/**
+ * Takes an old revision of an entity and a new desired state, and merges
+ * them into a complete new state.
+ *
+ * The previous revision is expected to be a complete model loaded from the
+ * catalog, including the uid, etag and generation fields.
+ *
+ * @param previous The old state of the entity
+ * @param next The new state of the entity
+ * @returns An entity with the merged state of both
+ */
+export function generateUpdatedEntity(previous: Entity, next: Entity): Entity {
+  const { uid, etag, generation } = previous.metadata;
+  if (!uid || !etag || !generation) {
+    throw new Error('Previous entity must have uid, etag and generation');
+  }
+
+  const result = lodash.cloneDeep(next);
+
+  // Annotations are merged, with the new ones taking precedence
+  if (previous.metadata.annotations) {
+    next.metadata.annotations = {
+      ...previous.metadata.annotations,
+      ...next.metadata.annotations,
+    };
+  }
+
+  // Generated fields are copied and updated
+  const bumpEtag = entityHasChanges(previous, result);
+  const bumpGeneration = !lodash.isEqual(previous.spec, result.spec);
+  result.metadata.uid = uid;
+  result.metadata.etag = bumpEtag ? generateEntityEtag() : etag;
+  result.metadata.generation = bumpGeneration ? generation + 1 : generation;
+
+  return result;
+}
+
+function entityHasAnnotationChanges(previous: Entity, next: Entity): boolean {
+  // Since the next annotations get merged into the previous, extract only
+  // the overlapping keys and check if their values match.
+  if (next.metadata.annotations) {
+    if (!previous.metadata.annotations) {
+      return true;
+    }
+    if (
+      !lodash.isEqual(
+        next.metadata.annotations,
+        lodash.pick(
+          previous.metadata.annotations,
+          Object.keys(next.metadata.annotations),
+        ),
+      )
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}

packages/config-loader/src/lib/reader.ts

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-import yaml from 'yaml';
+import { AppConfig, JsonObject, JsonValue } from '@backstage/config';
 import { basename } from 'path';
-import { isObject } from './utils';
-import { JsonValue, JsonObject, AppConfig } from '@backstage/config';
+import yaml from 'yaml';
 import { ReaderContext } from './types';
+import { isObject } from './utils';
 
 /**
  * Reads and parses, and validates, and transforms a single config file.
@@ -67,9 +67,12 @@ export async function readConfigFile(
     const out: JsonObject = {};
 
     for (const [key, value] of Object.entries(obj)) {
-      const result = await transform(value, `${path}.${key}`);
-      if (result !== undefined) {
-        out[key] = result;
+      // undefined covers optional fields
+      if (value !== undefined) {
+        const result = await transform(value, `${path}.${key}`);
+        if (result !== undefined) {
+          out[key] = result;
+        }
       }
     }
 

packages/config-loader/src/lib/secrets.ts

@@ -122,7 +122,7 @@ export async function readSecret(
     const { path } = secret;
     const parts = typeof path === 'string' ? path.split('.') : path;
 
-    let value: JsonValue = await parser(content);
+    let value: JsonValue | undefined = await parser(content);
     for (const [index, part] of parts.entries()) {
       if (!isObject(value)) {
         const errPath = parts.slice(0, index).join('.');
@@ -132,6 +132,7 @@ export async function readSecret(
       }
       value = value[part];
     }
+
     return String(value);
   }
 

packages/config/src/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-export type JsonObject = { [key in string]: JsonValue };
+export type JsonObject = { [key in string]?: JsonValue };
 export type JsonArray = JsonValue[];
 export type JsonValue =
   | JsonObject