Merge pull request #938 from spotify/freben/more-fields

Add uid, etag, and generation, plus docs, to envelope

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -15,7 +15,7 @@
  */
 
 import { Database } from '../database';
-import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
+import { DescriptorEnvelope } from '../ingestion/types';
 import { EntitiesCatalog } from './types';
 
 export class DatabaseEntitiesCatalog implements EntitiesCatalog {

plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts

@@ -15,8 +15,8 @@
  */
 
 import { NotFoundError } from '@backstage/backend-common';
+import { DescriptorEnvelope } from '../ingestion';
 import { EntitiesCatalog } from './types';
-import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
 
 export class StaticEntitiesCatalog implements EntitiesCatalog {
   private _entities: DescriptorEnvelope[];

plugins/catalog-backend/src/catalog/types.ts

@@ -15,7 +15,7 @@
  */
 
 import * as yup from 'yup';
-import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
+import { DescriptorEnvelope } from '../ingestion';
 
 //
 // Items

plugins/catalog-backend/src/database/Database.ts

@@ -17,7 +17,7 @@
 import { InputError, NotFoundError } from '@backstage/backend-common';
 import Knex from 'knex';
 import { v4 as uuidv4 } from 'uuid';
-import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
+import { DescriptorEnvelope } from '../ingestion';
 import {
   AddDatabaseLocation,
   DatabaseLocationUpdateLogEvent,

plugins/catalog-backend/src/database/types.ts

@@ -15,7 +15,7 @@
  */
 
 import * as yup from 'yup';
-import { DescriptorEnvelope } from '../ingestion/descriptors/DescriptorEnvelopeParser';
+import { DescriptorEnvelope } from '../ingestion';
 
 export type DbEntitiesRow = {
   id: string;

plugins/catalog-backend/src/ingestion/DescriptorParsers.ts

@@ -16,12 +16,13 @@
 
 import { makeValidator } from '../validation';
 import { ComponentDescriptorV1beta1Parser } from './descriptors/ComponentDescriptorV1beta1Parser';
+import { DescriptorEnvelopeParser } from './descriptors/DescriptorEnvelopeParser';
 import {
   DescriptorEnvelope,
-  DescriptorEnvelopeParser,
-} from './descriptors/DescriptorEnvelopeParser';
-import { KindParser } from './descriptors/types';
-import { DescriptorParser, ParserError } from './types';
+  DescriptorParser,
+  KindParser,
+  ParserError,
+} from './types';
 
 export class DescriptorParsers implements DescriptorParser {
   static create(): DescriptorParser {

plugins/catalog-backend/src/ingestion/descriptors/ComponentDescriptorV1beta1Parser.ts

@@ -15,14 +15,10 @@
  */
 
 import * as yup from 'yup';
-import { ParserError } from '../types';
-import { DescriptorEnvelope } from './DescriptorEnvelopeParser';
+import { DescriptorEnvelope, ParserError } from '../types';
 import { KindParser } from './types';
 
 export interface ComponentDescriptorV1beta1 extends DescriptorEnvelope {
-  metadata: {
-    name: string;
-  };
   spec: {
     type: string;
   };

plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.test.ts

@@ -27,6 +27,9 @@ describe('DescriptorEnvelopeParser', () => {
       apiVersion: backstage.io/v1beta1
       kind: Component
       metadata:
+        uid: e01199ab-08cc-44c2-8e19-5c29ded82521
+        etag: lsndfkjsndfkjnsdfkjnsd==
+        generation: 13
         name: my-component-yay
         namespace: the-namespace
         labels:
@@ -77,6 +80,36 @@ describe('DescriptorEnvelopeParser', () => {
     await expect(parser.parse(data)).rejects.toThrow(/metadata/);
   });
 
+  it('accepts missing uid', async () => {
+    delete data.metadata.uid;
+    await expect(parser.parse(data)).resolves.toBe(data);
+  });
+
+  it('rejects bad uid', async () => {
+    data.metadata.uid = 7;
+    await expect(parser.parse(data)).rejects.toThrow(/uid/);
+  });
+
+  it('accepts missing etag', async () => {
+    delete data.metadata.etag;
+    await expect(parser.parse(data)).resolves.toBe(data);
+  });
+
+  it('rejects bad etag', async () => {
+    data.metadata.etag = 7;
+    await expect(parser.parse(data)).rejects.toThrow(/etag/);
+  });
+
+  it('accepts missing generation', async () => {
+    delete data.metadata.generation;
+    await expect(parser.parse(data)).resolves.toBe(data);
+  });
+
+  it('rejects bad generation', async () => {
+    data.metadata.generation = 'a';
+    await expect(parser.parse(data)).rejects.toThrow(/generation/);
+  });
+
   it('accepts missing spec', async () => {
     delete data.spec;
     await expect(parser.parse(data)).resolves.toBe(data);

plugins/catalog-backend/src/ingestion/descriptors/DescriptorEnvelopeParser.ts

@@ -16,21 +16,7 @@
 
 import * as yup from 'yup';
 import { Validators } from '../../validation';
-
-/**
- * The format envelope that's common to all versions/kinds
- */
-export type DescriptorEnvelope = {
-  apiVersion: string;
-  kind: string;
-  metadata?: {
-    name?: string;
-    namespace?: string;
-    labels?: object;
-    annotations?: object;
-  };
-  spec?: object;
-};
+import { DescriptorEnvelope } from '../types';
 
 /**
  * Parses some raw structured data as a descriptor envelope
@@ -57,6 +43,33 @@ export class DescriptorEnvelopeParser {
         validators.isValidKind,
       );
 
+    const uidSchema = yup
+      .string()
+      .notRequired()
+      .test(
+        'metadata.uid',
+        'The uid is not formatted according to schema',
+        value => value === undefined || value.length > 0,
+      );
+
+    const etagSchema = yup
+      .string()
+      .notRequired()
+      .test(
+        'metadata.etag',
+        'The etag value is not according to schema',
+        value => value === undefined || value.length > 0,
+      );
+
+    const generationSchema = yup
+      .number()
+      .notRequired()
+      .test(
+        'metadata.generation',
+        'The generation value is not according to schema',
+        value => value === undefined || value > 0,
+      );
+
     const nameSchema = yup
       .string()
       .notRequired()
@@ -125,6 +138,9 @@ export class DescriptorEnvelopeParser {
 
     const metadataSchema = yup
       .object({
+        uid: uidSchema,
+        etag: etagSchema,
+        generation: generationSchema,
         name: nameSchema,
         namespace: namespaceSchema,
         labels: labelsSchema,
@@ -158,6 +174,9 @@ export class DescriptorEnvelopeParser {
     const reservedKeys = [
       'apiVersion',
       'kind',
+      'uid',
+      'etag',
+      'generation',
       'name',
       'namespace',
       'labels',

plugins/catalog-backend/src/ingestion/descriptors/types.ts

@@ -14,18 +14,21 @@
  * limitations under the License.
  */
 
-import { DescriptorEnvelope } from './DescriptorEnvelopeParser';
+import { DescriptorEnvelope } from '../types';
 
+/**
+ * Parses and validates a single envelope into its materialized kind.
+ *
+ * These parsers may assume that the envelope is already validated and well
+ * formed.
+ */
 export type KindParser = {
   /**
-   * Parses and validates a single envelope into its materialized kind.
-   *
-   * These parsers may assume that the envelope is already validated and
-   * well formed.
+   * Try to parse an envelope into a materialized kind.
    *
    * @param envelope A valid descriptor envelope
    * @returns A materialized type, or undefined if the given version/kind is
-   *          not handled by this parser
+   *          not meant to be handled by this parser
    * @throws An Error if the type was handled and found to not be properly
    *         formatted
    */

plugins/catalog-backend/src/ingestion/types.ts

@@ -15,10 +15,101 @@
  */
 
 import { ComponentDescriptorV1beta1 } from './descriptors/ComponentDescriptorV1beta1Parser';
-import { DescriptorEnvelope } from './descriptors/DescriptorEnvelopeParser';
 
 export type ComponentDescriptor = ComponentDescriptorV1beta1;
 
+/**
+ * The format envelope that's common to all versions/kinds.
+ *
+ * @see https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/
+ */
+export type DescriptorEnvelope = {
+  /**
+   * The version of specification format for this particular entity that
+   * this is written against.
+   */
+  apiVersion: string;
+
+  /**
+   * The high level entity type being described.
+   */
+  kind: string;
+
+  /**
+   * Optional metadata related to the entity.
+   *
+   * @see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
+   */
+  metadata?: {
+    /**
+     * A globally unique ID for the entity.
+     *
+     * This field can not be set by the user at creation time, and the server
+     * will reject an attempt to do so. The field will be populated in read
+     * operations. The field can (optionally) be specified when performing
+     * update or delete operations, but the server is free to reject requests
+     * that do so in such a way that it breaks semantics.
+     */
+    uid?: string;
+
+    /**
+     * An opaque string that changes for each update operation to any part of
+     * the entity, including metadata.
+     *
+     * This field can not be set by the user at creation time, and the server
+     * will reject an attempt to do so. The field will be populated in read
+     * operations. The field can (optionally) be specified when performing
+     * update or delete operations, and the server will then reject the
+     * operation if it does not match the current stored value.
+     */
+    etag?: string;
+
+    /**
+     * A positive nonzero number that indicates the current generation of data
+     * for this entity; the value is incremented each time the spec changes.
+     *
+     * This field can not be set by the user at creation time, and the server
+     * will reject an attempt to do so. The field will be populated in read
+     * operations.
+     */
+    generation?: number;
+
+    /**
+     * The name of the entity.
+     *
+     * Must be uniqe within the catalog at any given point in time, for any
+     * given namespace, for any given kind.
+     */
+    name?: string;
+
+    /**
+     * The namespace that the entity belongs to.
+     */
+    namespace?: string;
+
+    /**
+     * Key/value pairs of identifying information attached to the entity.
+     */
+    labels?: object;
+
+    /**
+     * Key/value pairs of non-identifying auxiliary information attached to the
+     * entity.
+     */
+    annotations?: object;
+  };
+
+  /**
+   * The specification data describing the entity itself.
+   */
+  spec?: object;
+};
+
+/**
+ * Parses and validates descriptors.
+ *
+ * The output must be validated and well formed.
+ */
 export type DescriptorParser = {
   /**
    * Parses and validates a single raw descriptor.
@@ -30,6 +121,27 @@ export type DescriptorParser = {
   parse(descriptor: object): Promise<DescriptorEnvelope>;
 };
 
+/**
+ * Parses and validates a single envelope into its materialized kind.
+ *
+ * These parsers may assume that the envelope is already validated and well
+ * formed.
+ */
+export type KindParser = {
+  /**
+   * Try to parse an envelope into a materialized kind.
+   *
+   * @param envelope A valid descriptor envelope
+   * @returns A materialized type, or undefined if the given version/kind is
+   *          not meant to be handled by this parser
+   * @throws An Error if the type was handled and found to not be properly
+   *         formatted
+   */
+  tryParse(
+    envelope: DescriptorEnvelope,
+  ): Promise<DescriptorEnvelope | undefined>;
+};
+
 export class ParserError extends Error {
   constructor(message?: string, private _entityName?: string | undefined) {
     super(message);
@@ -55,3 +167,14 @@ export type LocationReader = {
    */
   read(type: string, target: string): Promise<ReaderOutput[]>;
 };
+
+export type LocationSource = {
+  /**
+   * Reads the contents of a single location.
+   *
+   * @param target The location target to read
+   * @returns The parsed contents, as an array of unverified descriptors
+   * @throws An error if the location target could not be read
+   */
+  read(target: string): Promise<ReaderOutput[]>;
+};

yarn.lock

@@ -5370,7 +5370,7 @@ aws4@^1.8.0:
   resolved "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz#7e33d8f7d449b3f673cd72deb9abdc552dbe528e"
   integrity sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==
 
-axios@^0.19.0:
+axios@^0.19.0, axios@^0.19.2:
   version "0.19.2"
   resolved "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27"
   integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==