feat(catalog-backend): store entity json in data column, not separate metadata+spec

part of #2749

plugins/catalog-backend/migrations/20201006130744_entity_data_column.js

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('entities', table => {
+    table
+      .text('data')
+      .nullable()
+      .comment('The entire JSON data blob of the entity');
+  });
+
+  await knex('entities').update({
+    // apiVersion and kind should not contain any JSON unsafe chars, and both
+    // metadata and spec are already valid serialized JSON
+    data: knex.raw(
+      `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`,
+    ),
+  });
+
+  await knex.schema.alterTable('entities', table => {
+    table.dropColumn('metadata');
+    table.dropColumn('spec');
+  });
+
+  // SQLite does not support ALTER COLUMN. Note that we do not use the try/
+  // catch method as in other migrations, because if the transaction is
+  // partially failed, it will further mess up the already messed-up
+  // statement below this.
+  if (knex.client.config.client !== 'sqlite3') {
+    await knex.schema.alterTable('entities', table => {
+      table.text('data').notNullable().alter();
+    });
+  }
+
+  // NOTE(freben): For some reason, specifically sqlite3 in-mem just drops some
+  // subset of constraints sometimes, when a table column is dropped - even if
+  // the column had no relation at all to the constraint. We therefore recreate
+  // the constraint here as a stupid fix.
+  if (knex.client.config.client === 'sqlite3') {
+    await knex.schema.alterTable('entities', table => {
+      table.unique(['full_name'], 'entities_unique_full_name');
+    });
+  }
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('entities', table => {
+    table
+      .text('metadata')
+      .notNullable()
+      .comment('The entire metadata JSON blob of the entity');
+    table
+      .text('spec')
+      .nullable()
+      .comment('The entire spec JSON blob of the entity');
+    table.dropColumn('data');
+  });
+};

plugins/catalog-backend/src/database/CommonDatabase.ts

@@ -21,7 +21,6 @@ import {
 } from '@backstage/backend-common';
 import {
   Entity,
-  EntityMeta,
   EntityName,
   ENTITY_DEFAULT_NAMESPACE,
   ENTITY_META_GENERATED_FIELDS,
@@ -389,6 +388,11 @@ export class CommonDatabase implements Database {
     ).toLowerCase();
     const lowerName = entity.metadata.name.toLowerCase();
 
+    const data = {
+      ...entity,
+      metadata: lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS),
+    };
+
     return {
       id: entity.metadata.uid!,
       location_id: locationId || null,
@@ -399,29 +403,15 @@ export class CommonDatabase implements Database {
       kind: entity.kind,
       name: entity.metadata.name,
       namespace: entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE,
-      metadata: JSON.stringify(
-        lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS),
-      ),
-      spec: entity.spec ? JSON.stringify(entity.spec) : null,
+      data: JSON.stringify(data),
     };
   }
 
   private toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
-    const entity: Entity = {
-      apiVersion: row.api_version,
-      kind: row.kind,
-      metadata: {
-        ...(JSON.parse(row.metadata) as EntityMeta),
-        uid: row.id,
-        etag: row.etag,
-        generation: Number(row.generation), // cast because of sqlite
-      },
-    };
-
-    if (row.spec) {
-      const spec = JSON.parse(row.spec);
-      entity.spec = spec;
-    }
+    const entity = JSON.parse(row.data) as Entity;
+    entity.metadata.uid = row.id;
+    entity.metadata.etag = row.etag;
+    entity.metadata.generation = Number(row.generation); // cast due to sqlite
 
     return {
       locationId: row.location_id || undefined,

plugins/catalog-backend/src/database/types.ts

@@ -26,8 +26,7 @@ export type DbEntitiesRow = {
   etag: string;
   generation: number;
   full_name: string;
-  metadata: string;
-  spec: string | null;
+  data: string;
 };
 
 export type DbEntityRequest = {

plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts

@@ -197,7 +197,7 @@ export class HigherOrderOperations implements HigherOrderOperation {
     }
 
     const delta = process.hrtime(startTimestamp);
-    const durationMs = ((delta[0] * 1e9 + delta[1]) / 1e6).toFixed(1);
+    const durationMs = ((delta[0] * 1e9 + delta[1]) / 1e9).toFixed(1);
     this.logger.info(
       `Wrote ${readerOutput.entities.length} entities from location ${location.type} ${location.target} in ${durationMs} seconds`,
     );

plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts

@@ -21,7 +21,7 @@
  * The RDN is the name of the leftmost attribute that identifies the item; for
  * example, for an item with the fully qualified DN
  * uid=john,ou=people,ou=spotify,dc=spotify,dc=net the generated entity would
- * have this attribute, with the value "john".
+ * have this annotation, with the value "john".
  */
 export const LDAP_RDN_ANNOTATION = 'backstage.io/ldap-rdn';
 
@@ -31,7 +31,7 @@ export const LDAP_RDN_ANNOTATION = 'backstage.io/ldap-rdn';
  *
  * The DN is the fully qualified name that identifies the item; for example,
  * for an item with the DN uid=john,ou=people,ou=spotify,dc=spotify,dc=net the
- * generated entity would have this attribute, with that full string as its
+ * generated entity would have this annotation, with that full string as its
  * value.
  */
 export const LDAP_DN_ANNOTATION = 'backstage.io/ldap-dn';
@@ -42,7 +42,7 @@ export const LDAP_DN_ANNOTATION = 'backstage.io/ldap-dn';
  *
  * The UUID is the globally unique ID that identifies the item; for example,
  * for an item with the UUID 76ef928a-b251-1037-9840-d78227f36a7e, the
- * generated entity would have this attribute, with that full string as its
+ * generated entity would have this annotation, with that full string as its
  * value.
  */
 export const LDAP_UUID_ANNOTATION = 'backstage.io/ldap-uuid';