Merge pull request #971 from spotify/freben/search

Add code for building a search index over added/modified entities

plugins/catalog-backend/src/database/Database.ts

@@ -204,4 +204,16 @@ export class Database {
       message,
     });
   }
+
+  /*
+  private async updateEntitiesSearch(
+    tx: Knex.Transaction<any, any>,
+    entityId: string,
+    data: DescriptorEnvelope,
+  ): Promise<void> {
+    const entries = buildEntitySearch(entityId, data);
+    await tx('entities_search').where({ entity_id: entityId }).del();
+    await tx('entities_search').insert(entries);
+  }
+  */
 }

plugins/catalog-backend/src/database/migrations/20200511113813_init.ts

@@ -83,9 +83,41 @@ export async function up(knex: Knex): Promise<any> {
           .nullable()
           .comment('The entire spec JSON blob of the entity');
       })
+      .alterTable('entities', table => {
+        // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta
+        table.unique(['kind', 'name', 'namespace'], 'entities_unique_name');
+      })
+      //
+      // entities_search
+      //
+      .createTable('entities_search', table => {
+        table.comment(
+          'Flattened key-values from the entities, used for quick filtering',
+        );
+        table
+          .uuid('entity_id')
+          .references('id')
+          .inTable('entities')
+          .onDelete('CASCADE')
+          .comment('The entity that matches this key/value');
+        table
+          .string('key')
+          .notNullable()
+          .comment('A key that occurs in the entity');
+        table
+          .string('value')
+          .nullable()
+          .comment('The corresponding value to match on');
+      })
   );
 }
 
 export async function down(knex: Knex): Promise<any> {
-  return knex.schema.dropTable('entities').dropTable('locations');
+  return knex.schema
+    .dropTable('entities_search')
+    .alterTable('entities', table => {
+      table.dropUnique([], 'entities_unique_name');
+    })
+    .dropTable('entities')
+    .dropTable('locations');
 }

plugins/catalog-backend/src/database/search.test.ts

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { buildEntitySearch, visitEntityPart } from './search';
+import { DbEntitiesSearchRow } from './types';
+import { DescriptorEnvelope } from '../ingestion';
+
+describe('search', () => {
+  describe('visitEntityPart', () => {
+    it('expands lists of strings to several rows', () => {
+      const input = { a: ['b', 'c', 'd'] };
+      const output: DbEntitiesSearchRow[] = [];
+      visitEntityPart('eid', '', input, output);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'a', value: 'b' },
+        { entity_id: 'eid', key: 'a', value: 'c' },
+        { entity_id: 'eid', key: 'a', value: 'd' },
+      ]);
+    });
+
+    it('expands objects', () => {
+      const input = { a: { b: { c: 'd' }, e: 'f' } };
+      const output: DbEntitiesSearchRow[] = [];
+      visitEntityPart('eid', '', input, output);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'a.b.c', value: 'd' },
+        { entity_id: 'eid', key: 'a.e', value: 'f' },
+      ]);
+    });
+
+    it('converts base types to strings or null', () => {
+      const input = {
+        a: true,
+        b: false,
+        c: 7,
+        d: 'string',
+        e: null,
+        f: undefined,
+      };
+      const output: DbEntitiesSearchRow[] = [];
+      visitEntityPart('eid', '', input, output);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'a', value: 'true' },
+        { entity_id: 'eid', key: 'b', value: 'false' },
+        { entity_id: 'eid', key: 'c', value: '7' },
+        { entity_id: 'eid', key: 'd', value: 'string' },
+        { entity_id: 'eid', key: 'e', value: null },
+        { entity_id: 'eid', key: 'f', value: null },
+      ]);
+    });
+
+    it('skips over special keys', () => {
+      const input = {
+        a: 'a',
+        metadata: {
+          b: 'b',
+          name: 'name',
+          namespace: 'namespace',
+          uid: 'uid',
+          etag: 'etag',
+          generation: 'generation',
+          c: 'c',
+        },
+        d: 'd',
+      };
+      const output: DbEntitiesSearchRow[] = [];
+      visitEntityPart('eid', '', input, output);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'a', value: 'a' },
+        { entity_id: 'eid', key: 'metadata.b', value: 'b' },
+        { entity_id: 'eid', key: 'metadata.c', value: 'c' },
+        { entity_id: 'eid', key: 'd', value: 'd' },
+      ]);
+    });
+
+    it('expands list of objects', () => {
+      const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
+      const output: DbEntitiesSearchRow[] = [];
+      visitEntityPart('eid', '', input, output);
+      expect(output).toEqual([
+        { entity_id: 'eid', key: 'root.list.a', value: '1' },
+        { entity_id: 'eid', key: 'root.list.a', value: '2' },
+      ]);
+    });
+  });
+
+  describe('buildEntitySearch', () => {
+    it('adds special keys even if missing', () => {
+      const input: DescriptorEnvelope = {
+        apiVersion: 'a',
+        kind: 'b',
+      };
+      expect(buildEntitySearch('eid', input)).toEqual([
+        { entity_id: 'eid', key: 'metadata.name', value: null },
+        { entity_id: 'eid', key: 'metadata.namespace', value: null },
+        { entity_id: 'eid', key: 'metadata.uid', value: null },
+        { entity_id: 'eid', key: 'apiVersion', value: 'a' },
+        { entity_id: 'eid', key: 'kind', value: 'b' },
+        { entity_id: 'eid', key: 'name', value: null },
+        { entity_id: 'eid', key: 'namespace', value: null },
+        { entity_id: 'eid', key: 'uid', value: null },
+      ]);
+    });
+
+    it('adds prefix-stripped versions', () => {
+      const input: DescriptorEnvelope = {
+        apiVersion: 'a',
+        kind: 'b',
+        metadata: {
+          name: 'name',
+          labels: {
+            lbl: 'lbl',
+          },
+          annotations: {
+            ann: 'ann',
+          },
+        },
+        spec: {
+          sub: {
+            spc: 'spc',
+          },
+        },
+      };
+      expect(buildEntitySearch('eid', input)).toStrictEqual(
+        expect.arrayContaining([
+          { entity_id: 'eid', key: 'metadata.name', value: 'name' },
+          { entity_id: 'eid', key: 'name', value: 'name' },
+          { entity_id: 'eid', key: 'metadata.labels.lbl', value: 'lbl' },
+          { entity_id: 'eid', key: 'labels.lbl', value: 'lbl' },
+          { entity_id: 'eid', key: 'lbl', value: 'lbl' },
+          { entity_id: 'eid', key: 'metadata.annotations.ann', value: 'ann' },
+          { entity_id: 'eid', key: 'annotations.ann', value: 'ann' },
+          { entity_id: 'eid', key: 'ann', value: 'ann' },
+          { entity_id: 'eid', key: 'spec.sub.spc', value: 'spc' },
+          { entity_id: 'eid', key: 'sub.spc', value: 'spc' },
+        ]),
+      );
+    });
+  });
+});

plugins/catalog-backend/src/database/search.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DescriptorEnvelope } from '../ingestion';
+import { DbEntitiesSearchRow } from './types';
+
+// Search entries that start with these prefixes, also get a shorthand without
+// that prefix
+const SHORTHAND_KEY_PREFIXES = [
+  'metadata.',
+  'metadata.labels.',
+  'metadata.annotations.',
+  'spec.',
+];
+
+// These are exluded in the generic loop, either because they do not make sense
+// to index, or because they are special-case always inserted whether they are
+// null or not
+const SPECIAL_KEYS = [
+  'metadata.name',
+  'metadata.namespace',
+  'metadata.uid',
+  'metadata.etag',
+  'metadata.generation',
+];
+
+function toValue(current: any): string | null {
+  if (current === undefined || current === null) {
+    return null;
+  }
+
+  return String(current);
+}
+
+// Helper for iterating through a nested structure and outputting a list of
+// path->value entries.
+//
+// For example, this yaml structure
+//
+// a: 1
+// b:
+//   c: null
+//   e: [f, g]
+// h:
+//  - i: 1
+//    j: k
+//  - i: 2
+//    j: l
+//
+// will result in
+//
+// "a", "1"
+// "b.c", null
+// "b.e": "f"
+// "b.e": "g"
+// "h.i": "1"
+// "h.j": "k"
+// "h.i": "2"
+// "h.j": "l"
+export function visitEntityPart(
+  entityId: string,
+  path: string,
+  current: any,
+  output: DbEntitiesSearchRow[],
+) {
+  // ignored
+  if (SPECIAL_KEYS.includes(path)) {
+    return;
+  }
+
+  // empty or scalar
+  if (
+    current === undefined ||
+    current === null ||
+    ['string', 'number', 'boolean'].includes(typeof current)
+  ) {
+    output.push({ entity_id: entityId, key: path, value: toValue(current) });
+    return;
+  }
+
+  // unknown
+  if (typeof current !== 'object') {
+    return;
+  }
+
+  // array
+  if (Array.isArray(current)) {
+    for (const item of current) {
+      visitEntityPart(entityId, path, item, output);
+    }
+    return;
+  }
+
+  // object
+  for (const [key, value] of Object.entries(current)) {
+    visitEntityPart(entityId, path ? `${path}.${key}` : key, value, output);
+  }
+}
+
+/**
+ * Generates all of the search rows that are relevant for this entity.
+ *
+ * @param entityId The uid of the entity
+ * @param entity The entity
+ * @returns A list of entity search rows
+ */
+export function buildEntitySearch(
+  entityId: string,
+  entity: DescriptorEnvelope,
+): DbEntitiesSearchRow[] {
+  // Start with some special keys that are always present because you want to
+  // be able to easily search for null specifically
+  const result: DbEntitiesSearchRow[] = [
+    {
+      entity_id: entityId,
+      key: 'metadata.name',
+      value: toValue(entity.metadata?.name),
+    },
+    {
+      entity_id: entityId,
+      key: 'metadata.namespace',
+      value: toValue(entity.metadata?.namespace),
+    },
+    {
+      entity_id: entityId,
+      key: 'metadata.uid',
+      value: toValue(entity.metadata?.uid),
+    },
+  ];
+
+  // Visit the entire structure recursively
+  visitEntityPart(entityId, '', entity, result);
+
+  // Generate shorthands for fields directly under some common collections
+  for (const row of result.slice()) {
+    for (const stripPrefix of SHORTHAND_KEY_PREFIXES) {
+      if (row.key.startsWith(stripPrefix)) {
+        result.push({ ...row, key: row.key.substr(stripPrefix.length) });
+      }
+    }
+  }
+
+  return result;
+}

plugins/catalog-backend/src/database/types.ts

@@ -40,6 +40,12 @@ export type DbEntityResponse = {
   entity: DescriptorEnvelope;
 };
 
+export type DbEntitiesSearchRow = {
+  entity_id: string;
+  key: string;
+  value: string | null;
+};
+
 export type DbLocationsRow = {
   id: string;
   type: string;