diff --git a/plugins/catalog-backend/src/database/Database.ts b/plugins/catalog-backend/src/database/Database.ts index e90bc408f9..c797a57f3c 100644 --- a/plugins/catalog-backend/src/database/Database.ts +++ b/plugins/catalog-backend/src/database/Database.ts @@ -204,4 +204,16 @@ export class Database { message, }); } + + /* + private async updateEntitiesSearch( + tx: Knex.Transaction, + entityId: string, + data: DescriptorEnvelope, + ): Promise { + const entries = buildEntitySearch(entityId, data); + await tx('entities_search').where({ entity_id: entityId }).del(); + await tx('entities_search').insert(entries); + } + */ } diff --git a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts index 945f636ea4..f029871a70 100644 --- a/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts +++ b/plugins/catalog-backend/src/database/migrations/20200511113813_init.ts @@ -83,9 +83,41 @@ export async function up(knex: Knex): Promise { .nullable() .comment('The entire spec JSON blob of the entity'); }) + .alterTable('entities', table => { + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta + table.unique(['kind', 'name', 'namespace'], 'entities_unique_name'); + }) + // + // entities_search + // + .createTable('entities_search', table => { + table.comment( + 'Flattened key-values from the entities, used for quick filtering', + ); + table + .uuid('entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .comment('The entity that matches this key/value'); + table + .string('key') + .notNullable() + .comment('A key that occurs in the entity'); + table + .string('value') + .nullable() + .comment('The corresponding value to match on'); + }) ); } export async function down(knex: Knex): Promise { - return knex.schema.dropTable('entities').dropTable('locations'); + return knex.schema + .dropTable('entities_search') + .alterTable('entities', table => { + table.dropUnique([], 'entities_unique_name'); + }) + .dropTable('entities') + .dropTable('locations'); } diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts new file mode 100644 index 0000000000..7fe4928f51 --- /dev/null +++ b/plugins/catalog-backend/src/database/search.test.ts @@ -0,0 +1,153 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { buildEntitySearch, visitEntityPart } from './search'; +import { DbEntitiesSearchRow } from './types'; +import { DescriptorEnvelope } from '../ingestion'; + +describe('search', () => { + describe('visitEntityPart', () => { + it('expands lists of strings to several rows', () => { + const input = { a: ['b', 'c', 'd'] }; + const output: DbEntitiesSearchRow[] = []; + visitEntityPart('eid', '', input, output); + expect(output).toEqual([ + { entity_id: 'eid', key: 'a', value: 'b' }, + { entity_id: 'eid', key: 'a', value: 'c' }, + { entity_id: 'eid', key: 'a', value: 'd' }, + ]); + }); + + it('expands objects', () => { + const input = { a: { b: { c: 'd' }, e: 'f' } }; + const output: DbEntitiesSearchRow[] = []; + visitEntityPart('eid', '', input, output); + expect(output).toEqual([ + { entity_id: 'eid', key: 'a.b.c', value: 'd' }, + { entity_id: 'eid', key: 'a.e', value: 'f' }, + ]); + }); + + it('converts base types to strings or null', () => { + const input = { + a: true, + b: false, + c: 7, + d: 'string', + e: null, + f: undefined, + }; + const output: DbEntitiesSearchRow[] = []; + visitEntityPart('eid', '', input, output); + expect(output).toEqual([ + { entity_id: 'eid', key: 'a', value: 'true' }, + { entity_id: 'eid', key: 'b', value: 'false' }, + { entity_id: 'eid', key: 'c', value: '7' }, + { entity_id: 'eid', key: 'd', value: 'string' }, + { entity_id: 'eid', key: 'e', value: null }, + { entity_id: 'eid', key: 'f', value: null }, + ]); + }); + + it('skips over special keys', () => { + const input = { + a: 'a', + metadata: { + b: 'b', + name: 'name', + namespace: 'namespace', + uid: 'uid', + etag: 'etag', + generation: 'generation', + c: 'c', + }, + d: 'd', + }; + const output: DbEntitiesSearchRow[] = []; + visitEntityPart('eid', '', input, output); + expect(output).toEqual([ + { entity_id: 'eid', key: 'a', value: 'a' }, + { entity_id: 'eid', key: 'metadata.b', value: 'b' }, + { entity_id: 'eid', key: 'metadata.c', value: 'c' }, + { entity_id: 'eid', key: 'd', value: 'd' }, + ]); + }); + + it('expands list of objects', () => { + const input = { root: { list: [{ a: 1 }, { a: 2 }] } }; + const output: DbEntitiesSearchRow[] = []; + visitEntityPart('eid', '', input, output); + expect(output).toEqual([ + { entity_id: 'eid', key: 'root.list.a', value: '1' }, + { entity_id: 'eid', key: 'root.list.a', value: '2' }, + ]); + }); + }); + + describe('buildEntitySearch', () => { + it('adds special keys even if missing', () => { + const input: DescriptorEnvelope = { + apiVersion: 'a', + kind: 'b', + }; + expect(buildEntitySearch('eid', input)).toEqual([ + { entity_id: 'eid', key: 'metadata.name', value: null }, + { entity_id: 'eid', key: 'metadata.namespace', value: null }, + { entity_id: 'eid', key: 'metadata.uid', value: null }, + { entity_id: 'eid', key: 'apiVersion', value: 'a' }, + { entity_id: 'eid', key: 'kind', value: 'b' }, + { entity_id: 'eid', key: 'name', value: null }, + { entity_id: 'eid', key: 'namespace', value: null }, + { entity_id: 'eid', key: 'uid', value: null }, + ]); + }); + + it('adds prefix-stripped versions', () => { + const input: DescriptorEnvelope = { + apiVersion: 'a', + kind: 'b', + metadata: { + name: 'name', + labels: { + lbl: 'lbl', + }, + annotations: { + ann: 'ann', + }, + }, + spec: { + sub: { + spc: 'spc', + }, + }, + }; + expect(buildEntitySearch('eid', input)).toStrictEqual( + expect.arrayContaining([ + { entity_id: 'eid', key: 'metadata.name', value: 'name' }, + { entity_id: 'eid', key: 'name', value: 'name' }, + { entity_id: 'eid', key: 'metadata.labels.lbl', value: 'lbl' }, + { entity_id: 'eid', key: 'labels.lbl', value: 'lbl' }, + { entity_id: 'eid', key: 'lbl', value: 'lbl' }, + { entity_id: 'eid', key: 'metadata.annotations.ann', value: 'ann' }, + { entity_id: 'eid', key: 'annotations.ann', value: 'ann' }, + { entity_id: 'eid', key: 'ann', value: 'ann' }, + { entity_id: 'eid', key: 'spec.sub.spc', value: 'spc' }, + { entity_id: 'eid', key: 'sub.spc', value: 'spc' }, + ]), + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts new file mode 100644 index 0000000000..f54cfde7e6 --- /dev/null +++ b/plugins/catalog-backend/src/database/search.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DescriptorEnvelope } from '../ingestion'; +import { DbEntitiesSearchRow } from './types'; + +// Search entries that start with these prefixes, also get a shorthand without +// that prefix +const SHORTHAND_KEY_PREFIXES = [ + 'metadata.', + 'metadata.labels.', + 'metadata.annotations.', + 'spec.', +]; + +// These are exluded in the generic loop, either because they do not make sense +// to index, or because they are special-case always inserted whether they are +// null or not +const SPECIAL_KEYS = [ + 'metadata.name', + 'metadata.namespace', + 'metadata.uid', + 'metadata.etag', + 'metadata.generation', +]; + +function toValue(current: any): string | null { + if (current === undefined || current === null) { + return null; + } + + return String(current); +} + +// Helper for iterating through a nested structure and outputting a list of +// path->value entries. +// +// For example, this yaml structure +// +// a: 1 +// b: +// c: null +// e: [f, g] +// h: +// - i: 1 +// j: k +// - i: 2 +// j: l +// +// will result in +// +// "a", "1" +// "b.c", null +// "b.e": "f" +// "b.e": "g" +// "h.i": "1" +// "h.j": "k" +// "h.i": "2" +// "h.j": "l" +export function visitEntityPart( + entityId: string, + path: string, + current: any, + output: DbEntitiesSearchRow[], +) { + // ignored + if (SPECIAL_KEYS.includes(path)) { + return; + } + + // empty or scalar + if ( + current === undefined || + current === null || + ['string', 'number', 'boolean'].includes(typeof current) + ) { + output.push({ entity_id: entityId, key: path, value: toValue(current) }); + return; + } + + // unknown + if (typeof current !== 'object') { + return; + } + + // array + if (Array.isArray(current)) { + for (const item of current) { + visitEntityPart(entityId, path, item, output); + } + return; + } + + // object + for (const [key, value] of Object.entries(current)) { + visitEntityPart(entityId, path ? `${path}.${key}` : key, value, output); + } +} + +/** + * Generates all of the search rows that are relevant for this entity. + * + * @param entityId The uid of the entity + * @param entity The entity + * @returns A list of entity search rows + */ +export function buildEntitySearch( + entityId: string, + entity: DescriptorEnvelope, +): DbEntitiesSearchRow[] { + // Start with some special keys that are always present because you want to + // be able to easily search for null specifically + const result: DbEntitiesSearchRow[] = [ + { + entity_id: entityId, + key: 'metadata.name', + value: toValue(entity.metadata?.name), + }, + { + entity_id: entityId, + key: 'metadata.namespace', + value: toValue(entity.metadata?.namespace), + }, + { + entity_id: entityId, + key: 'metadata.uid', + value: toValue(entity.metadata?.uid), + }, + ]; + + // Visit the entire structure recursively + visitEntityPart(entityId, '', entity, result); + + // Generate shorthands for fields directly under some common collections + for (const row of result.slice()) { + for (const stripPrefix of SHORTHAND_KEY_PREFIXES) { + if (row.key.startsWith(stripPrefix)) { + result.push({ ...row, key: row.key.substr(stripPrefix.length) }); + } + } + } + + return result; +} diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 478606a79c..06e234545f 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -40,6 +40,12 @@ export type DbEntityResponse = { entity: DescriptorEnvelope; }; +export type DbEntitiesSearchRow = { + entity_id: string; + key: string; + value: string | null; +}; + export type DbLocationsRow = { id: string; type: string;