From 252024d04734c38f80af4cd026c447a2b5a59f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 9 Oct 2020 16:47:27 +0200 Subject: [PATCH] fix(catalog-backend): limit search value lengths --- .../src/database/search.test.ts | 100 ++++++------ .../catalog-backend/src/database/search.ts | 154 +++++++++--------- 2 files changed, 128 insertions(+), 126 deletions(-) diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts index fad17da733..d108820ca9 100644 --- a/plugins/catalog-backend/src/database/search.test.ts +++ b/plugins/catalog-backend/src/database/search.test.ts @@ -14,51 +14,36 @@ * limitations under the License. */ -import { ENTITY_DEFAULT_NAMESPACE, Entity } from '@backstage/catalog-model'; -import { buildEntitySearch, visitEntityPart } from './search'; -import type { DbEntitiesSearchRow } from './types'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { buildEntitySearch, mapToRows, traverse } from './search'; describe('search', () => { - describe('visitEntityPart', () => { + describe('traverse', () => { it('expands lists of strings to several rows', () => { const input = { a: ['b', 'c', 'd'] }; - const output: DbEntitiesSearchRow[] = []; - visitEntityPart('eid', '', input, output); + const output = traverse(input); expect(output).toEqual([ - { entity_id: 'eid', key: 'a', value: 'b' }, - { entity_id: 'eid', key: 'a', value: 'c' }, - { entity_id: 'eid', key: 'a', value: 'd' }, + { key: 'a', value: 'b' }, + { key: 'a', value: 'c' }, + { key: 'a', value: 'd' }, ]); }); it('expands objects', () => { const input = { a: { b: { c: 'd' }, e: 'f' } }; - const output: DbEntitiesSearchRow[] = []; - visitEntityPart('eid', '', input, output); + const output = traverse(input); expect(output).toEqual([ - { entity_id: 'eid', key: 'a.b.c', value: 'd' }, - { entity_id: 'eid', key: 'a.e', value: 'f' }, + { key: 'a.b.c', value: 'd' }, + { key: 'a.e', value: 'f' }, ]); }); - it('converts base types to strings or null', () => { - const input = { - a: true, - b: false, - c: 7, - d: 'string', - e: null, - f: undefined, - }; - const output: DbEntitiesSearchRow[] = []; - visitEntityPart('eid', '', input, output); + it('expands list of objects', () => { + const input = { root: { list: [{ a: 1 }, { a: 2 }] } }; + const output = traverse(input); expect(output).toEqual([ - { entity_id: 'eid', key: 'a', value: 'true' }, - { entity_id: 'eid', key: 'b', value: 'false' }, - { entity_id: 'eid', key: 'c', value: '7' }, - { entity_id: 'eid', key: 'd', value: 'string' }, - { entity_id: 'eid', key: 'e', value: null }, - { entity_id: 'eid', key: 'f', value: null }, + { key: 'root.list.a', value: 1 }, + { key: 'root.list.a', value: 2 }, ]); }); @@ -76,34 +61,47 @@ describe('search', () => { }, d: 'd', }; - const output: DbEntitiesSearchRow[] = []; - visitEntityPart('eid', '', input, output); + const output = traverse(input); expect(output).toEqual([ - { entity_id: 'eid', key: 'a', value: 'a' }, - { entity_id: 'eid', key: 'metadata.b', value: 'b' }, - { entity_id: 'eid', key: 'metadata.c', value: 'c' }, - { entity_id: 'eid', key: 'd', value: 'd' }, + { key: 'a', value: 'a' }, + { key: 'metadata.b', value: 'b' }, + { key: 'metadata.c', value: 'c' }, + { key: 'd', value: 'd' }, ]); }); + }); - it('expands list of objects', () => { - const input = { root: { list: [{ a: 1 }, { a: 2 }] } }; - const output: DbEntitiesSearchRow[] = []; - visitEntityPart('eid', '', input, output); + describe('mapToRows', () => { + it('converts base types to strings or null', () => { + const input = [ + { key: 'a', value: true }, + { key: 'b', value: false }, + { key: 'c', value: 7 }, + { key: 'd', value: 'string' }, + { key: 'e', value: null }, + { key: 'f', value: undefined }, + ]; + const output = mapToRows(input, 'eid'); expect(output).toEqual([ - { entity_id: 'eid', key: 'root.list.a', value: '1' }, - { entity_id: 'eid', key: 'root.list.a', value: '2' }, + { entity_id: 'eid', key: 'a', value: 'true' }, + { entity_id: 'eid', key: 'b', value: 'false' }, + { entity_id: 'eid', key: 'c', value: '7' }, + { entity_id: 'eid', key: 'd', value: 'string' }, + { entity_id: 'eid', key: 'e', value: null }, + { entity_id: 'eid', key: 'f', value: null }, ]); }); it('emits lowercase version of keys and values', () => { - const input = { theRoot: { listItems: [{ a: 'One' }, { a: 2 }] } }; - const output: DbEntitiesSearchRow[] = []; - visitEntityPart('eid', '', input, output); - expect(output).toEqual([ - { entity_id: 'eid', key: 'theroot.listitems.a', value: 'one' }, - { entity_id: 'eid', key: 'theroot.listitems.a', value: '2' }, - ]); + const input = [{ key: 'fOo', value: 'BaR' }]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]); + }); + + it('skips very large values', () => { + const input = [{ key: 'foo', value: 'a'.repeat(10000) }]; + const output = mapToRows(input, 'eid'); + expect(output).toEqual([]); }); }); @@ -115,6 +113,8 @@ describe('search', () => { metadata: { name: 'n' }, }; expect(buildEntitySearch('eid', input)).toEqual([ + { entity_id: 'eid', key: 'apiversion', value: 'a' }, + { entity_id: 'eid', key: 'kind', value: 'b' }, { entity_id: 'eid', key: 'metadata.name', value: 'n' }, { entity_id: 'eid', key: 'metadata.namespace', value: null }, { entity_id: 'eid', key: 'metadata.uid', value: null }, @@ -123,8 +123,6 @@ describe('search', () => { key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE, }, - { entity_id: 'eid', key: 'apiversion', value: 'a' }, - { entity_id: 'eid', key: 'kind', value: 'b' }, ]); }); }); diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts index bbed2fac47..863b86ee9a 100644 --- a/plugins/catalog-backend/src/database/search.ts +++ b/plugins/catalog-backend/src/database/search.ts @@ -28,16 +28,18 @@ const SPECIAL_KEYS = [ 'metadata.generation', ]; -function toValue(current: any): string | null { - if (current === undefined || current === null) { - return null; - } +// The maximum length allowed for search values. These columns are indexed, and +// database engines do not like to index on massive values. For example, +// postgres will balk after 8191 byte line sizes. +const MAX_VALUE_LENGTH = 200; - return String(current).toLowerCase(); -} +type Kv = { + key: string; + value: any; +}; -// Helper for iterating through a nested structure and outputting a list of -// path->value entries. +// Helper for traversing through a nested structure and outputting a list of +// path->value entries of the leaves. // // For example, this yaml structure // @@ -53,57 +55,77 @@ function toValue(current: any): string | null { // // will result in // -// "a", "1" +// "a", 1 // "b.c", null // "b.e": "f" // "b.e": "g" -// "h.i": "1" +// "h.i": 1 // "h.j": "k" -// "h.i": "2" +// "h.i": 2 // "h.j": "l" -export function visitEntityPart( - entityId: string, - path: string, - current: any, - output: DbEntitiesSearchRow[], -) { - // ignored - if (SPECIAL_KEYS.includes(path)) { - return; - } +export function traverse(root: any): Kv[] { + const output: Kv[] = []; - // empty or scalar - if ( - current === undefined || - current === null || - ['string', 'number', 'boolean'].includes(typeof current) - ) { - output.push({ entity_id: entityId, key: path, value: toValue(current) }); - return; - } - - // unknown - if (typeof current !== 'object') { - return; - } - - // array - if (Array.isArray(current)) { - for (const item of current) { - visitEntityPart(entityId, path, item, output); + function visit(path: string, current: any) { + if (SPECIAL_KEYS.includes(path)) { + return; + } + + // empty or scalar + if ( + current === undefined || + current === null || + ['string', 'number', 'boolean'].includes(typeof current) + ) { + output.push({ key: path, value: current }); + return; + } + + // unknown + if (typeof current !== 'object') { + return; + } + + // array + if (Array.isArray(current)) { + for (const item of current) { + // keep the same path as currently + visit(path, item); + } + return; + } + + // object + for (const [key, value] of Object.entries(current)) { + visit(path ? `${path}.${key}` : key, value); } - return; } - // object - for (const [key, value] of Object.entries(current)) { - visitEntityPart( - entityId, - (path ? `${path}.${key}` : key).toLowerCase(), - value, - output, - ); + visit('', root); + + return output; +} + +// Translates a number of raw data rows to search table rows +export function mapToRows( + input: Kv[], + entityId: string, +): DbEntitiesSearchRow[] { + const result: DbEntitiesSearchRow[] = []; + + for (let { key, value } of input) { + key = key.toLowerCase(); + if (value === undefined || value === null) { + result.push({ entity_id: entityId, key, value: null }); + } else { + value = String(value).toLowerCase(); + if (value.length <= MAX_VALUE_LENGTH) { + result.push({ entity_id: entityId, key, value }); + } + } } + + return result; } /** @@ -117,38 +139,20 @@ export function buildEntitySearch( entityId: string, entity: Entity, ): DbEntitiesSearchRow[] { + // Visit the entire structure recursively + const raw = traverse(entity); + // Start with some special keys that are always present because you want to // be able to easily search for null specifically - const result: DbEntitiesSearchRow[] = [ - { - entity_id: entityId, - key: 'metadata.name', - value: toValue(entity.metadata.name), - }, - { - entity_id: entityId, - key: 'metadata.namespace', - value: toValue(entity.metadata.namespace), - }, - { - entity_id: entityId, - key: 'metadata.uid', - value: toValue(entity.metadata.uid), - }, - ]; + raw.push({ key: 'metadata.name', value: entity.metadata.name }); + raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace }); + raw.push({ key: 'metadata.uid', value: entity.metadata.uid }); // Namespace not specified has the default value "default", so we want to // match on that as well if (!entity.metadata.namespace) { - result.push({ - entity_id: entityId, - key: 'metadata.namespace', - value: toValue(ENTITY_DEFAULT_NAMESPACE), - }); + raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE }); } - // Visit the entire structure recursively - visitEntityPart(entityId, '', entity, result); - - return result; + return mapToRows(raw, entityId); }