paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 36 Packages Projects Releases Wiki Activity

fix(catalog-backend): limit search value lengths

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-10-09 16:47:27 +02:00
parent 2cbb8f4355
commit 252024d047
2 changed files with 128 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/database/search.test.ts
+49 -51
View File
@@ -14,51 +14,36 @@
* limitations under the License.
*/
import { ENTITY_DEFAULT_NAMESPACE, Entity } from '@backstage/catalog-model';
import { buildEntitySearch, visitEntityPart } from './search';
import type { DbEntitiesSearchRow } from './types';
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { buildEntitySearch, mapToRows, traverse } from './search';
describe('search', () => {
describe('visitEntityPart', () => {
describe('traverse', () => {
it('expands lists of strings to several rows', () => {
const input = { a: ['b', 'c', 'd'] };
const output: DbEntitiesSearchRow[] = [];
visitEntityPart('eid', '', input, output);
const output = traverse(input);
expect(output).toEqual([
{ entity_id: 'eid', key: 'a', value: 'b' },
{ entity_id: 'eid', key: 'a', value: 'c' },
{ entity_id: 'eid', key: 'a', value: 'd' },
{ key: 'a', value: 'b' },
{ key: 'a', value: 'c' },
{ key: 'a', value: 'd' },
]);
});
it('expands objects', () => {
const input = { a: { b: { c: 'd' }, e: 'f' } };
const output: DbEntitiesSearchRow[] = [];
visitEntityPart('eid', '', input, output);
const output = traverse(input);
expect(output).toEqual([
{ entity_id: 'eid', key: 'a.b.c', value: 'd' },
{ entity_id: 'eid', key: 'a.e', value: 'f' },
{ key: 'a.b.c', value: 'd' },
{ key: 'a.e', value: 'f' },
]);
});
it('converts base types to strings or null', () => {
const input = {
a: true,
b: false,
c: 7,
d: 'string',
e: null,
f: undefined,
};
const output: DbEntitiesSearchRow[] = [];
visitEntityPart('eid', '', input, output);
it('expands list of objects', () => {
const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
const output = traverse(input);
expect(output).toEqual([
{ entity_id: 'eid', key: 'a', value: 'true' },
{ entity_id: 'eid', key: 'b', value: 'false' },
{ entity_id: 'eid', key: 'c', value: '7' },
{ entity_id: 'eid', key: 'd', value: 'string' },
{ entity_id: 'eid', key: 'e', value: null },
{ entity_id: 'eid', key: 'f', value: null },
{ key: 'root.list.a', value: 1 },
{ key: 'root.list.a', value: 2 },
]);
});
@@ -76,34 +61,47 @@ describe('search', () => {
},
d: 'd',
};
const output: DbEntitiesSearchRow[] = [];
visitEntityPart('eid', '', input, output);
const output = traverse(input);
expect(output).toEqual([
{ entity_id: 'eid', key: 'a', value: 'a' },
{ entity_id: 'eid', key: 'metadata.b', value: 'b' },
{ entity_id: 'eid', key: 'metadata.c', value: 'c' },
{ entity_id: 'eid', key: 'd', value: 'd' },
{ key: 'a', value: 'a' },
{ key: 'metadata.b', value: 'b' },
{ key: 'metadata.c', value: 'c' },
{ key: 'd', value: 'd' },
]);
});
});
it('expands list of objects', () => {
const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
const output: DbEntitiesSearchRow[] = [];
visitEntityPart('eid', '', input, output);
describe('mapToRows', () => {
it('converts base types to strings or null', () => {
const input = [
{ key: 'a', value: true },
{ key: 'b', value: false },
{ key: 'c', value: 7 },
{ key: 'd', value: 'string' },
{ key: 'e', value: null },
{ key: 'f', value: undefined },
];
const output = mapToRows(input, 'eid');
expect(output).toEqual([
{ entity_id: 'eid', key: 'root.list.a', value: '1' },
{ entity_id: 'eid', key: 'root.list.a', value: '2' },
{ entity_id: 'eid', key: 'a', value: 'true' },
{ entity_id: 'eid', key: 'b', value: 'false' },
{ entity_id: 'eid', key: 'c', value: '7' },
{ entity_id: 'eid', key: 'd', value: 'string' },
{ entity_id: 'eid', key: 'e', value: null },
{ entity_id: 'eid', key: 'f', value: null },
]);
});
it('emits lowercase version of keys and values', () => {
const input = { theRoot: { listItems: [{ a: 'One' }, { a: 2 }] } };
const output: DbEntitiesSearchRow[] = [];
visitEntityPart('eid', '', input, output);
expect(output).toEqual([
{ entity_id: 'eid', key: 'theroot.listitems.a', value: 'one' },
{ entity_id: 'eid', key: 'theroot.listitems.a', value: '2' },
]);
const input = [{ key: 'fOo', value: 'BaR' }];
const output = mapToRows(input, 'eid');
expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]);
});
it('skips very large values', () => {
const input = [{ key: 'foo', value: 'a'.repeat(10000) }];
const output = mapToRows(input, 'eid');
expect(output).toEqual([]);
});
});
@@ -115,6 +113,8 @@ describe('search', () => {
metadata: { name: 'n' },
};
expect(buildEntitySearch('eid', input)).toEqual([
{ entity_id: 'eid', key: 'apiversion', value: 'a' },
{ entity_id: 'eid', key: 'kind', value: 'b' },
{ entity_id: 'eid', key: 'metadata.name', value: 'n' },
{ entity_id: 'eid', key: 'metadata.namespace', value: null },
{ entity_id: 'eid', key: 'metadata.uid', value: null },
@@ -123,8 +123,6 @@ describe('search', () => {
key: 'metadata.namespace',
value: ENTITY_DEFAULT_NAMESPACE,
},
{ entity_id: 'eid', key: 'apiversion', value: 'a' },
{ entity_id: 'eid', key: 'kind', value: 'b' },
]);
});
});
plugins/catalog-backend/src/database/search.ts
+79 -75
View File
@@ -28,16 +28,18 @@ const SPECIAL_KEYS = [
'metadata.generation',
];
function toValue(current: any): string | null {
if (current === undefined || current === null) {
return null;
}
// The maximum length allowed for search values. These columns are indexed, and
// database engines do not like to index on massive values. For example,
// postgres will balk after 8191 byte line sizes.
const MAX_VALUE_LENGTH = 200;
return String(current).toLowerCase();
}
type Kv = {
key: string;
value: any;
};
// Helper for iterating through a nested structure and outputting a list of
// path->value entries.
// Helper for traversing through a nested structure and outputting a list of
// path->value entries of the leaves.
//
// For example, this yaml structure
//
@@ -53,57 +55,77 @@ function toValue(current: any): string | null {
//
// will result in
//
// "a", "1"
// "a", 1
// "b.c", null
// "b.e": "f"
// "b.e": "g"
// "h.i": "1"
// "h.i": 1
// "h.j": "k"
// "h.i": "2"
// "h.i": 2
// "h.j": "l"
export function visitEntityPart(
entityId: string,
path: string,
current: any,
output: DbEntitiesSearchRow[],
) {
// ignored
if (SPECIAL_KEYS.includes(path)) {
return;
}
export function traverse(root: any): Kv[] {
const output: Kv[] = [];
// empty or scalar
if (
current === undefined ||
current === null ||
['string', 'number', 'boolean'].includes(typeof current)
) {
output.push({ entity_id: entityId, key: path, value: toValue(current) });
return;
}
// unknown
if (typeof current !== 'object') {
return;
}
// array
if (Array.isArray(current)) {
for (const item of current) {
visitEntityPart(entityId, path, item, output);
function visit(path: string, current: any) {
if (SPECIAL_KEYS.includes(path)) {
return;
}
// empty or scalar
if (
current === undefined ||
current === null ||
['string', 'number', 'boolean'].includes(typeof current)
) {
output.push({ key: path, value: current });
return;
}
// unknown
if (typeof current !== 'object') {
return;
}
// array
if (Array.isArray(current)) {
for (const item of current) {
// keep the same path as currently
visit(path, item);
}
return;
}
// object
for (const [key, value] of Object.entries(current)) {
visit(path ? `${path}.${key}` : key, value);
}
return;
}
// object
for (const [key, value] of Object.entries(current)) {
visitEntityPart(
entityId,
(path ? `${path}.${key}` : key).toLowerCase(),
value,
output,
);
visit('', root);
return output;
}
// Translates a number of raw data rows to search table rows
export function mapToRows(
input: Kv[],
entityId: string,
): DbEntitiesSearchRow[] {
const result: DbEntitiesSearchRow[] = [];
for (let { key, value } of input) {
key = key.toLowerCase();
if (value === undefined || value === null) {
result.push({ entity_id: entityId, key, value: null });
} else {
value = String(value).toLowerCase();
if (value.length <= MAX_VALUE_LENGTH) {
result.push({ entity_id: entityId, key, value });
}
}
}
return result;
}
/**
@@ -117,38 +139,20 @@ export function buildEntitySearch(
entityId: string,
entity: Entity,
): DbEntitiesSearchRow[] {
// Visit the entire structure recursively
const raw = traverse(entity);
// Start with some special keys that are always present because you want to
// be able to easily search for null specifically
const result: DbEntitiesSearchRow[] = [
{
entity_id: entityId,
key: 'metadata.name',
value: toValue(entity.metadata.name),
},
{
entity_id: entityId,
key: 'metadata.namespace',
value: toValue(entity.metadata.namespace),
},
{
entity_id: entityId,
key: 'metadata.uid',
value: toValue(entity.metadata.uid),
},
];
raw.push({ key: 'metadata.name', value: entity.metadata.name });
raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace });
raw.push({ key: 'metadata.uid', value: entity.metadata.uid });
// Namespace not specified has the default value "default", so we want to
// match on that as well
if (!entity.metadata.namespace) {
result.push({
entity_id: entityId,
key: 'metadata.namespace',
value: toValue(ENTITY_DEFAULT_NAMESPACE),
});
raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE });
}
// Visit the entire structure recursively
visitEntityPart(entityId, '', entity, result);
return result;
return mapToRows(raw, entityId);
}