paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 50 Packages Projects Releases Wiki Activity

Merge pull request #7843 from backstage/joonp/nested-entity-search-filter

Browse Source 
Allow singleton and flexibly nested EntityFilters
This commit is contained in:
Fredrik Adelöw
2021-11-08 18:49:43 +01:00
committed by GitHub
parent 76b9945796 c74834a62d
commit c0a7f95c87
6 changed files with 280 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/slimy-days-leave.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
Allow singleton and flexibly nested EntityFilters
plugins/catalog-backend/api-report.md
+8 -5
View File
@@ -870,11 +870,14 @@ export type EntityAncestryResponse = {
// Warning: (ae-missing-release-tag) "EntityFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export type EntityFilter = {
anyOf: {
allOf: EntitiesSearchFilter[];
}[];
};
export type EntityFilter =
| {
allOf: EntityFilter[];
}
| {
anyOf: EntityFilter[];
}
| EntitiesSearchFilter;
// Warning: (ae-missing-release-tag) "EntityPagination" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
plugins/catalog-backend/src/catalog/types.ts
+4 -3
View File
@@ -22,9 +22,10 @@ import { Entity, EntityRelationSpec } from '@backstage/catalog-model';
* Any (at least one) of the outer sets must match, within which all of the
* individual filters must match.
*/
export type EntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
export type EntityFilter =
| { allOf: EntityFilter[] }
| { anyOf: EntityFilter[] }
| EntitiesSearchFilter;
/**
* A pagination rule for entities.
plugins/catalog-backend/src/legacy/database/CommonDatabase.ts
+26 -7
View File
@@ -46,7 +46,11 @@ import {
DbPageInfo,
Transaction,
} from './types';
import { EntityPagination } from '../../catalog/types';
import { EntityPagination, EntitiesSearchFilter } from '../../catalog/types';
type LegacyEntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
// The number of items that are sent per batch to the database layer, when
// doing .batchInsert calls to knex. This needs to be low enough to not cause
@@ -217,13 +221,28 @@ export class CommonDatabase implements Database {
let entitiesQuery = tx<DbEntitiesRow>('entities');
for (const singleFilter of request?.filter?.anyOf ?? []) {
if (
request?.filter &&
(request.filter.hasOwnProperty('key') ||
request.filter.hasOwnProperty('allOf'))
) {
throw new Error(
'Filters for the legacy CommonDatabase must obey the { anyOf: [{ allOf: [] }] } format.',
);
}
for (const singleFilter of (request?.filter as LegacyEntityFilter)?.anyOf ??
[]) {
entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
for (const {
key,
matchValueIn,
matchValueExists,
} of singleFilter.allOf) {
for (const filter of singleFilter.allOf) {
if (
filter.hasOwnProperty('anyOf') ||
filter.hasOwnProperty('allOf')
) {
throw new Error(
'Nested filters are not supported in the legacy CommonDatabase',
);
}
const { key, matchValueIn, matchValueExists } = filter;
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
// make a lot of sense. However, it had abysmal performance on sqlite
// when datasets grew large, so we're using IN instead.
plugins/catalog-backend/src/service/NextEntitiesCatalog.test.ts
+149
View File
@@ -23,6 +23,7 @@ import {
DbFinalEntitiesRow,
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbSearchRow,
} from '../database/tables';
import { NextEntitiesCatalog } from './NextEntitiesCatalog';
@@ -73,6 +74,52 @@ describe('NextEntitiesCatalog', () => {
}
}
async function addEntityToSearch(knex: Knex, entity: Entity) {
const id = uuid();
const entityRef = stringifyEntityRef(entity);
const entityJson = JSON.stringify(entity);
await knex<DbRefreshStateRow>('refresh_state').insert({
entity_id: id,
entity_ref: entityRef,
unprocessed_entity: entityJson,
errors: '[]',
next_update_at: '2031-01-01 23:00:00',
last_discovery_at: '2021-04-01 13:37:00',
});
await knex<DbFinalEntitiesRow>('final_entities').insert({
entity_id: id,
final_entity: entityJson,
hash: 'h',
stitch_ticket: '',
});
await insertSearchRow(knex, id, null, entity);
}
async function insertSearchRow(
knex: Knex,
id: string,
previousKey: string | null,
previousValue: Object,
) {
return Promise.all(
Object.entries(previousValue).map(async ([key, value]) => {
const currentKey = `${previousKey ? `${previousKey}.` : ``}${key}`;
if (typeof value === 'object') {
await insertSearchRow(knex, id, currentKey, value);
} else {
await knex<DbSearchRow>('search').insert({
entity_id: id,
key: currentKey,
value: value,
});
}
}),
);
}
describe('entityAncestry', () => {
it.each(databases.eachSupportedId())(
'should return the ancestry with one parent, %p',
@@ -209,4 +256,106 @@ describe('NextEntitiesCatalog', () => {
60_000,
);
});
describe('entities', () => {
it.each(databases.eachSupportedId())(
'should return correct entity for simple filter',
async databaseId => {
const { knex } = await createDatabase(databaseId);
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'one' },
spec: {},
};
const entity2: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'two' },
spec: {
test: 'test value',
},
};
await addEntityToSearch(knex, entity1);
await addEntityToSearch(knex, entity2);
const catalog = new NextEntitiesCatalog(knex);
const testFilter = {
key: 'spec.test',
matchValueExists: true,
};
const request = { filter: testFilter };
const { entities } = await catalog.entities(request);
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity2);
},
);
it.each(databases.eachSupportedId())(
'should return correct entity for nested filter',
async databaseId => {
const { knex } = await createDatabase(databaseId);
const entity1: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'one', org: 'a', desc: 'description' },
spec: {},
};
const entity2: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'two', org: 'b', desc: 'description' },
spec: {},
};
const entity3: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'three', org: 'b', color: 'red' },
spec: {},
};
const entity4: Entity = {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'four', org: 'b', color: 'blue' },
spec: {},
};
await addEntityToSearch(knex, entity1);
await addEntityToSearch(knex, entity2);
await addEntityToSearch(knex, entity3);
await addEntityToSearch(knex, entity4);
const catalog = new NextEntitiesCatalog(knex);
const testFilter1 = {
key: 'metadata.org',
matchValueExists: true,
matchValueIn: ['b'],
};
const testFilter2 = {
key: 'metadata.desc',
matchValueExists: true,
};
const testFilter3 = {
key: 'metadata.color',
matchValueExists: true,
matchValueIn: ['blue'],
};
const request = {
filter: {
allOf: [
testFilter1,
{
anyOf: [testFilter2, testFilter3],
},
],
},
};
const { entities } = await catalog.entities(request);
expect(entities.length).toBe(2);
expect(entities).toContainEqual(entity2);
expect(entities).toContainEqual(entity4);
},
);
});
});
plugins/catalog-backend/src/service/NextEntitiesCatalog.ts
+88 -35
View File
@@ -23,6 +23,8 @@ import {
EntitiesResponse,
EntityAncestryResponse,
EntityPagination,
EntityFilter,
EntitiesSearchFilter,
} from '../catalog/types';
import {
DbFinalEntitiesRow,
@@ -73,6 +75,90 @@ function stringifyPagination(input: { limit: number; offset: number }) {
return base64;
}
function addCondition(
queryBuilder: Knex.QueryBuilder,
db: Knex,
{ key, matchValueIn, matchValueExists }: EntitiesSearchFilter,
) {
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
// make a lot of sense. However, it had abysmal performance on sqlite
// when datasets grew large, so we're using IN instead.
const matchQuery = db<DbSearchRow>('search')
.select('entity_id')
.where(function keyFilter() {
this.andWhere({ key: key.toLowerCase() });
if (matchValueExists !== false && matchValueIn) {
if (matchValueIn.length === 1) {
this.andWhere({ value: matchValueIn[0].toLowerCase() });
} else if (matchValueIn.length > 1) {
this.andWhere(
'value',
'in',
matchValueIn.map(v => v.toLowerCase()),
);
}
}
});
// Explicitly evaluate matchValueExists as a boolean since it may be undefined
queryBuilder.andWhere(
'entity_id',
matchValueExists === false ? 'not in' : 'in',
matchQuery,
);
}
function isEntitiesSearchFilter(
filter: EntitiesSearchFilter | EntityFilter,
): filter is EntitiesSearchFilter {
return filter.hasOwnProperty('key');
}
function isAndEntityFilter(
filter: { allOf: EntityFilter[] } | EntityFilter,
): filter is { allOf: EntityFilter[] } {
return filter.hasOwnProperty('allOf');
}
function isOrEntityFilter(
filter: { anyOf: EntityFilter[] } | EntityFilter,
): filter is { anyOf: EntityFilter[] } {
return filter.hasOwnProperty('anyOf');
}
function parseFilter(
filter: EntityFilter,
query: Knex.QueryBuilder,
db: Knex,
): Knex.QueryBuilder {
if (isEntitiesSearchFilter(filter)) {
return query.where(function filterFunction() {
addCondition(this, db, filter);
});
}
if (isOrEntityFilter(filter)) {
let cumulativeQuery = query;
for (const subFilter of filter.anyOf ?? []) {
cumulativeQuery = cumulativeQuery.orWhere(subQuery =>
parseFilter(subFilter, subQuery, db),
);
}
return cumulativeQuery;
}
if (isAndEntityFilter(filter)) {
let cumulativeQuery = query;
for (const subFilter of filter.allOf ?? []) {
cumulativeQuery = cumulativeQuery.andWhere(subQuery =>
parseFilter(subFilter, subQuery, db),
);
}
return cumulativeQuery;
}
return query;
}
export class NextEntitiesCatalog implements EntitiesCatalog {
constructor(private readonly database: Knex) {}
@@ -80,41 +166,8 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
const db = this.database;
let entitiesQuery = db<DbFinalEntitiesRow>('final_entities');
for (const singleFilter of request?.filter?.anyOf ?? []) {
entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
for (const {
key,
matchValueIn,
matchValueExists,
} of singleFilter.allOf) {
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
// make a lot of sense. However, it had abysmal performance on sqlite
// when datasets grew large, so we're using IN instead.
const matchQuery = db<DbSearchRow>('search')
.select('entity_id')
.where(function keyFilter() {
this.andWhere({ key: key.toLowerCase() });
if (matchValueExists !== false && matchValueIn) {
if (matchValueIn.length === 1) {
this.andWhere({ value: matchValueIn[0].toLowerCase() });
} else if (matchValueIn.length > 1) {
this.andWhere(
'value',
'in',
matchValueIn.map(v => v.toLowerCase()),
);
}
}
});
// Explicitly evaluate matchValueExists as a boolean since it may be undefined
this.andWhere(
'entity_id',
matchValueExists === false ? 'not in' : 'in',
matchQuery,
);
}
});
if (request?.filter) {
entitiesQuery = parseFilter(request.filter, entitiesQuery, db);
}
// TODO: move final_entities to use entity_ref