paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 18 Packages Projects Releases Wiki Activity

Merge pull request #33290 from backstage/freben/catalog-exists-query-optimization

Browse Source 
catalog-backend: optimize entity filter queries with EXISTS
This commit is contained in:
Fredrik Adelöw
2026-03-29 12:48:38 +02:00
committed by GitHub
parent 9f3affb864 4538f05b24
commit 0eb8d1a31f
5 changed files with 116 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/catalog-exists-query-optimization.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
Improved catalog entity filter query performance by switching from `IN (subquery)` to `EXISTS (correlated subquery)` patterns. This enables PostgreSQL semi-join optimizations and fixes `NOT IN` NULL-semantics pitfalls by using `NOT EXISTS` instead.
plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts
+1 -4
View File
@@ -31,8 +31,6 @@ import { buildEntitySearch } from '../../database/operations/stitcher/buildEntit
jest.setTimeout(60_000);
const databases = TestDatabases.create();
const strategies = ['in', 'join'] as const;
describe.each(databases.eachSupportedId())(
'applyEntityFilterToQuery, %p',
databaseId => {
@@ -107,7 +105,7 @@ describe.each(databases.eachSupportedId())(
}
// #endregion
describe.each(strategies)('with strategy %p', strategy => {
describe('exists strategy', () => {
async function query(filter: EntityFilter): Promise<string[]> {
const q =
knex<DbFinalEntitiesRow>('final_entities').whereNotNull(
@@ -118,7 +116,6 @@ describe.each(databases.eachSupportedId())(
targetQuery: q,
onEntityIdField: 'final_entities.entity_id',
knex,
strategy,
});
return await q.then(rows =>
rows
plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts
+37 -27
View File
@@ -20,8 +20,8 @@ import {
} from '@backstage/plugin-catalog-node';
import { FilterPredicate } from '@backstage/filter-predicates';
import { Knex } from 'knex';
import { DbSearchRow } from '../../database/tables';
import { applyPredicateEntityFilterToQuery } from './applyPredicateEntityFilterToQuery';
import { searchExists, SEARCH_FLT_ALIAS } from './searchSubquery';
function isEntitiesSearchFilter(
filter: EntitiesSearchFilter | EntityFilter,
@@ -42,27 +42,29 @@ function isNegationEntityFilter(
}
/**
* Applies filtering through a number of WHERE IN subqueries. Example:
* Applies filtering through correlated EXISTS subqueries. Example:
*
* ```
* SELECT * FROM final_entities
* WHERE
* entity_id IN (
* SELECT entity_id FROM search
* WHERE key = 'kind' AND value = 'component'
* EXISTS (
* SELECT 1 FROM search AS search_flt
* WHERE search_flt.entity_id = final_entities.entity_id
* AND key = 'kind' AND value = 'component'
* )
* AND entity_id IN (
* SELECT entity_id FROM search
* WHERE key = 'spec.lifecycle' AND value = 'production'
* AND EXISTS (
* SELECT 1 FROM search AS search_flt
* WHERE search_flt.entity_id = final_entities.entity_id
* AND key = 'spec.lifecycle' AND value = 'production'
* )
* AND final_entities.final_entity IS NOT NULL
* ```
*
* This strategy is a good all-rounder, in the sense that it has medium-good
* performance on most queries on all database engines. However, it does not
* scale well down to very short runtimes as well as the JOIN strategy.
* The EXISTS strategy enables efficient semi-join plans, particularly on
* PostgreSQL with large datasets, since the database can stop scanning as
* soon as the first matching row is found.
*/
function applyInStrategy(
function applyExistsStrategy(
filter: EntityFilter,
targetQuery: Knex.QueryBuilder,
onEntityIdField: string,
@@ -70,7 +72,7 @@ function applyInStrategy(
negate: boolean,
): Knex.QueryBuilder {
if (isNegationEntityFilter(filter)) {
return applyInStrategy(
return applyExistsStrategy(
filter.not,
targetQuery,
onEntityIdField,
@@ -82,21 +84,18 @@ function applyInStrategy(
if (isEntitiesSearchFilter(filter)) {
const key = filter.key.toLowerCase();
const values = filter.values?.map(v => v.toLowerCase());
const matchQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({ key })
const subquery = searchExists(knex, onEntityIdField)
.where(`${SEARCH_FLT_ALIAS}.key`, key)
.andWhere(function keyFilter() {
if (values?.length === 1) {
this.where({ value: values.at(0) });
this.where(`${SEARCH_FLT_ALIAS}.value`, values.at(0));
} else if (values) {
this.andWhere('value', 'in', values);
this.whereIn(`${SEARCH_FLT_ALIAS}.value`, values);
}
});
return targetQuery.andWhere(
onEntityIdField,
negate ? 'not in' : 'in',
matchQuery,
);
return negate
? targetQuery.whereNotExists(subquery)
: targetQuery.whereExists(subquery);
}
return targetQuery[negate ? 'andWhereNot' : 'andWhere'](
@@ -104,13 +103,25 @@ function applyInStrategy(
if (isOrEntityFilter(filter)) {
for (const subFilter of filter.anyOf ?? []) {
this.orWhere(subQuery =>
applyInStrategy(subFilter, subQuery, onEntityIdField, knex, false),
applyExistsStrategy(
subFilter,
subQuery,
onEntityIdField,
knex,
false,
),
);
}
} else {
for (const subFilter of filter.allOf ?? []) {
this.andWhere(subQuery =>
applyInStrategy(subFilter, subQuery, onEntityIdField, knex, false),
applyExistsStrategy(
subFilter,
subQuery,
onEntityIdField,
knex,
false,
),
);
}
}
@@ -125,14 +136,13 @@ export function applyEntityFilterToQuery(options: {
targetQuery: Knex.QueryBuilder;
onEntityIdField: string;
knex: Knex;
strategy?: 'in' | 'join';
}): Knex.QueryBuilder {
const { filter, query, targetQuery, onEntityIdField, knex } = options;
let result = targetQuery;
if (filter) {
result = applyInStrategy(filter, result, onEntityIdField, knex, false);
result = applyExistsStrategy(filter, result, onEntityIdField, knex, false);
}
if (query) {
plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts
+37 -37
View File
@@ -21,7 +21,7 @@ import {
} from '@backstage/filter-predicates';
import { InputError } from '@backstage/errors';
import { Knex } from 'knex';
import { DbSearchRow } from '../../database/tables';
import { searchExists, SEARCH_FLT_ALIAS as S } from './searchSubquery';
function isPrimitive(value: unknown): value is FilterPredicatePrimitive {
return (
@@ -128,44 +128,45 @@ function applyFieldCondition(options: {
const { key, value, targetQuery, onEntityIdField, knex } = options;
if (isPrimitive(value)) {
const matchQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({
key,
value: String(value).toLocaleLowerCase('en-US'),
});
return targetQuery.andWhere(onEntityIdField, 'in', matchQuery);
return targetQuery.whereExists(
searchExists(knex, onEntityIdField)
.where(`${S}.key`, key)
.where(`${S}.value`, String(value).toLocaleLowerCase('en-US')),
);
}
if (isObject(value)) {
if ('$exists' in value) {
const existsQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({ key });
return targetQuery.andWhere(
onEntityIdField,
value.$exists ? 'in' : 'not in',
existsQuery,
const subquery = searchExists(knex, onEntityIdField).where(
`${S}.key`,
key,
);
return value.$exists
? targetQuery.whereExists(subquery)
: targetQuery.whereNotExists(subquery);
}
if ('$in' in value) {
const values = value.$in.map(v => String(v).toLocaleLowerCase('en-US'));
const matchQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({ key })
.whereIn('value', values);
return targetQuery.andWhere(onEntityIdField, 'in', matchQuery);
return targetQuery.whereExists(
searchExists(knex, onEntityIdField)
.where(`${S}.key`, key)
.whereIn(`${S}.value`, values),
);
}
if ('$hasPrefix' in value) {
const prefix = value.$hasPrefix.toLocaleLowerCase('en-US');
const escaped = prefix.replace(/[%_\\]/g, c => `\\${c}`);
const matchQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({ key })
.andWhereRaw('?? like ? escape ?', ['value', `${escaped}%`, '\\']);
return targetQuery.andWhere(onEntityIdField, 'in', matchQuery);
return targetQuery.whereExists(
searchExists(knex, onEntityIdField)
.where(`${S}.key`, key)
.andWhereRaw('?? like ? escape ?', [
`${S}.value`,
`${escaped}%`,
'\\',
]),
);
}
if ('$contains' in value) {
@@ -182,13 +183,11 @@ function applyFieldCondition(options: {
// "b" key with a primitive value. We'll consider that an acceptable
// tradeoff though.
if (isPrimitive(target)) {
const matchQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({
key,
value: String(target).toLocaleLowerCase('en-US'),
});
return targetQuery.andWhere(onEntityIdField, 'in', matchQuery);
return targetQuery.whereExists(
searchExists(knex, onEntityIdField)
.where(`${S}.key`, key)
.where(`${S}.value`, String(target).toLocaleLowerCase('en-US')),
);
}
// Object form of $contains - currently only supports relation-style
@@ -317,13 +316,14 @@ function applyContainsRelation(options: {
);
}
const matchQuery = knex<DbSearchRow>('search')
.select('search.entity_id')
.where({ key: `relations.${type.toLocaleLowerCase('en-US')}` });
const subquery = searchExists(knex, onEntityIdField).where(
`${S}.key`,
`relations.${type.toLocaleLowerCase('en-US')}`,
);
if (targetRef) {
matchQuery.whereIn('value', targetRef);
subquery.whereIn(`${S}.value`, targetRef);
}
return targetQuery.andWhere(onEntityIdField, 'in', matchQuery);
return targetQuery.whereExists(subquery);
}
plugins/catalog-backend/src/service/request/searchSubquery.ts
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex } from 'knex';
/**
* Alias used for the search table in EXISTS subqueries, to avoid ambiguity
* when the outer query is also on the search table (e.g. facets queries).
*/
export const SEARCH_FLT_ALIAS = 'search_flt';
/**
* Creates an EXISTS subquery base against the search table, correlated on
* entity_id with the outer query's entity id field.
*/
export function searchExists(
knex: Knex,
onEntityIdField: string,
): Knex.QueryBuilder {
return knex(`search as ${SEARCH_FLT_ALIAS}`)
.select(knex.raw('1'))
.whereRaw('?? = ??', [`${SEARCH_FLT_ALIAS}.entity_id`, onEntityIdField]);
}