diff --git a/.changeset/catalog-exists-query-optimization.md b/.changeset/catalog-exists-query-optimization.md new file mode 100644 index 0000000000..e0eb2cbb31 --- /dev/null +++ b/.changeset/catalog-exists-query-optimization.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Improved catalog entity filter query performance by switching from `IN (subquery)` to `EXISTS (correlated subquery)` patterns. This enables PostgreSQL semi-join optimizations and fixes `NOT IN` NULL-semantics pitfalls by using `NOT EXISTS` instead. diff --git a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts index 9b540c95b7..92b1cd093f 100644 --- a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts +++ b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts @@ -31,8 +31,6 @@ import { buildEntitySearch } from '../../database/operations/stitcher/buildEntit jest.setTimeout(60_000); const databases = TestDatabases.create(); -const strategies = ['in', 'join'] as const; - describe.each(databases.eachSupportedId())( 'applyEntityFilterToQuery, %p', databaseId => { @@ -107,7 +105,7 @@ describe.each(databases.eachSupportedId())( } // #endregion - describe.each(strategies)('with strategy %p', strategy => { + describe('exists strategy', () => { async function query(filter: EntityFilter): Promise { const q = knex('final_entities').whereNotNull( @@ -118,7 +116,6 @@ describe.each(databases.eachSupportedId())( targetQuery: q, onEntityIdField: 'final_entities.entity_id', knex, - strategy, }); return await q.then(rows => rows diff --git a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts index 49356cd0ff..5605632b52 100644 --- a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts @@ -20,8 +20,8 @@ import { } from '@backstage/plugin-catalog-node'; import { FilterPredicate } from '@backstage/filter-predicates'; import { Knex } from 'knex'; -import { DbSearchRow } from '../../database/tables'; import { applyPredicateEntityFilterToQuery } from './applyPredicateEntityFilterToQuery'; +import { searchExists, SEARCH_FLT_ALIAS } from './searchSubquery'; function isEntitiesSearchFilter( filter: EntitiesSearchFilter | EntityFilter, @@ -42,27 +42,29 @@ function isNegationEntityFilter( } /** - * Applies filtering through a number of WHERE IN subqueries. Example: + * Applies filtering through correlated EXISTS subqueries. Example: * * ``` * SELECT * FROM final_entities * WHERE - * entity_id IN ( - * SELECT entity_id FROM search - * WHERE key = 'kind' AND value = 'component' + * EXISTS ( + * SELECT 1 FROM search AS search_flt + * WHERE search_flt.entity_id = final_entities.entity_id + * AND key = 'kind' AND value = 'component' * ) - * AND entity_id IN ( - * SELECT entity_id FROM search - * WHERE key = 'spec.lifecycle' AND value = 'production' + * AND EXISTS ( + * SELECT 1 FROM search AS search_flt + * WHERE search_flt.entity_id = final_entities.entity_id + * AND key = 'spec.lifecycle' AND value = 'production' * ) * AND final_entities.final_entity IS NOT NULL * ``` * - * This strategy is a good all-rounder, in the sense that it has medium-good - * performance on most queries on all database engines. However, it does not - * scale well down to very short runtimes as well as the JOIN strategy. + * The EXISTS strategy enables efficient semi-join plans, particularly on + * PostgreSQL with large datasets, since the database can stop scanning as + * soon as the first matching row is found. */ -function applyInStrategy( +function applyExistsStrategy( filter: EntityFilter, targetQuery: Knex.QueryBuilder, onEntityIdField: string, @@ -70,7 +72,7 @@ function applyInStrategy( negate: boolean, ): Knex.QueryBuilder { if (isNegationEntityFilter(filter)) { - return applyInStrategy( + return applyExistsStrategy( filter.not, targetQuery, onEntityIdField, @@ -82,21 +84,18 @@ function applyInStrategy( if (isEntitiesSearchFilter(filter)) { const key = filter.key.toLowerCase(); const values = filter.values?.map(v => v.toLowerCase()); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) + const subquery = searchExists(knex, onEntityIdField) + .where(`${SEARCH_FLT_ALIAS}.key`, key) .andWhere(function keyFilter() { if (values?.length === 1) { - this.where({ value: values.at(0) }); + this.where(`${SEARCH_FLT_ALIAS}.value`, values.at(0)); } else if (values) { - this.andWhere('value', 'in', values); + this.whereIn(`${SEARCH_FLT_ALIAS}.value`, values); } }); - return targetQuery.andWhere( - onEntityIdField, - negate ? 'not in' : 'in', - matchQuery, - ); + return negate + ? targetQuery.whereNotExists(subquery) + : targetQuery.whereExists(subquery); } return targetQuery[negate ? 'andWhereNot' : 'andWhere']( @@ -104,13 +103,25 @@ function applyInStrategy( if (isOrEntityFilter(filter)) { for (const subFilter of filter.anyOf ?? []) { this.orWhere(subQuery => - applyInStrategy(subFilter, subQuery, onEntityIdField, knex, false), + applyExistsStrategy( + subFilter, + subQuery, + onEntityIdField, + knex, + false, + ), ); } } else { for (const subFilter of filter.allOf ?? []) { this.andWhere(subQuery => - applyInStrategy(subFilter, subQuery, onEntityIdField, knex, false), + applyExistsStrategy( + subFilter, + subQuery, + onEntityIdField, + knex, + false, + ), ); } } @@ -125,14 +136,13 @@ export function applyEntityFilterToQuery(options: { targetQuery: Knex.QueryBuilder; onEntityIdField: string; knex: Knex; - strategy?: 'in' | 'join'; }): Knex.QueryBuilder { const { filter, query, targetQuery, onEntityIdField, knex } = options; let result = targetQuery; if (filter) { - result = applyInStrategy(filter, result, onEntityIdField, knex, false); + result = applyExistsStrategy(filter, result, onEntityIdField, knex, false); } if (query) { diff --git a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts index d915b8c4b5..c077d45e54 100644 --- a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts @@ -21,7 +21,7 @@ import { } from '@backstage/filter-predicates'; import { InputError } from '@backstage/errors'; import { Knex } from 'knex'; -import { DbSearchRow } from '../../database/tables'; +import { searchExists, SEARCH_FLT_ALIAS as S } from './searchSubquery'; function isPrimitive(value: unknown): value is FilterPredicatePrimitive { return ( @@ -128,44 +128,45 @@ function applyFieldCondition(options: { const { key, value, targetQuery, onEntityIdField, knex } = options; if (isPrimitive(value)) { - const matchQuery = knex('search') - .select('search.entity_id') - .where({ - key, - value: String(value).toLocaleLowerCase('en-US'), - }); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .where(`${S}.value`, String(value).toLocaleLowerCase('en-US')), + ); } if (isObject(value)) { if ('$exists' in value) { - const existsQuery = knex('search') - .select('search.entity_id') - .where({ key }); - return targetQuery.andWhere( - onEntityIdField, - value.$exists ? 'in' : 'not in', - existsQuery, + const subquery = searchExists(knex, onEntityIdField).where( + `${S}.key`, + key, ); + return value.$exists + ? targetQuery.whereExists(subquery) + : targetQuery.whereNotExists(subquery); } if ('$in' in value) { const values = value.$in.map(v => String(v).toLocaleLowerCase('en-US')); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) - .whereIn('value', values); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .whereIn(`${S}.value`, values), + ); } if ('$hasPrefix' in value) { const prefix = value.$hasPrefix.toLocaleLowerCase('en-US'); const escaped = prefix.replace(/[%_\\]/g, c => `\\${c}`); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) - .andWhereRaw('?? like ? escape ?', ['value', `${escaped}%`, '\\']); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .andWhereRaw('?? like ? escape ?', [ + `${S}.value`, + `${escaped}%`, + '\\', + ]), + ); } if ('$contains' in value) { @@ -182,13 +183,11 @@ function applyFieldCondition(options: { // "b" key with a primitive value. We'll consider that an acceptable // tradeoff though. if (isPrimitive(target)) { - const matchQuery = knex('search') - .select('search.entity_id') - .where({ - key, - value: String(target).toLocaleLowerCase('en-US'), - }); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .where(`${S}.value`, String(target).toLocaleLowerCase('en-US')), + ); } // Object form of $contains - currently only supports relation-style @@ -317,13 +316,14 @@ function applyContainsRelation(options: { ); } - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key: `relations.${type.toLocaleLowerCase('en-US')}` }); + const subquery = searchExists(knex, onEntityIdField).where( + `${S}.key`, + `relations.${type.toLocaleLowerCase('en-US')}`, + ); if (targetRef) { - matchQuery.whereIn('value', targetRef); + subquery.whereIn(`${S}.value`, targetRef); } - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists(subquery); } diff --git a/plugins/catalog-backend/src/service/request/searchSubquery.ts b/plugins/catalog-backend/src/service/request/searchSubquery.ts new file mode 100644 index 0000000000..db2b3c83a6 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/searchSubquery.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; + +/** + * Alias used for the search table in EXISTS subqueries, to avoid ambiguity + * when the outer query is also on the search table (e.g. facets queries). + */ +export const SEARCH_FLT_ALIAS = 'search_flt'; + +/** + * Creates an EXISTS subquery base against the search table, correlated on + * entity_id with the outer query's entity id field. + */ +export function searchExists( + knex: Knex, + onEntityIdField: string, +): Knex.QueryBuilder { + return knex(`search as ${SEARCH_FLT_ALIAS}`) + .select(knex.raw('1')) + .whereRaw('?? = ??', [`${SEARCH_FLT_ALIAS}.entity_id`, onEntityIdField]); +}