From 688481429c2eda7356987a90616c79082cd2ae22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sat, 28 Mar 2026 22:45:48 +0100 Subject: [PATCH 1/2] catalog-backend: optimize entity filter queries with EXISTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch filter query builders from IN (subquery) to EXISTS (correlated subquery) patterns. This enables PostgreSQL semi-join optimizations (stops at first match) and replaces NOT IN with NOT EXISTS (faster, no NULL-semantics pitfalls). Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .../catalog-exists-query-optimization.md | 5 ++ .../request/applyEntityFilterToQuery.ts | 23 ++--- .../applyPredicateEntityFilterToQuery.ts | 87 +++++++++++-------- 3 files changed, 67 insertions(+), 48 deletions(-) create mode 100644 .changeset/catalog-exists-query-optimization.md diff --git a/.changeset/catalog-exists-query-optimization.md b/.changeset/catalog-exists-query-optimization.md new file mode 100644 index 0000000000..e0eb2cbb31 --- /dev/null +++ b/.changeset/catalog-exists-query-optimization.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Improved catalog entity filter query performance by switching from `IN (subquery)` to `EXISTS (correlated subquery)` patterns. This enables PostgreSQL semi-join optimizations and fixes `NOT IN` NULL-semantics pitfalls by using `NOT EXISTS` instead. diff --git a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts index 49356cd0ff..dce7f2ead1 100644 --- a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts @@ -20,9 +20,11 @@ import { } from '@backstage/plugin-catalog-node'; import { FilterPredicate } from '@backstage/filter-predicates'; import { Knex } from 'knex'; -import { DbSearchRow } from '../../database/tables'; import { applyPredicateEntityFilterToQuery } from './applyPredicateEntityFilterToQuery'; +// Alias used for the search table in EXISTS subqueries +const S = 'search_flt'; + function isEntitiesSearchFilter( filter: EntitiesSearchFilter | EntityFilter, ): filter is EntitiesSearchFilter { @@ -82,21 +84,20 @@ function applyInStrategy( if (isEntitiesSearchFilter(filter)) { const key = filter.key.toLowerCase(); const values = filter.values?.map(v => v.toLowerCase()); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) + const subquery = knex(`search as ${S}`) + .select(knex.raw('1')) + .whereRaw('?? = ??', [`${S}.entity_id`, onEntityIdField]) + .where(`${S}.key`, key) .andWhere(function keyFilter() { if (values?.length === 1) { - this.where({ value: values.at(0) }); + this.where(`${S}.value`, values.at(0)); } else if (values) { - this.andWhere('value', 'in', values); + this.whereIn(`${S}.value`, values); } }); - return targetQuery.andWhere( - onEntityIdField, - negate ? 'not in' : 'in', - matchQuery, - ); + return negate + ? targetQuery.whereNotExists(subquery) + : targetQuery.whereExists(subquery); } return targetQuery[negate ? 'andWhereNot' : 'andWhere']( diff --git a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts index d915b8c4b5..c6a8eb47eb 100644 --- a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts @@ -21,7 +21,6 @@ import { } from '@backstage/filter-predicates'; import { InputError } from '@backstage/errors'; import { Knex } from 'knex'; -import { DbSearchRow } from '../../database/tables'; function isPrimitive(value: unknown): value is FilterPredicatePrimitive { return ( @@ -35,6 +34,20 @@ function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +// Alias used for the search table in EXISTS subqueries, to avoid ambiguity +// when the outer query is also on the search table (e.g. facets queries). +const S = 'search_flt'; + +/** + * Creates an EXISTS subquery base against the search table, correlated on + * entity_id with the outer query's entity id field. + */ +function searchExists(knex: Knex, onEntityIdField: string): Knex.QueryBuilder { + return knex(`search as ${S}`) + .select(knex.raw('1')) + .whereRaw('?? = ??', [`${S}.entity_id`, onEntityIdField]); +} + export function applyPredicateEntityFilterToQuery(options: { filter: FilterPredicate; targetQuery: Knex.QueryBuilder; @@ -128,44 +141,45 @@ function applyFieldCondition(options: { const { key, value, targetQuery, onEntityIdField, knex } = options; if (isPrimitive(value)) { - const matchQuery = knex('search') - .select('search.entity_id') - .where({ - key, - value: String(value).toLocaleLowerCase('en-US'), - }); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .where(`${S}.value`, String(value).toLocaleLowerCase('en-US')), + ); } if (isObject(value)) { if ('$exists' in value) { - const existsQuery = knex('search') - .select('search.entity_id') - .where({ key }); - return targetQuery.andWhere( - onEntityIdField, - value.$exists ? 'in' : 'not in', - existsQuery, + const subquery = searchExists(knex, onEntityIdField).where( + `${S}.key`, + key, ); + return value.$exists + ? targetQuery.whereExists(subquery) + : targetQuery.whereNotExists(subquery); } if ('$in' in value) { const values = value.$in.map(v => String(v).toLocaleLowerCase('en-US')); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) - .whereIn('value', values); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .whereIn(`${S}.value`, values), + ); } if ('$hasPrefix' in value) { const prefix = value.$hasPrefix.toLocaleLowerCase('en-US'); const escaped = prefix.replace(/[%_\\]/g, c => `\\${c}`); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) - .andWhereRaw('?? like ? escape ?', ['value', `${escaped}%`, '\\']); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .andWhereRaw('?? like ? escape ?', [ + `${S}.value`, + `${escaped}%`, + '\\', + ]), + ); } if ('$contains' in value) { @@ -182,13 +196,11 @@ function applyFieldCondition(options: { // "b" key with a primitive value. We'll consider that an acceptable // tradeoff though. if (isPrimitive(target)) { - const matchQuery = knex('search') - .select('search.entity_id') - .where({ - key, - value: String(target).toLocaleLowerCase('en-US'), - }); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .where(`${S}.value`, String(target).toLocaleLowerCase('en-US')), + ); } // Object form of $contains - currently only supports relation-style @@ -317,13 +329,14 @@ function applyContainsRelation(options: { ); } - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key: `relations.${type.toLocaleLowerCase('en-US')}` }); + const subquery = searchExists(knex, onEntityIdField).where( + `${S}.key`, + `relations.${type.toLocaleLowerCase('en-US')}`, + ); if (targetRef) { - matchQuery.whereIn('value', targetRef); + subquery.whereIn(`${S}.value`, targetRef); } - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists(subquery); } From 4538f05b24f672b53aa2e3f9b8541ada256e7cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sun, 29 Mar 2026 12:31:01 +0200 Subject: [PATCH 2/2] Address PR review: extract shared searchExists helper and rename strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract shared `searchExists` helper into `searchSubquery.ts` to avoid duplicate EXISTS subquery logic between the two filter modules - Rename `applyInStrategy` to `applyExistsStrategy` to reflect the actual query shape - Update block comment with correct EXISTS SQL examples - Remove unused `strategy` parameter from `applyEntityFilterToQuery` Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .../request/applyEntityFilterToQuery.test.ts | 5 +- .../request/applyEntityFilterToQuery.ts | 59 +++++++++++-------- .../applyPredicateEntityFilterToQuery.ts | 15 +---- .../src/service/request/searchSubquery.ts | 36 +++++++++++ 4 files changed, 72 insertions(+), 43 deletions(-) create mode 100644 plugins/catalog-backend/src/service/request/searchSubquery.ts diff --git a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts index 9b540c95b7..92b1cd093f 100644 --- a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts +++ b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.test.ts @@ -31,8 +31,6 @@ import { buildEntitySearch } from '../../database/operations/stitcher/buildEntit jest.setTimeout(60_000); const databases = TestDatabases.create(); -const strategies = ['in', 'join'] as const; - describe.each(databases.eachSupportedId())( 'applyEntityFilterToQuery, %p', databaseId => { @@ -107,7 +105,7 @@ describe.each(databases.eachSupportedId())( } // #endregion - describe.each(strategies)('with strategy %p', strategy => { + describe('exists strategy', () => { async function query(filter: EntityFilter): Promise { const q = knex('final_entities').whereNotNull( @@ -118,7 +116,6 @@ describe.each(databases.eachSupportedId())( targetQuery: q, onEntityIdField: 'final_entities.entity_id', knex, - strategy, }); return await q.then(rows => rows diff --git a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts index dce7f2ead1..5605632b52 100644 --- a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts @@ -21,9 +21,7 @@ import { import { FilterPredicate } from '@backstage/filter-predicates'; import { Knex } from 'knex'; import { applyPredicateEntityFilterToQuery } from './applyPredicateEntityFilterToQuery'; - -// Alias used for the search table in EXISTS subqueries -const S = 'search_flt'; +import { searchExists, SEARCH_FLT_ALIAS } from './searchSubquery'; function isEntitiesSearchFilter( filter: EntitiesSearchFilter | EntityFilter, @@ -44,27 +42,29 @@ function isNegationEntityFilter( } /** - * Applies filtering through a number of WHERE IN subqueries. Example: + * Applies filtering through correlated EXISTS subqueries. Example: * * ``` * SELECT * FROM final_entities * WHERE - * entity_id IN ( - * SELECT entity_id FROM search - * WHERE key = 'kind' AND value = 'component' + * EXISTS ( + * SELECT 1 FROM search AS search_flt + * WHERE search_flt.entity_id = final_entities.entity_id + * AND key = 'kind' AND value = 'component' * ) - * AND entity_id IN ( - * SELECT entity_id FROM search - * WHERE key = 'spec.lifecycle' AND value = 'production' + * AND EXISTS ( + * SELECT 1 FROM search AS search_flt + * WHERE search_flt.entity_id = final_entities.entity_id + * AND key = 'spec.lifecycle' AND value = 'production' * ) * AND final_entities.final_entity IS NOT NULL * ``` * - * This strategy is a good all-rounder, in the sense that it has medium-good - * performance on most queries on all database engines. However, it does not - * scale well down to very short runtimes as well as the JOIN strategy. + * The EXISTS strategy enables efficient semi-join plans, particularly on + * PostgreSQL with large datasets, since the database can stop scanning as + * soon as the first matching row is found. */ -function applyInStrategy( +function applyExistsStrategy( filter: EntityFilter, targetQuery: Knex.QueryBuilder, onEntityIdField: string, @@ -72,7 +72,7 @@ function applyInStrategy( negate: boolean, ): Knex.QueryBuilder { if (isNegationEntityFilter(filter)) { - return applyInStrategy( + return applyExistsStrategy( filter.not, targetQuery, onEntityIdField, @@ -84,15 +84,13 @@ function applyInStrategy( if (isEntitiesSearchFilter(filter)) { const key = filter.key.toLowerCase(); const values = filter.values?.map(v => v.toLowerCase()); - const subquery = knex(`search as ${S}`) - .select(knex.raw('1')) - .whereRaw('?? = ??', [`${S}.entity_id`, onEntityIdField]) - .where(`${S}.key`, key) + const subquery = searchExists(knex, onEntityIdField) + .where(`${SEARCH_FLT_ALIAS}.key`, key) .andWhere(function keyFilter() { if (values?.length === 1) { - this.where(`${S}.value`, values.at(0)); + this.where(`${SEARCH_FLT_ALIAS}.value`, values.at(0)); } else if (values) { - this.whereIn(`${S}.value`, values); + this.whereIn(`${SEARCH_FLT_ALIAS}.value`, values); } }); return negate @@ -105,13 +103,25 @@ function applyInStrategy( if (isOrEntityFilter(filter)) { for (const subFilter of filter.anyOf ?? []) { this.orWhere(subQuery => - applyInStrategy(subFilter, subQuery, onEntityIdField, knex, false), + applyExistsStrategy( + subFilter, + subQuery, + onEntityIdField, + knex, + false, + ), ); } } else { for (const subFilter of filter.allOf ?? []) { this.andWhere(subQuery => - applyInStrategy(subFilter, subQuery, onEntityIdField, knex, false), + applyExistsStrategy( + subFilter, + subQuery, + onEntityIdField, + knex, + false, + ), ); } } @@ -126,14 +136,13 @@ export function applyEntityFilterToQuery(options: { targetQuery: Knex.QueryBuilder; onEntityIdField: string; knex: Knex; - strategy?: 'in' | 'join'; }): Knex.QueryBuilder { const { filter, query, targetQuery, onEntityIdField, knex } = options; let result = targetQuery; if (filter) { - result = applyInStrategy(filter, result, onEntityIdField, knex, false); + result = applyExistsStrategy(filter, result, onEntityIdField, knex, false); } if (query) { diff --git a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts index c6a8eb47eb..c077d45e54 100644 --- a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts @@ -21,6 +21,7 @@ import { } from '@backstage/filter-predicates'; import { InputError } from '@backstage/errors'; import { Knex } from 'knex'; +import { searchExists, SEARCH_FLT_ALIAS as S } from './searchSubquery'; function isPrimitive(value: unknown): value is FilterPredicatePrimitive { return ( @@ -34,20 +35,6 @@ function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -// Alias used for the search table in EXISTS subqueries, to avoid ambiguity -// when the outer query is also on the search table (e.g. facets queries). -const S = 'search_flt'; - -/** - * Creates an EXISTS subquery base against the search table, correlated on - * entity_id with the outer query's entity id field. - */ -function searchExists(knex: Knex, onEntityIdField: string): Knex.QueryBuilder { - return knex(`search as ${S}`) - .select(knex.raw('1')) - .whereRaw('?? = ??', [`${S}.entity_id`, onEntityIdField]); -} - export function applyPredicateEntityFilterToQuery(options: { filter: FilterPredicate; targetQuery: Knex.QueryBuilder; diff --git a/plugins/catalog-backend/src/service/request/searchSubquery.ts b/plugins/catalog-backend/src/service/request/searchSubquery.ts new file mode 100644 index 0000000000..db2b3c83a6 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/searchSubquery.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2026 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; + +/** + * Alias used for the search table in EXISTS subqueries, to avoid ambiguity + * when the outer query is also on the search table (e.g. facets queries). + */ +export const SEARCH_FLT_ALIAS = 'search_flt'; + +/** + * Creates an EXISTS subquery base against the search table, correlated on + * entity_id with the outer query's entity id field. + */ +export function searchExists( + knex: Knex, + onEntityIdField: string, +): Knex.QueryBuilder { + return knex(`search as ${SEARCH_FLT_ALIAS}`) + .select(knex.raw('1')) + .whereRaw('?? = ??', [`${SEARCH_FLT_ALIAS}.entity_id`, onEntityIdField]); +}