diff --git a/.changeset/catalog-exists-query-optimization.md b/.changeset/catalog-exists-query-optimization.md new file mode 100644 index 0000000000..e0eb2cbb31 --- /dev/null +++ b/.changeset/catalog-exists-query-optimization.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Improved catalog entity filter query performance by switching from `IN (subquery)` to `EXISTS (correlated subquery)` patterns. This enables PostgreSQL semi-join optimizations and fixes `NOT IN` NULL-semantics pitfalls by using `NOT EXISTS` instead. diff --git a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts index 49356cd0ff..dce7f2ead1 100644 --- a/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyEntityFilterToQuery.ts @@ -20,9 +20,11 @@ import { } from '@backstage/plugin-catalog-node'; import { FilterPredicate } from '@backstage/filter-predicates'; import { Knex } from 'knex'; -import { DbSearchRow } from '../../database/tables'; import { applyPredicateEntityFilterToQuery } from './applyPredicateEntityFilterToQuery'; +// Alias used for the search table in EXISTS subqueries +const S = 'search_flt'; + function isEntitiesSearchFilter( filter: EntitiesSearchFilter | EntityFilter, ): filter is EntitiesSearchFilter { @@ -82,21 +84,20 @@ function applyInStrategy( if (isEntitiesSearchFilter(filter)) { const key = filter.key.toLowerCase(); const values = filter.values?.map(v => v.toLowerCase()); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) + const subquery = knex(`search as ${S}`) + .select(knex.raw('1')) + .whereRaw('?? = ??', [`${S}.entity_id`, onEntityIdField]) + .where(`${S}.key`, key) .andWhere(function keyFilter() { if (values?.length === 1) { - this.where({ value: values.at(0) }); + this.where(`${S}.value`, values.at(0)); } else if (values) { - this.andWhere('value', 'in', values); + this.whereIn(`${S}.value`, values); } }); - return targetQuery.andWhere( - onEntityIdField, - negate ? 'not in' : 'in', - matchQuery, - ); + return negate + ? targetQuery.whereNotExists(subquery) + : targetQuery.whereExists(subquery); } return targetQuery[negate ? 'andWhereNot' : 'andWhere']( diff --git a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts index d915b8c4b5..c6a8eb47eb 100644 --- a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts +++ b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts @@ -21,7 +21,6 @@ import { } from '@backstage/filter-predicates'; import { InputError } from '@backstage/errors'; import { Knex } from 'knex'; -import { DbSearchRow } from '../../database/tables'; function isPrimitive(value: unknown): value is FilterPredicatePrimitive { return ( @@ -35,6 +34,20 @@ function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +// Alias used for the search table in EXISTS subqueries, to avoid ambiguity +// when the outer query is also on the search table (e.g. facets queries). +const S = 'search_flt'; + +/** + * Creates an EXISTS subquery base against the search table, correlated on + * entity_id with the outer query's entity id field. + */ +function searchExists(knex: Knex, onEntityIdField: string): Knex.QueryBuilder { + return knex(`search as ${S}`) + .select(knex.raw('1')) + .whereRaw('?? = ??', [`${S}.entity_id`, onEntityIdField]); +} + export function applyPredicateEntityFilterToQuery(options: { filter: FilterPredicate; targetQuery: Knex.QueryBuilder; @@ -128,44 +141,45 @@ function applyFieldCondition(options: { const { key, value, targetQuery, onEntityIdField, knex } = options; if (isPrimitive(value)) { - const matchQuery = knex('search') - .select('search.entity_id') - .where({ - key, - value: String(value).toLocaleLowerCase('en-US'), - }); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .where(`${S}.value`, String(value).toLocaleLowerCase('en-US')), + ); } if (isObject(value)) { if ('$exists' in value) { - const existsQuery = knex('search') - .select('search.entity_id') - .where({ key }); - return targetQuery.andWhere( - onEntityIdField, - value.$exists ? 'in' : 'not in', - existsQuery, + const subquery = searchExists(knex, onEntityIdField).where( + `${S}.key`, + key, ); + return value.$exists + ? targetQuery.whereExists(subquery) + : targetQuery.whereNotExists(subquery); } if ('$in' in value) { const values = value.$in.map(v => String(v).toLocaleLowerCase('en-US')); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) - .whereIn('value', values); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .whereIn(`${S}.value`, values), + ); } if ('$hasPrefix' in value) { const prefix = value.$hasPrefix.toLocaleLowerCase('en-US'); const escaped = prefix.replace(/[%_\\]/g, c => `\\${c}`); - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key }) - .andWhereRaw('?? like ? escape ?', ['value', `${escaped}%`, '\\']); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .andWhereRaw('?? like ? escape ?', [ + `${S}.value`, + `${escaped}%`, + '\\', + ]), + ); } if ('$contains' in value) { @@ -182,13 +196,11 @@ function applyFieldCondition(options: { // "b" key with a primitive value. We'll consider that an acceptable // tradeoff though. if (isPrimitive(target)) { - const matchQuery = knex('search') - .select('search.entity_id') - .where({ - key, - value: String(target).toLocaleLowerCase('en-US'), - }); - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists( + searchExists(knex, onEntityIdField) + .where(`${S}.key`, key) + .where(`${S}.value`, String(target).toLocaleLowerCase('en-US')), + ); } // Object form of $contains - currently only supports relation-style @@ -317,13 +329,14 @@ function applyContainsRelation(options: { ); } - const matchQuery = knex('search') - .select('search.entity_id') - .where({ key: `relations.${type.toLocaleLowerCase('en-US')}` }); + const subquery = searchExists(knex, onEntityIdField).where( + `${S}.key`, + `relations.${type.toLocaleLowerCase('en-US')}`, + ); if (targetRef) { - matchQuery.whereIn('value', targetRef); + subquery.whereIn(`${S}.value`, targetRef); } - return targetQuery.andWhere(onEntityIdField, 'in', matchQuery); + return targetQuery.whereExists(subquery); }