From 4829e8961e158cbabcd3622afad9d418e1b6ac28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 20 May 2026 15:48:22 +0200 Subject: [PATCH 1/5] fix(catalog-backend): split queryEntities list and count queries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filtered CTE was referenced twice (count + data), preventing Postgres 12+ from inlining it. This forced full materialization of the filtered set before LIMIT could short-circuit. Split into two separate queries run via Promise.all so the list CTE is only referenced once and the planner can short-circuit on LIMIT. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .changeset/split-query-entities-count.md | 5 ++ .../src/service/DefaultEntitiesCatalog.ts | 79 ++++++++++++++----- 2 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 .changeset/split-query-entities-count.md diff --git a/.changeset/split-query-entities-count.md b/.changeset/split-query-entities-count.md new file mode 100644 index 0000000000..de5cb40e43 --- /dev/null +++ b/.changeset/split-query-entities-count.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Split the `queryEntities` list and count into separate queries instead of a multi-reference CTE. When the `filtered` CTE was referenced twice (once for the count, once for the data), PostgreSQL refused to inline it, forcing full materialization of the filtered set before applying `LIMIT`. By running the count as a standalone query, the list CTE is only referenced once, allowing the planner to short-circuit on `LIMIT` and return the first page in milliseconds instead of waiting for the full filtered set to materialize. diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 547d276eb8..d45986cc4b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -484,22 +484,9 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }, ); - // Only pay the cost of counting the number of items if needed - if (shouldComputeTotalItems) { - // Note the intentional cross join here. The filtered_count dataset is - // always exactly one row, so it won't grow the result unnecessarily. But - // it's also important that there IS at least one row, because even if the - // filtered dataset is empty, we still want to know the total number of - // items. - dbQuery - .with('filtered_count', ['count'], inner => - inner.from('filtered').count('*', { as: 'count' }), - ) - .fromRaw('filtered_count, filtered') - .select('count', 'filtered.*'); - } else { - dbQuery.from('filtered').select('*'); - } + // The list query always references the 'filtered' CTE exactly once, + // allowing Postgres 12+ to inline it and short-circuit on LIMIT. + dbQuery.from('filtered').select('*'); const isOrderingDescending = sortField?.order === 'desc'; @@ -585,15 +572,69 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { // fetch an extra item to check if there are more items. dbQuery.limit(isFetchingBackwards ? limit : limit + 1); - const rows = shouldComputeTotalItems || limit > 0 ? await dbQuery : []; + // Build a standalone count query that reproduces the same filtering as + // the 'filtered' CTE. Running it separately ensures the list CTE is + // only referenced once, letting Postgres 12+ inline it and + // short-circuit on LIMIT. + let countQuery: Knex.QueryBuilder | undefined; + if (shouldComputeTotalItems) { + countQuery = this.database('final_entities') + .whereNotNull('final_entities.final_entity') + .count('*', { as: 'count' }); + + if (sortField) { + countQuery.whereExists( + this.database('search') + .select(this.database.raw(1)) + .whereRaw('search.entity_id = final_entities.entity_id') + .where('search.key', sortField.field), + ); + } + + if (cursor.filter || cursor.query) { + applyEntityFilterToQuery({ + filter: cursor.filter, + query: cursor.query, + targetQuery: countQuery, + onEntityIdField: 'final_entities.entity_id', + knex: this.database, + }); + } + + const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim(); + if (normalizedFullTextFilterTerm) { + const textFilterFields = cursor.fullTextFilter?.fields ?? [ + sortField?.field || 'metadata.uid', + ]; + const matchQuery = this.database('search') + .select('search.entity_id') + .whereIn( + 'search.key', + textFilterFields.map(field => field.toLocaleLowerCase('en-US')), + ) + .andWhere(function keyFilter() { + this.andWhereRaw( + 'search.value like ?', + `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, + ); + }); + countQuery.andWhere('final_entities.entity_id', 'in', matchQuery); + } + } + + // Run list and count queries concurrently + const [rows, countResult] = await Promise.all([ + limit > 0 ? dbQuery : Promise.resolve([]), + countQuery ?? Promise.resolve(undefined), + ]); let totalItems: number; if (cursor.totalItems !== undefined) { totalItems = cursor.totalItems; } else if (cursor.skipTotalItems) { totalItems = 0; - } else if (rows.length) { - totalItems = Number(rows[0].count); + } else if (countResult?.[0]) { + totalItems = Number(countResult[0].count); } else { totalItems = 0; } From 0119806204a2390562a1753c9641632e6179635b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 20 May 2026 16:29:17 +0200 Subject: [PATCH 2/5] Update changeset to document totalItems semantic fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .changeset/split-query-entities-count.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.changeset/split-query-entities-count.md b/.changeset/split-query-entities-count.md index de5cb40e43..4a0ac5492d 100644 --- a/.changeset/split-query-entities-count.md +++ b/.changeset/split-query-entities-count.md @@ -3,3 +3,5 @@ --- Split the `queryEntities` list and count into separate queries instead of a multi-reference CTE. When the `filtered` CTE was referenced twice (once for the count, once for the data), PostgreSQL refused to inline it, forcing full materialization of the filtered set before applying `LIMIT`. By running the count as a standalone query, the list CTE is only referenced once, allowing the planner to short-circuit on `LIMIT` and return the first page in milliseconds instead of waiting for the full filtered set to materialize. + +The standalone count query also fixes a pre-existing bug where `totalItems` was inflated for entities with multi-valued sort fields (e.g. tags). The old CTE-based count counted search rows, so an entity with 3 tags would be counted 3 times. The new count uses `EXISTS` to count distinct entities, aligning `totalItems` with the number of entities actually reachable through cursor pagination. From 59fef3e80b09480d1ac86f4cf0895cb806d76ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 20 May 2026 16:31:30 +0200 Subject: [PATCH 3/5] Add patch entry for #34323, remove shipped patches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove pr-33721, pr-34001, pr-34004 — all shipped in v1.51.0. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .patches/pr-33721.txt | 1 - .patches/pr-34001.txt | 1 - .patches/pr-34004.txt | 1 - .patches/pr-34323.txt | 1 + 4 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 .patches/pr-33721.txt delete mode 100644 .patches/pr-34001.txt delete mode 100644 .patches/pr-34004.txt create mode 100644 .patches/pr-34323.txt diff --git a/.patches/pr-33721.txt b/.patches/pr-33721.txt deleted file mode 100644 index a4c71542c3..0000000000 --- a/.patches/pr-33721.txt +++ /dev/null @@ -1 +0,0 @@ -Fix home page widgets not being draggable or resizable after the first save diff --git a/.patches/pr-34001.txt b/.patches/pr-34001.txt deleted file mode 100644 index 760a059e5f..0000000000 --- a/.patches/pr-34001.txt +++ /dev/null @@ -1 +0,0 @@ -Fix facets endpoint performance regression when filters or permissions are applied diff --git a/.patches/pr-34004.txt b/.patches/pr-34004.txt deleted file mode 100644 index 506fd76855..0000000000 --- a/.patches/pr-34004.txt +++ /dev/null @@ -1 +0,0 @@ -Preserve external hrefs in BUI link components under non-root base path \ No newline at end of file diff --git a/.patches/pr-34323.txt b/.patches/pr-34323.txt new file mode 100644 index 0000000000..8435da76c5 --- /dev/null +++ b/.patches/pr-34323.txt @@ -0,0 +1 @@ +Split queryEntities list and count queries to fix CTE materialization bottleneck From 02a6410992e0418c37ce8020568bc76e13cf2245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 20 May 2026 16:35:41 +0200 Subject: [PATCH 4/5] Extract shared applyPredicates helper to deduplicate filter logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list CTE and count query both need the same filter, query, and full-text-search predicates. Extract them into a shared applyPredicates closure so they can't drift apart. Move the count query construction next to the CTE for readability. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .../src/service/DefaultEntitiesCatalog.ts | 186 ++++++++---------- 1 file changed, 80 insertions(+), 106 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index d45986cc4b..e194d492f0 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -398,13 +398,63 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const sortField = cursor.orderFields.at(0); - // The first part of the query builder is a subquery that applies all of the - // filtering. When a sort field is specified, the search table for that key - // drives the query via INNER JOIN so that the (key, value, entity_id) - // index walks rows in sort order, letting LIMIT short-circuit. Entities - // that lack the sort field are excluded from both the result set and the - // count — this is a deliberate choice that aligns totalItems with the - // number of entities actually reachable through cursor pagination. + const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim(); + const textFilterFields = cursor.fullTextFilter?.fields ?? [ + sortField?.field || 'metadata.uid', + ]; + + // Shared predicate logic applied to both the list CTE and the + // standalone count query so they stay in sync. The `searchInScope` + // flag indicates whether a `search` table is already joined in the + // target query (true for the list CTE when a sort field is set), + // enabling a fast-path LIKE on the already-joined row. + const applyPredicates = ( + q: Knex.QueryBuilder, + options?: { searchInScope?: boolean }, + ) => { + if (cursor.filter || cursor.query) { + applyEntityFilterToQuery({ + filter: cursor.filter, + query: cursor.query, + targetQuery: q, + onEntityIdField: 'final_entities.entity_id', + knex: this.database, + }); + } + + if (normalizedFullTextFilterTerm) { + if ( + options?.searchInScope && + textFilterFields.length === 1 && + textFilterFields[0] === sortField?.field + ) { + q.andWhereRaw( + 'search.value like ?', + `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, + ); + } else { + const matchQuery = this.database('search') + .select('search.entity_id') + .whereIn( + 'search.key', + textFilterFields.map(field => field.toLocaleLowerCase('en-US')), + ) + .andWhere(function keyFilter() { + this.andWhereRaw( + 'search.value like ?', + `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, + ); + }); + q.andWhere('final_entities.entity_id', 'in', matchQuery); + } + } + }; + + // The list CTE. When a sort field is specified, the search table for + // that key drives the query via INNER JOIN so that the covering index + // walks rows in sort order, letting LIMIT short-circuit. Entities + // that lack the sort field are excluded — this aligns totalItems with + // the set reachable through cursor pagination. const dbQuery = this.database.with( 'filtered', ['entity_id', 'final_entity', ...(sortField ? ['value'] : [])], @@ -434,60 +484,34 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }); } - // Add regular filters and/or predicate query, if given - if (cursor.filter || cursor.query) { - applyEntityFilterToQuery({ - filter: cursor.filter, - query: cursor.query, - targetQuery: inner, - onEntityIdField: 'final_entities.entity_id', - knex: this.database, - }); - } - - // Add full text search filters, if given - const normalizedFullTextFilterTerm = - cursor.fullTextFilter?.term?.trim(); - const textFilterFields = cursor.fullTextFilter?.fields ?? [ - sortField?.field || 'metadata.uid', - ]; - if (normalizedFullTextFilterTerm) { - if ( - textFilterFields.length === 1 && - textFilterFields[0] === sortField?.field - ) { - // If there is one item, apply the like query to the top level query which is already - // filtered based on the singular sortField. - inner.andWhereRaw( - 'search.value like ?', - `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, - ); - } else { - const matchQuery = this.database('search') - .select('search.entity_id') - // textFilterFields must be lowercased to match searchable keys in database, i.e. spec.profile.displayName -> spec.profile.displayname - .whereIn( - 'search.key', - textFilterFields.map(field => field.toLocaleLowerCase('en-US')), - ) - .andWhere(function keyFilter() { - this.andWhereRaw( - 'search.value like ?', - `%${normalizedFullTextFilterTerm.toLocaleLowerCase( - 'en-US', - )}%`, - ); - }); - inner.andWhere('final_entities.entity_id', 'in', matchQuery); - } - } + applyPredicates(inner, { searchInScope: !!sortField }); }, ); - // The list query always references the 'filtered' CTE exactly once, - // allowing Postgres 12+ to inline it and short-circuit on LIMIT. + // The list query references the CTE exactly once, allowing Postgres + // 12+ to inline it and short-circuit on LIMIT. dbQuery.from('filtered').select('*'); + // Standalone count query — runs concurrently with the list so the + // CTE stays single-referenced and inlineable. + let countQuery: Knex.QueryBuilder | undefined; + if (shouldComputeTotalItems) { + countQuery = this.database('final_entities') + .whereNotNull('final_entities.final_entity') + .count('*', { as: 'count' }); + + if (sortField) { + countQuery.whereExists( + this.database('search') + .select(this.database.raw(1)) + .whereRaw('search.entity_id = final_entities.entity_id') + .where('search.key', sortField.field), + ); + } + + applyPredicates(countQuery); + } + const isOrderingDescending = sortField?.order === 'desc'; // Move forward (or backward) in the set to the correct cursor position @@ -572,56 +596,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { // fetch an extra item to check if there are more items. dbQuery.limit(isFetchingBackwards ? limit : limit + 1); - // Build a standalone count query that reproduces the same filtering as - // the 'filtered' CTE. Running it separately ensures the list CTE is - // only referenced once, letting Postgres 12+ inline it and - // short-circuit on LIMIT. - let countQuery: Knex.QueryBuilder | undefined; - if (shouldComputeTotalItems) { - countQuery = this.database('final_entities') - .whereNotNull('final_entities.final_entity') - .count('*', { as: 'count' }); - - if (sortField) { - countQuery.whereExists( - this.database('search') - .select(this.database.raw(1)) - .whereRaw('search.entity_id = final_entities.entity_id') - .where('search.key', sortField.field), - ); - } - - if (cursor.filter || cursor.query) { - applyEntityFilterToQuery({ - filter: cursor.filter, - query: cursor.query, - targetQuery: countQuery, - onEntityIdField: 'final_entities.entity_id', - knex: this.database, - }); - } - - const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim(); - if (normalizedFullTextFilterTerm) { - const textFilterFields = cursor.fullTextFilter?.fields ?? [ - sortField?.field || 'metadata.uid', - ]; - const matchQuery = this.database('search') - .select('search.entity_id') - .whereIn( - 'search.key', - textFilterFields.map(field => field.toLocaleLowerCase('en-US')), - ) - .andWhere(function keyFilter() { - this.andWhereRaw( - 'search.value like ?', - `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, - ); - }); - countQuery.andWhere('final_entities.entity_id', 'in', matchQuery); - } - } - // Run list and count queries concurrently const [rows, countResult] = await Promise.all([ limit > 0 ? dbQuery : Promise.resolve([]), From 2c4b4912c2ed6ab5466176d1409df09ea6fceee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 20 May 2026 16:59:39 +0200 Subject: [PATCH 5/5] Add whereNotNull to count query sort-field EXISTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list CTE's INNER JOIN on search already excludes entities whose sort-field value is NULL (truncated long values stored as NULL by buildEntitySearch). Add the same whereNotNull to the count query's EXISTS so totalItems matches the reachable set. Matters for fields like metadata.description where ~128K entities have NULL values. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index e194d492f0..2a3c7aad9d 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -505,7 +505,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { this.database('search') .select(this.database.raw(1)) .whereRaw('search.entity_id = final_entities.entity_id') - .where('search.key', sortField.field), + .where('search.key', sortField.field) + .whereNotNull('search.value'), ); }