From 8f20cc2f5265ce36016b72ef07a4326cbc0ed575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 21 May 2026 11:52:37 +0200 Subject: [PATCH 1/3] Add totalItems mode, sort key lowercasing, and simplified ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer the remaining concepts from the original PR on top of the split-query foundation: - totalItems parameter on REST API (GET and POST) and CatalogApi - TotalItemsMode type replacing internal skipTotalItems boolean - Sort field key lowercasing for search table comparison - whereNotNull(search.value) on the list CTE to exclude truncated values - Simplified ORDER BY (NULLS LAST removed since NULL values are already excluded by whereNotNull) - New tests for NULL sort field values and multi-valued sort field totalItems accuracy Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .changeset/catalog-backend-totalitems-mode.md | 9 ++ .changeset/catalog-client-totalitems-mode.md | 5 + packages/catalog-client/report.api.md | 1 + packages/catalog-client/src/CatalogClient.ts | 8 ++ .../openapi/generated/apis/Api.client.ts | 4 +- .../QueryEntitiesByPredicateRequest.model.ts | 11 ++ packages/catalog-client/src/types/api.ts | 9 ++ plugins/catalog-backend/src/catalog/types.ts | 15 ++- .../catalog-backend/src/schema/openapi.yaml | 26 ++++ .../openapi/generated/apis/Api.server.ts | 1 + .../QueryEntitiesByPredicateRequest.model.ts | 11 ++ .../src/schema/openapi/generated/router.ts | 21 +++ .../service/DefaultEntitiesCatalog.test.ts | 123 +++++++++++++++++- .../src/service/DefaultEntitiesCatalog.ts | 87 ++++--------- .../src/service/createRouter.test.ts | 2 +- .../src/service/createRouter.ts | 2 +- .../service/request/parseEntityQuery.test.ts | 21 +++ .../src/service/request/parseEntityQuery.ts | 24 +++- .../request/parseQueryEntitiesParams.test.ts | 5 + .../request/parseQueryEntitiesParams.ts | 23 ++++ 20 files changed, 339 insertions(+), 69 deletions(-) create mode 100644 .changeset/catalog-backend-totalitems-mode.md create mode 100644 .changeset/catalog-client-totalitems-mode.md diff --git a/.changeset/catalog-backend-totalitems-mode.md b/.changeset/catalog-backend-totalitems-mode.md new file mode 100644 index 0000000000..3a355f8d85 --- /dev/null +++ b/.changeset/catalog-backend-totalitems-mode.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +`/entities/by-query` now accepts a `totalItems` parameter (`'include'` or `'exclude'`, default `'include'`) that controls whether the response's `totalItems` count is computed. Pass `'exclude'` to skip the count entirely when the caller doesn't need it — useful for cursor-paginated user interfaces that only display the count cosmetically. The accepted values list is forward-compatible: future modes (e.g. approximate counts) can be added without breaking existing callers. + +The internal `queryEntities` implementation has also been refactored to run the list and count queries concurrently via `Promise.all`. The list query is now a single statement that the planner can drive with `LIMIT` short-circuiting (no longer wrapped in a multi-reference CTE that forced materialization). Together with `totalItems: 'exclude'` this materially improves the wall-clock time of paginated catalog list views — particularly cursor-paginated user interfaces and second-page-and-onwards traffic where the count is already cached on the cursor. + +The internal `QueryEntitiesInitialRequest.skipTotalItems` option has been replaced by `totalItems: 'include' | 'exclude'`. Note that `skipTotalItems` was never exposed as a REST API parameter, so this is only a TypeScript-level change affecting direct callers of `EntitiesCatalog.queryEntities`. diff --git a/.changeset/catalog-client-totalitems-mode.md b/.changeset/catalog-client-totalitems-mode.md new file mode 100644 index 0000000000..f1ff5349e8 --- /dev/null +++ b/.changeset/catalog-client-totalitems-mode.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +`CatalogApi.queryEntities` now accepts a `totalItems` option (`'include'` or `'exclude'`, default `'include'`) on initial requests. Pass `'exclude'` to skip the `totalItems` count when the caller doesn't need it. diff --git a/packages/catalog-client/report.api.md b/packages/catalog-client/report.api.md index c54d765a3e..9e815f4f23 100644 --- a/packages/catalog-client/report.api.md +++ b/packages/catalog-client/report.api.md @@ -346,6 +346,7 @@ export type QueryEntitiesInitialRequest = { term: string; fields?: string[]; }; + totalItems?: 'include' | 'exclude'; }; // @public diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 0b93b777f2..3799ec1884 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -326,6 +326,7 @@ export class CatalogClient implements CatalogApi { offset, orderFields, fullTextFilter, + totalItems, } = request; params.filter = this.getFilterValue(filter); @@ -343,6 +344,9 @@ export class CatalogClient implements CatalogApi { if (fields.length) { params.fields = fields; } + if (totalItems !== undefined) { + params.totalItems = totalItems; + } const normalizedFullTextFilterTerm = fullTextFilter?.term?.trim(); if (normalizedFullTextFilterTerm) { @@ -387,6 +391,7 @@ export class CatalogClient implements CatalogApi { orderFields, fullTextFilter, fields, + totalItems, } = request; let filterPredicate: FilterPredicate | undefined; @@ -425,6 +430,9 @@ export class CatalogClient implements CatalogApi { if (fields?.length) { body.fields = fields; } + if (totalItems !== undefined) { + body.totalItems = totalItems; + } } else { body.cursor = request.cursor; if (request.limit !== undefined) { diff --git a/packages/catalog-client/src/schema/openapi/generated/apis/Api.client.ts b/packages/catalog-client/src/schema/openapi/generated/apis/Api.client.ts index 279ba21a76..196815f948 100644 --- a/packages/catalog-client/src/schema/openapi/generated/apis/Api.client.ts +++ b/packages/catalog-client/src/schema/openapi/generated/apis/Api.client.ts @@ -92,6 +92,7 @@ export type GetEntitiesByQuery = { orderField?: Array; cursor?: string; filter?: Array; + totalItems?: 'include' | 'exclude'; fullTextFilterTerm?: string; fullTextFilterFields?: Array; }; @@ -308,6 +309,7 @@ export class DefaultApiClient { * @param orderField - By default the entities are returned ordered by their internal uid. You can customize the `orderField` query parameters to affect that ordering. For example, to return entities by their name: `/entities/by-query?orderField=metadata.name,asc` Each parameter can be followed by `asc` for ascending lexicographical order or `desc` for descending (reverse) lexicographical order. * @param cursor - You may pass the `cursor` query parameters to perform cursor based pagination through the set of entities. The value of `cursor` will be returned in the response, under the `pageInfo` property: ```json \"pageInfo\": { \"nextCursor\": \"a-cursor\", \"prevCursor\": \"another-cursor\" } ``` If `nextCursor` exists, it can be used to retrieve the next batch of entities. Following the same approach, if `prevCursor` exists, it can be used to retrieve the previous batch of entities. - [`filter`](#filtering), for selecting only a subset of all entities - [`fields`](#field-selection), for selecting only parts of the full data structure of each entity - `limit` for limiting the number of entities returned (20 is the default) - [`orderField`](#ordering), for deciding the order of the entities - `fullTextFilter` **NOTE**: [`filter`, `orderField`, `fullTextFilter`] and `cursor` are mutually exclusive. This means that, it isn\'t possible to change any of [`filter`, `orderField`, `fullTextFilter`] when passing `cursor` as query parameters, as changing any of these properties will affect pagination. If any of `filter`, `orderField`, `fullTextFilter` is specified together with `cursor`, only the latter is taken into consideration. * @param filter - You can pass in one or more filter sets that get matched against each entity. Each filter set is a number of conditions that all have to match for the condition to be true (conditions effectively have an AND between them). At least one filter set has to be true for the entity to be part of the result set (filter sets effectively have an OR between them). Example: ```text /entities/by-query?filter=kind=user,metadata.namespace=default&filter=kind=group,spec.type Return entities that match Filter set 1: Condition 1: kind = user AND Condition 2: metadata.namespace = default OR Filter set 2: Condition 1: kind = group AND Condition 2: spec.type exists ``` Each condition is either on the form `<key>`, or on the form `<key>=<value>`. The first form asserts on the existence of a certain key (with any value), and the second asserts that the key exists and has a certain value. All checks are always case _insensitive_. In all cases, the key is a simplified JSON path in a given piece of entity data. Each part of the path is a key of an object, and the traversal also descends through arrays. There are two special forms: - Array items that are simple value types (such as strings) match on a key-value pair where the key is the item as a string, and the value is the string `true` - Relations can be matched on a `relations.<type>=<targetRef>` form Let\'s look at a simplified example to illustrate the concept: ```json { \"a\": { \"b\": [\"c\", { \"d\": 1 }], \"e\": 7 } } ``` This would match any one of the following conditions: - `a` - `a.b` - `a.b.c` - `a.b.c=true` - `a.b.d` - `a.b.d=1` - `a.e` - `a.e=7` Some more real world usable examples: - Return all orphaned entities: `/entities/by-query?filter=metadata.annotations.backstage.io/orphan=true` - Return all users and groups: `/entities/by-query?filter=kind=user&filter=kind=group` - Return all service components: `/entities/by-query?filter=kind=component,spec.type=service` - Return all entities with the `java` tag: `/entities/by-query?filter=metadata.tags.java` - Return all users who are members of the `ops` group (note that the full [reference](references.md) of the group is used): `/entities/by-query?filter=kind=user,relations.memberof=group:default/ops` + * @param totalItems - Controls whether the response\'s `totalItems` field is computed. Computing the total may be expensive for large catalogs; pass `exclude` if the caller does not need it (e.g. cursor-paginated UIs that only display the count cosmetically). Defaults to `include`. New values may be added in the future, such as an approximate mode. * @param fullTextFilterTerm - Text search term. * @param fullTextFilterFields - A comma separated list of fields to sort returned results by. */ @@ -318,7 +320,7 @@ export class DefaultApiClient { ): Promise> { const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); - const uriTemplate = `/entities/by-query{?fields,limit,offset,orderField*,cursor,filter*,fullTextFilterTerm,fullTextFilterFields}`; + const uriTemplate = `/entities/by-query{?fields,limit,offset,orderField*,cursor,filter*,totalItems,fullTextFilterTerm,fullTextFilterFields}`; const uri = parser.parse(uriTemplate).expand({ ...request.query, diff --git a/packages/catalog-client/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts b/packages/catalog-client/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts index 7196156f55..71e33601f6 100644 --- a/packages/catalog-client/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts +++ b/packages/catalog-client/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts @@ -30,8 +30,19 @@ export interface QueryEntitiesByPredicateRequest { orderBy?: Array; fullTextFilter?: QueryEntitiesByPredicateRequestFullTextFilter; fields?: Array; + /** + * Controls whether the response\'s `totalItems` field is computed. Pass `exclude` to skip the count when the caller doesn\'t need it. Defaults to `include`. + */ + totalItems?: QueryEntitiesByPredicateRequestTotalItemsEnum; /** * A type representing all allowed JSON object values. */ query?: { [key: string]: any }; } + +/** + * @public + */ +export type QueryEntitiesByPredicateRequestTotalItemsEnum = + | 'include' + | 'exclude'; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 06e6902197..7db505494c 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -488,6 +488,15 @@ export type QueryEntitiesInitialRequest = { term: string; fields?: string[]; }; + /** + * Controls whether the response's `totalItems` field is computed. + * + * `'include'` (default) — compute it. `'exclude'` — skip the count entirely; + * the response `totalItems` will be `0`. Useful for cursor-paginated UIs + * that only display the count cosmetically. Additional values may be added + * in the future. + */ + totalItems?: 'include' | 'exclude'; }; /** diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 3d6aff55ba..32734e0d9b 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -230,9 +230,22 @@ export interface QueryEntitiesInitialRequest { term: string; fields?: string[]; }; - skipTotalItems?: boolean; + /** + * Controls whether the response's `totalItems` is computed. + * + * `'include'` (default) — compute it. `'exclude'` — skip the count query + * entirely; the response `totalItems` will be `0`. Additional modes (e.g. + * approximate counts) may be added in the future. + */ + totalItems?: TotalItemsMode; } +/** + * Controls whether {@link EntitiesCatalog.queryEntities} computes the + * `totalItems` field on the response. + */ +export type TotalItemsMode = 'include' | 'exclude'; + /** * Request for {@link EntitiesCatalog.queryEntities} used to * move forward or backward on the data. diff --git a/plugins/catalog-backend/src/schema/openapi.yaml b/plugins/catalog-backend/src/schema/openapi.yaml index 0b86a75bfa..16ee0e367b 100644 --- a/plugins/catalog-backend/src/schema/openapi.yaml +++ b/plugins/catalog-backend/src/schema/openapi.yaml @@ -285,6 +285,22 @@ components: Order descending by owner: value: - spec.owner,desc + totalItems: + name: totalItems + in: query + description: | + Controls whether the response's `totalItems` field is computed. Computing + the total may be expensive for large catalogs; pass `exclude` if the + caller does not need it (e.g. cursor-paginated UIs that only display the + count cosmetically). Defaults to `include`. New values may be added in + the future, such as an approximate mode. + required: false + allowReserved: true + schema: + type: string + enum: + - include + - exclude requestBodies: {} responses: ErrorResponse: @@ -1084,6 +1100,7 @@ paths: - $ref: '#/components/parameters/orderField' - $ref: '#/components/parameters/cursor' - $ref: '#/components/parameters/filter' + - $ref: '#/components/parameters/totalItems' - name: fullTextFilterTerm in: query description: Text search term. @@ -1162,6 +1179,15 @@ paths: type: array items: type: string + totalItems: + type: string + enum: + - include + - exclude + description: | + Controls whether the response's `totalItems` field is + computed. Pass `exclude` to skip the count when the caller + doesn't need it. Defaults to `include`. query: $ref: '#/components/schemas/JsonObject' /entity-facets: diff --git a/plugins/catalog-backend/src/schema/openapi/generated/apis/Api.server.ts b/plugins/catalog-backend/src/schema/openapi/generated/apis/Api.server.ts index 9e67e075c6..09931b3cfe 100644 --- a/plugins/catalog-backend/src/schema/openapi/generated/apis/Api.server.ts +++ b/plugins/catalog-backend/src/schema/openapi/generated/apis/Api.server.ts @@ -65,6 +65,7 @@ export type GetEntitiesByQuery = { orderField?: Array; cursor?: string; filter?: Array; + totalItems?: 'include' | 'exclude'; fullTextFilterTerm?: string; fullTextFilterFields?: Array; }; diff --git a/plugins/catalog-backend/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts b/plugins/catalog-backend/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts index 7196156f55..71e33601f6 100644 --- a/plugins/catalog-backend/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts +++ b/plugins/catalog-backend/src/schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model.ts @@ -30,8 +30,19 @@ export interface QueryEntitiesByPredicateRequest { orderBy?: Array; fullTextFilter?: QueryEntitiesByPredicateRequestFullTextFilter; fields?: Array; + /** + * Controls whether the response\'s `totalItems` field is computed. Pass `exclude` to skip the count when the caller doesn\'t need it. Defaults to `include`. + */ + totalItems?: QueryEntitiesByPredicateRequestTotalItemsEnum; /** * A type representing all allowed JSON object values. */ query?: { [key: string]: any }; } + +/** + * @public + */ +export type QueryEntitiesByPredicateRequestTotalItemsEnum = + | 'include' + | 'exclude'; diff --git a/plugins/catalog-backend/src/schema/openapi/generated/router.ts b/plugins/catalog-backend/src/schema/openapi/generated/router.ts index 05ec8450c1..9eeb654486 100644 --- a/plugins/catalog-backend/src/schema/openapi/generated/router.ts +++ b/plugins/catalog-backend/src/schema/openapi/generated/router.ts @@ -199,6 +199,18 @@ export const spec = { }, }, }, + totalItems: { + name: 'totalItems', + in: 'query', + description: + "Controls whether the response's `totalItems` field is computed. Computing\nthe total may be expensive for large catalogs; pass `exclude` if the\ncaller does not need it (e.g. cursor-paginated UIs that only display the\ncount cosmetically). Defaults to `include`. New values may be added in\nthe future, such as an approximate mode.\n", + required: false, + allowReserved: true, + schema: { + type: 'string', + enum: ['include', 'exclude'], + }, + }, }, requestBodies: {}, responses: { @@ -1224,6 +1236,9 @@ export const spec = { { $ref: '#/components/parameters/filter', }, + { + $ref: '#/components/parameters/totalItems', + }, { name: 'fullTextFilterTerm', in: 'query', @@ -1332,6 +1347,12 @@ export const spec = { type: 'string', }, }, + totalItems: { + type: 'string', + enum: ['include', 'exclude'], + description: + "Controls whether the response's `totalItems` field is\ncomputed. Pass `exclude` to skip the count when the caller\ndoesn't need it. Defaults to `include`.\n", + }, query: { $ref: '#/components/schemas/JsonObject', }, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 3836a1d30a..01e8855ccb 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1678,7 +1678,7 @@ describe.each(databases.eachSupportedId())( const request: QueryEntitiesInitialRequest = { limit: 10, credentials: mockCredentials.none(), - skipTotalItems: true, + totalItems: 'exclude', }; let response = await catalog.queryEntities(request); expect(response).toEqual({ @@ -2177,6 +2177,127 @@ describe.each(databases.eachSupportedId())( entityFrom('A', { kind: 'component' }), ]); }); + + it('should exclude entities with NULL sort-field values from all pages', async () => { + await createDatabase(); + + // n1, n2, n3 have spec.b with real values + // n4 has spec.b with a value exceeding MAX_VALUE_LENGTH (200 chars), + // which buildEntitySearch stores as value=NULL in the search table + // n5 has no spec.b at all + // + // When sorting by spec.b, queryEntities should exclude both n4 + // (NULL value from truncation) and n5 (missing key) from the + // result set AND the totalItems count, so that cursor pagination + // covers exactly the reachable set with no unreachable entities. + await addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n1', uid: 'uid-n1' }, + spec: { b: 'alpha', should_include_this: 'yes' }, + }); + await addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n2', uid: 'uid-n2' }, + spec: { b: 'beta', should_include_this: 'yes' }, + }); + await addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n3', uid: 'uid-n3' }, + spec: { b: 'gamma', should_include_this: 'yes' }, + }); + await addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n4', uid: 'uid-n4' }, + spec: { b: 'x'.repeat(201), should_include_this: 'yes' }, + }); + await addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n5', uid: 'uid-n5' }, + spec: { should_include_this: 'yes' }, + }); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: mockServices.logger.mock(), + stitcher, + }); + + const filter = { key: 'spec.should_include_this' }; + + // Page through all entities with limit=2, sorting by spec.b ASC. + // We expect to see n1(alpha), n2(beta), n3(gamma) — and NOT n4 or n5. + const response1 = await catalog.queryEntities({ + filter, + limit: 2, + orderFields: [{ field: 'spec.b', order: 'asc' }], + credentials: mockCredentials.none(), + }); + const page1 = entitiesResponseToObjects(response1.items).map( + e => e!.metadata.name, + ); + expect(page1).toEqual(['n1', 'n2']); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.totalItems).toBe(3); + + // Page 2 via cursor + const response2 = await catalog.queryEntities({ + cursor: response1.pageInfo.nextCursor!, + limit: 2, + credentials: mockCredentials.none(), + }); + const page2 = entitiesResponseToObjects(response2.items).map( + e => e!.metadata.name, + ); + expect(page2).toEqual(['n3']); + expect(response2.pageInfo.nextCursor).toBeUndefined(); + + // Verify: all entities across all pages = n1, n2, n3 (no n4, no n5) + expect([...page1, ...page2]).toEqual(['n1', 'n2', 'n3']); + }); + + it('should not inflate totalItems when a sort field has multiple search rows per entity', async () => { + await createDatabase(); + + // Entity e1 has TWO search rows for spec.tags: 'java' and 'go'. + // When sorting by spec.tags, the list query may return e1 twice + // (one row per tag value), but totalItems should still count e1 + // only once — not inflate the count. + const e1 = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'e1', uid: 'uid-e1', tags: ['java', 'go'] }, + spec: {}, + }; + const e2 = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'e2', uid: 'uid-e2', tags: ['rust'] }, + spec: {}, + }; + + await addEntityToSearch(e1); + await addEntityToSearch(e2); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: mockServices.logger.mock(), + stitcher, + }); + + const response = await catalog.queryEntities({ + orderFields: [{ field: 'metadata.tags', order: 'asc' }], + limit: 100, + credentials: mockCredentials.none(), + }); + + // totalItems counts distinct entities, not search rows + expect(response.totalItems).toBe(2); + }); }); describe('removeEntityByUid', () => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 2a3c7aad9d..c82a01166a 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -32,6 +32,7 @@ import { EntityPagination, QueryEntitiesRequest, QueryEntitiesResponse, + TotalItemsMode, } from '../catalog/types'; import { DbFinalEntitiesRow, @@ -375,21 +376,17 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ): Promise { const limit = request.limit ?? DEFAULT_LIMIT; - const cursor: Omit & { - orderFieldValues?: (string | null)[]; - skipTotalItems: boolean; - } = { - orderFields: [], + const { totalItemsMode, ...cursor } = { + orderFields: [] as EntityOrder[], isPrevious: false, ...parseCursorFromRequest(request), + } satisfies Omit & { + orderFieldValues?: (string | null)[]; + totalItemsMode: TotalItemsMode; }; - // For performance reasons we invoke the count query only on the first - // request. The result is then embedded into the cursor for subsequent - // requests. Therefore this can be undefined here, but will then get - // populated further down. const shouldComputeTotalItems = - cursor.totalItems === undefined && !cursor.skipTotalItems; + cursor.totalItems === undefined && totalItemsMode !== 'exclude'; const isFetchingBackwards = cursor.isPrevious; if (cursor.orderFields.length > 1) { @@ -397,10 +394,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } const sortField = cursor.orderFields.at(0); + const sortKey = sortField?.field.toLocaleLowerCase('en-US'); const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim(); const textFilterFields = cursor.fullTextFilter?.fields ?? [ - sortField?.field || 'metadata.uid', + sortKey || 'metadata.uid', ]; // Shared predicate logic applied to both the list CTE and the @@ -425,8 +423,9 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { if (normalizedFullTextFilterTerm) { if ( options?.searchInScope && + sortField && textFilterFields.length === 1 && - textFilterFields[0] === sortField?.field + textFilterFields[0] === sortKey ) { q.andWhereRaw( 'search.value like ?', @@ -467,7 +466,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { 'final_entities.entity_id', 'search.entity_id', ) - .where('search.key', sortField.field) + .where('search.key', sortKey!) + .whereNotNull('search.value') .whereNotNull('final_entities.final_entity') .select({ entity_id: 'final_entities.entity_id', @@ -505,7 +505,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { this.database('search') .select(this.database.raw(1)) .whereRaw('search.entity_id = final_entities.entity_id') - .where('search.key', sortField.field) + .where('search.key', sortKey!) .whereNotNull('search.value'), ); } @@ -540,52 +540,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } } - // Add the ordering let order = sortField?.order ?? 'asc'; if (isFetchingBackwards) { order = invertOrder(order); } - if (this.database.client.config.client === 'pg') { - // pg correctly orders by the column value and handling nulls in one go - dbQuery.orderBy([ - ...(sortField - ? [ - { - column: 'filtered.value', - order, - nulls: 'last', - }, - ] - : []), - { - column: 'filtered.entity_id', - order, - }, - ]); - } else { - // sqlite and mysql translate the above statement ONLY into "order by (value is null) asc" - // no matter what the order is, for some reason, so we have to manually add back the statement - // that translates to "order by value " while avoiding to give an order - dbQuery.orderBy([ - ...(sortField - ? [ - { - column: 'filtered.value', - order: undefined, - nulls: 'last', - }, - { - column: 'filtered.value', - order, - }, - ] - : []), - { - column: 'filtered.entity_id', - order, - }, - ]); - } + dbQuery.orderBy([ + ...(sortField ? [{ column: 'filtered.value', order }] : []), + { column: 'filtered.entity_id', order }, + ]); // Apply a manually set initial offset if ( @@ -606,7 +568,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { let totalItems: number; if (cursor.totalItems !== undefined) { totalItems = cursor.totalItems; - } else if (cursor.skipTotalItems) { + } else if (totalItemsMode === 'exclude') { totalItems = 0; } else if (countResult?.[0]) { totalItems = Number(countResult[0].count); @@ -888,32 +850,31 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { function parseCursorFromRequest( request?: QueryEntitiesRequest, -): Partial & { skipTotalItems: boolean } { +): Partial & { totalItemsMode: TotalItemsMode } { if (isQueryEntitiesInitialRequest(request)) { const { filter, query, orderFields: sortFields = [], fullTextFilter, - skipTotalItems = false, + totalItems: totalItemsMode = 'include', } = request; return { filter, query, orderFields: sortFields, fullTextFilter, - skipTotalItems, + totalItemsMode, }; } if (isQueryEntitiesCursorRequest(request)) { return { ...request.cursor, - // Doesn't matter here - skipTotalItems: false, + totalItemsMode: 'include', }; } return { - skipTotalItems: false, + totalItemsMode: 'include', }; } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index ebd5daaf3f..776af3d642 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -210,7 +210,7 @@ describe('createRouter readonly disabled', () => { }, limit: 10000, credentials: mockCredentials.user(), - skipTotalItems: true, + totalItems: 'exclude', }); }); diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 020e395976..2312efdde9 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -221,7 +221,7 @@ export async function createRouter( limit, filter, orderFields: order, - skipTotalItems: true, + totalItems: 'exclude', } : { credentials, fields, limit, cursor }, ); diff --git a/plugins/catalog-backend/src/service/request/parseEntityQuery.test.ts b/plugins/catalog-backend/src/service/request/parseEntityQuery.test.ts index 7380259188..552936913a 100644 --- a/plugins/catalog-backend/src/service/request/parseEntityQuery.test.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityQuery.test.ts @@ -152,6 +152,27 @@ describe('parseEntityQuery', () => { }), ).toThrow(/Invalid order field order/); }); + + it('parses totalItems include / exclude', () => { + expect(parseEntityQuery({ totalItems: 'exclude' })).toEqual( + expect.objectContaining({ totalItems: 'exclude' }), + ); + expect(parseEntityQuery({ totalItems: 'include' })).toEqual( + expect.objectContaining({ totalItems: 'include' }), + ); + }); + + it('omits totalItems when not provided', () => { + const result = parseEntityQuery({}); + expect(result).not.toHaveProperty('totalItems'); + }); + + it('throws on invalid totalItems value', () => { + expect(() => + // @ts-expect-error - invalid enum value + parseEntityQuery({ totalItems: 'maybe' }), + ).toThrow(/Invalid totalItems value/); + }); }); describe('cursor request', () => { diff --git a/plugins/catalog-backend/src/service/request/parseEntityQuery.ts b/plugins/catalog-backend/src/service/request/parseEntityQuery.ts index 27992c273a..b861a1205a 100644 --- a/plugins/catalog-backend/src/service/request/parseEntityQuery.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityQuery.ts @@ -22,10 +22,29 @@ import { import { z } from 'zod/v3'; import { fromZodError } from 'zod-validation-error/v3'; import { QueryEntitiesByPredicateRequest } from '../../schema/openapi/generated/models/QueryEntitiesByPredicateRequest.model'; -import { EntityOrder } from '../../catalog/types'; +import { EntityOrder, TotalItemsMode } from '../../catalog/types'; import { Cursor } from '../../catalog/types'; import { decodeCursor } from '../util'; +const TOTAL_ITEMS_MODES: TotalItemsMode[] = ['include', 'exclude']; + +function parseTotalItems(value: unknown): TotalItemsMode | undefined { + if (value === undefined) { + return undefined; + } + if ( + typeof value !== 'string' || + !TOTAL_ITEMS_MODES.includes(value as TotalItemsMode) + ) { + throw new InputError( + `Invalid totalItems value "${value}", expected one of: ${TOTAL_ITEMS_MODES.join( + ', ', + )}`, + ); + } + return value as TotalItemsMode; +} + const filterPredicateSchema = createZodV3FilterPredicateSchema(z); function isSupportedFilterPredicateRoot( @@ -67,6 +86,7 @@ export type ParsedEntityQuery = fields?: string[]; limit?: number; offset?: number; + totalItems?: TotalItemsMode; }; export function parseEntityQuery( @@ -98,6 +118,7 @@ export function parseEntityQuery( } const orderFields = parseOrderFields(request.orderBy); + const totalItemsMode = parseTotalItems(request.totalItems); return { query, @@ -111,5 +132,6 @@ export function parseEntityQuery( fields: request.fields, limit: request.limit, offset: request.offset, + ...(totalItemsMode !== undefined && { totalItems: totalItemsMode }), }; } diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts index e11656681a..b5b91a7117 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -33,6 +33,7 @@ describe('parseQueryEntitiesParams', () => { orderField: ['metadata.name,desc'], fullTextFilterTerm: 'query', fullTextFilterFields: ['metadata.name', 'metadata.namespace'], + totalItems: 'exclude' as const, }; const parsedObj = parseQueryEntitiesParams( validRequest, @@ -46,6 +47,7 @@ describe('parseQueryEntitiesParams', () => { term: 'query', fields: ['metadata.name', 'metadata.namespace'], }); + expect(parsedObj.totalItems).toBe('exclude'); expect(parsedObj).not.toHaveProperty('authorizationToken'); expect(parsedObj).not.toHaveProperty('cursor'); }); @@ -57,6 +59,7 @@ describe('parseQueryEntitiesParams', () => { expect(parsedObj.orderFields).toBeUndefined(); expect(parsedObj.filter).toBeUndefined(); expect(parsedObj.fullTextFilter).toEqual({ term: '', fields: undefined }); + expect(parsedObj.totalItems).toBeUndefined(); expect(parsedObj).not.toHaveProperty('authorizationToken'); expect(parsedObj).not.toHaveProperty('cursor'); }); @@ -65,6 +68,8 @@ describe('parseQueryEntitiesParams', () => { { filter: 3 }, { orderField: ['metadata.uid,diagonal'] }, { fields: [4] }, + { totalItems: 'maybe' }, + { totalItems: 42 }, ])('should throw if some parameter is not valid %p', (params: any) => { expect(() => parseQueryEntitiesParams(params)).toThrow(); }); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts index aef28c0b32..92c4250b1c 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -14,10 +14,12 @@ * limitations under the License. */ +import { InputError } from '@backstage/errors'; import { QueryEntitiesCursorRequest, QueryEntitiesInitialRequest, QueryEntitiesRequest, + TotalItemsMode, } from '../../catalog/types'; import { decodeCursor } from '../util'; import { parseEntityFilterParams } from './parseEntityFilterParams'; @@ -25,6 +27,25 @@ import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; import { parseEntityTransformParams } from './parseEntityTransformParams'; import { GetEntitiesByQuery } from '../../schema/openapi'; +const TOTAL_ITEMS_MODES: TotalItemsMode[] = ['include', 'exclude']; + +function parseTotalItems(value: unknown): TotalItemsMode | undefined { + if (value === undefined) { + return undefined; + } + if ( + typeof value !== 'string' || + !TOTAL_ITEMS_MODES.includes(value as TotalItemsMode) + ) { + throw new InputError( + `Invalid totalItems parameter "${value}", expected one of: ${TOTAL_ITEMS_MODES.join( + ', ', + )}`, + ); + } + return value as TotalItemsMode; +} + export function parseQueryEntitiesParams( params: GetEntitiesByQuery['query'], ): Omit { @@ -41,6 +62,7 @@ export function parseQueryEntitiesParams( const filter = parseEntityFilterParams(params); const orderFields = parseEntityOrderFieldParams(params); + const totalItemsMode = parseTotalItems(params.totalItems); const response: Omit = { fields, @@ -50,6 +72,7 @@ export function parseQueryEntitiesParams( term: params.fullTextFilterTerm || '', fields: params.fullTextFilterFields, }, + ...(totalItemsMode !== undefined && { totalItems: totalItemsMode }), }; return response; From 4cc7231379f69562008d839708d00f30ab72a78e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 21 May 2026 12:11:28 +0200 Subject: [PATCH 2/3] Update changeset to reflect delta on top of #34323 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- .changeset/catalog-backend-totalitems-mode.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/catalog-backend-totalitems-mode.md b/.changeset/catalog-backend-totalitems-mode.md index 3a355f8d85..4bad06f040 100644 --- a/.changeset/catalog-backend-totalitems-mode.md +++ b/.changeset/catalog-backend-totalitems-mode.md @@ -4,6 +4,6 @@ `/entities/by-query` now accepts a `totalItems` parameter (`'include'` or `'exclude'`, default `'include'`) that controls whether the response's `totalItems` count is computed. Pass `'exclude'` to skip the count entirely when the caller doesn't need it — useful for cursor-paginated user interfaces that only display the count cosmetically. The accepted values list is forward-compatible: future modes (e.g. approximate counts) can be added without breaking existing callers. -The internal `queryEntities` implementation has also been refactored to run the list and count queries concurrently via `Promise.all`. The list query is now a single statement that the planner can drive with `LIMIT` short-circuiting (no longer wrapped in a multi-reference CTE that forced materialization). Together with `totalItems: 'exclude'` this materially improves the wall-clock time of paginated catalog list views — particularly cursor-paginated user interfaces and second-page-and-onwards traffic where the count is already cached on the cursor. - The internal `QueryEntitiesInitialRequest.skipTotalItems` option has been replaced by `totalItems: 'include' | 'exclude'`. Note that `skipTotalItems` was never exposed as a REST API parameter, so this is only a TypeScript-level change affecting direct callers of `EntitiesCatalog.queryEntities`. + +Sort field keys are now lowercased before comparing against `search.key`, fixing silent mismatches for camelCase field names. The `NULLS LAST` ordering clause has been removed since NULL sort values are already excluded by the `WHERE` clause. From 2139bf53d1e511bf9ad0125276c0eb3c47dd9b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 21 May 2026 16:38:37 +0200 Subject: [PATCH 3/3] Set cursor totalItemsMode to exclude with explanatory comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor requests already carry the computed totalItems from page 1, so the count query is skipped regardless of this value. Setting it to 'exclude' reflects what actually happens, and the comment explains why the value doesn't matter. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index c82a01166a..4f98d68e6e 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -870,7 +870,9 @@ function parseCursorFromRequest( if (isQueryEntitiesCursorRequest(request)) { return { ...request.cursor, - totalItemsMode: 'include', + // Doesn't matter — cursor already carries the computed totalItems + // number from the first page, so the count query is skipped regardless. + totalItemsMode: 'exclude', }; } return {