From c1307b4a0d29913dee8079e234c5e02b25fb9021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 28 Nov 2024 21:59:52 +0100 Subject: [PATCH 1/4] implement entities in terms of queryEntities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/fifty-countries-decide.md | 5 + .../src/service/createRouter.test.ts | 26 +-- .../src/service/createRouter.ts | 149 ++++++++++++++++-- 3 files changed, 153 insertions(+), 27 deletions(-) create mode 100644 .changeset/fifty-countries-decide.md diff --git a/.changeset/fifty-countries-decide.md b/.changeset/fifty-countries-decide.md new file mode 100644 index 0000000000..9c4553f3be --- /dev/null +++ b/.changeset/fifty-countries-decide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Implement `/entities` in terms of `queryEntities` to not run into memory and performance problems on large catalogs diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 7cba2edea9..8c4414984e 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -135,9 +135,10 @@ describe('createRouter readonly disabled', () => { { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; - entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [entities[0]], - pageInfo: { hasNextPage: false }, + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items: [entities[0]], + pageInfo: {}, + totalItems: 1, }); const response = await request(app).get('/entities'); @@ -147,17 +148,18 @@ describe('createRouter readonly disabled', () => { }); it('parses single and multiple request parameters and passes them down', async () => { - entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [], - pageInfo: { hasNextPage: false }, + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items: [], + pageInfo: {}, + totalItems: 0, }); const response = await request(app).get( '/entities?filter=a=1,a=2,b=3&filter=c=4', ); expect(response.status).toEqual(200); - expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.entities).toHaveBeenCalledWith({ + expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ filter: { anyOf: [ { @@ -169,6 +171,7 @@ describe('createRouter readonly disabled', () => { { key: 'c', values: ['4'] }, ], }, + limit: 10000, credentials: mockCredentials.user(), }); }); @@ -913,9 +916,10 @@ describe('createRouter readonly enabled', () => { { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; - entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [entities[0]], - pageInfo: { hasNextPage: false }, + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items: [entities[0]], + pageInfo: {}, + totalItems: 1, }); const response = await request(app).get('/entities'); diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 9b62d1318d..25df988bdd 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -19,6 +19,7 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, + parseEntityRef, parseLocationRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -27,7 +28,7 @@ import { InputError, NotFoundError, serializeError } from '@backstage/errors'; import express from 'express'; import yn from 'yn'; import { z } from 'zod'; -import { EntitiesCatalog } from '../catalog/types'; +import { Cursor, EntitiesCatalog } from '../catalog/types'; import { CatalogProcessingOrchestrator } from '../processing/types'; import { validateEntityEnvelope } from '../processing/util'; import { @@ -57,6 +58,7 @@ import { } from '@backstage/backend-plugin-api'; import { LocationAnalyzer } from '@backstage/plugin-catalog-node'; import { AuthorizedValidationService } from './AuthorizedValidationService'; +import { DeferredPromise, createDeferred } from '@backstage/types'; /** * Options used by {@link createRouter}. @@ -135,24 +137,139 @@ export async function createRouter( if (entitiesCatalog) { router .get('/entities', async (req, res) => { - const { entities, pageInfo } = await entitiesCatalog.entities({ - filter: parseEntityFilterParams(req.query), - fields: parseEntityTransformParams(req.query), - order: parseEntityOrderParams(req.query), - pagination: parseEntityPaginationParams(req.query), - credentials: await httpAuth.credentials(req), - }); + const filter = parseEntityFilterParams(req.query); + const fields = parseEntityTransformParams(req.query); + const order = parseEntityOrderParams(req.query); + const pagination = parseEntityPaginationParams(req.query); + const credentials = await httpAuth.credentials(req); - // Add a Link header to the next page - if (pageInfo.hasNextPage) { - const url = new URL(`http://ignored${req.url}`); - url.searchParams.delete('offset'); - url.searchParams.set('after', pageInfo.endCursor); - res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`); + // When pagination parameters are passed in, use the legacy slow path + // that loads all entities into memory + + if (pagination) { + const { entities, pageInfo } = await entitiesCatalog.entities({ + filter, + fields, + order, + pagination, + credentials, + }); + + // Add a Link header to the next page + if (pageInfo.hasNextPage) { + const url = new URL(`http://ignored${req.url}`); + url.searchParams.delete('offset'); + url.searchParams.set('after', pageInfo.endCursor); + res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`); + } + + res.json(entities); + return; } - // TODO(freben): encode the pageInfo in the response - res.json(entities); + // For other read-the-entire-world cases, use queryEntities and stream + // out results. + + // Imitate the httpRouter behavior of pretty-printing in development + const prettyPrint = process.env.NODE_ENV === 'development'; + + // The write lock is used for back pressure, preventing slow readers + // from forcing our read loop to pile up response data in userspace + // buffers faster than the kernel buffer is emptied. + // https://nodejs.org/api/http.html#http_response_write_chunk_encoding_callback + let writeLock: DeferredPromise | undefined; + const controller = new AbortController(); + req.on('end', () => { + controller.abort(); + writeLock?.resolve(); + writeLock = undefined; + }); + res.on('drain', () => { + writeLock?.resolve(); + writeLock = undefined; + }); + + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.status(200); + res.flushHeaders(); + res.write(prettyPrint ? '[\n' : '['); + + const limit = 10000; + let cursor: Cursor | undefined; + let firstSend = true; + do { + const result = await entitiesCatalog.queryEntities( + !cursor + ? { credentials, fields, limit, filter, orderFields: order } + : { credentials, fields, limit, cursor }, + ); + + if (result.items.length) { + // TODO(freben): This is added as a compatibility guarantee, until we can be + // sure that all adopters have re-stitched their entities so that the new + // targetRef field is present on them, and that they have stopped consuming + // the now-removed old field + // TODO(jhaals): Remove this in April 2022 + for (const entity of result.items) { + if (entity.relations) { + for (const relation of entity.relations as any) { + if (!relation.targetRef && relation.target) { + // This is the case where an old-form entity, not yet stitched with + // the updated code, was in the database + relation.targetRef = stringifyEntityRef(relation.target); + } else if (!relation.target && relation.targetRef) { + // This is the case where a new-form entity, stitched with the + // updated code, was in the database but we still want to produce + // the old data shape as well for compatibility reasons + relation.target = parseEntityRef(relation.targetRef); + } + } + } + } + + // Stringify + let data: string; + if (prettyPrint) { + data = JSON.stringify(result.items, null, 2).slice(2, -2); + if (!firstSend) { + data = `,\n${data}`; + } + } else { + data = JSON.stringify(result.items).slice(1, -1); + if (!firstSend) { + data = `,${data}`; + } + } + + if (writeLock) { + await writeLock; + } + + if (controller.signal.aborted) { + res.end(); + return; + } + + if (!res.write(data)) { + // The kernel buffer is full. Create the lock but do not await it + // yet - we can better spend our time going to the next round of + // the loop and read from the database while we wait for it to + // drain. + writeLock = createDeferred(); + } + + firstSend = false; + } + + if (controller.signal.aborted) { + res.end(); + return; + } + + cursor = result.pageInfo?.nextCursor; + } while (cursor); + + res.end(prettyPrint && !firstSend ? '\n]' : ']'); }) .get('/entities/by-query', async (req, res) => { const { items, pageInfo, totalItems } = From 9e2e744531a33893450b9e1900501dcbb7acfde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 2 Dec 2024 13:59:18 +0100 Subject: [PATCH 2/4] review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/service/DefaultEntitiesCatalog.ts | 29 +---- .../src/service/createRouter.ts | 112 ++++++------------ plugins/catalog-backend/src/service/util.ts | 84 ++++++++++++- 3 files changed, 120 insertions(+), 105 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index f7be502f40..2540fc1725 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -import { - Entity, - parseEntityRef, - stringifyEntityRef, -} from '@backstage/catalog-model'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { chunk as lodashChunk, isEqual } from 'lodash'; @@ -49,6 +45,7 @@ import { import { Stitcher } from '../stitching/types'; import { + expandLegacyCompoundRelationRefsInResponse, isQueryEntitiesCursorRequest, isQueryEntitiesInitialRequest, } from './util'; @@ -280,27 +277,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { entities = entities.map(e => request.fields!(e)); } - // TODO(freben): This is added as a compatibility guarantee, until we can be - // sure that all adopters have re-stitched their entities so that the new - // targetRef field is present on them, and that they have stopped consuming - // the now-removed old field - // TODO(jhaals): Remove this in April 2022 - for (const entity of entities) { - if (entity.relations) { - for (const relation of entity.relations as any) { - if (!relation.targetRef && relation.target) { - // This is the case where an old-form entity, not yet stitched with - // the updated code, was in the database - relation.targetRef = stringifyEntityRef(relation.target); - } else if (!relation.target && relation.targetRef) { - // This is the case where a new-form entity, stitched with the - // updated code, was in the database but we still want to produce - // the old data shape as well for compatibility reasons - relation.target = parseEntityRef(relation.targetRef); - } - } - } - } + expandLegacyCompoundRelationRefsInResponse(entities); return { entities, diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 25df988bdd..aa7dd1b755 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -19,7 +19,6 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - parseEntityRef, parseLocationRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -42,8 +41,10 @@ import { parseEntityFacetParams } from './request/parseEntityFacetParams'; import { parseEntityOrderParams } from './request/parseEntityOrderParams'; import { LocationService, RefreshService } from './types'; import { + createEntityArrayJsonStream, disallowReadonlyMode, encodeCursor, + expandLegacyCompoundRelationRefsInResponse, locationInput, validateRequestBody, } from './util'; @@ -170,17 +171,15 @@ export async function createRouter( // For other read-the-entire-world cases, use queryEntities and stream // out results. - // Imitate the httpRouter behavior of pretty-printing in development - const prettyPrint = process.env.NODE_ENV === 'development'; - // The write lock is used for back pressure, preventing slow readers // from forcing our read loop to pile up response data in userspace // buffers faster than the kernel buffer is emptied. // https://nodejs.org/api/http.html#http_response_write_chunk_encoding_callback let writeLock: DeferredPromise | undefined; const controller = new AbortController(); + const signal = controller.signal; req.on('end', () => { - controller.abort(); + controller.abort(new Error('Client closed connection')); writeLock?.resolve(); writeLock = undefined; }); @@ -189,87 +188,44 @@ export async function createRouter( writeLock = undefined; }); - res.setHeader('Content-Type', 'application/json; charset=utf-8'); - res.status(200); - res.flushHeaders(); - res.write(prettyPrint ? '[\n' : '['); - + const responseStream = createEntityArrayJsonStream(res); const limit = 10000; let cursor: Cursor | undefined; - let firstSend = true; - do { - const result = await entitiesCatalog.queryEntities( - !cursor - ? { credentials, fields, limit, filter, orderFields: order } - : { credentials, fields, limit, cursor }, - ); - if (result.items.length) { - // TODO(freben): This is added as a compatibility guarantee, until we can be - // sure that all adopters have re-stitched their entities so that the new - // targetRef field is present on them, and that they have stopped consuming - // the now-removed old field - // TODO(jhaals): Remove this in April 2022 - for (const entity of result.items) { - if (entity.relations) { - for (const relation of entity.relations as any) { - if (!relation.targetRef && relation.target) { - // This is the case where an old-form entity, not yet stitched with - // the updated code, was in the database - relation.targetRef = stringifyEntityRef(relation.target); - } else if (!relation.target && relation.targetRef) { - // This is the case where a new-form entity, stitched with the - // updated code, was in the database but we still want to produce - // the old data shape as well for compatibility reasons - relation.target = parseEntityRef(relation.targetRef); - } - } + try { + do { + const result = await entitiesCatalog.queryEntities( + !cursor + ? { credentials, fields, limit, filter, orderFields: order } + : { credentials, fields, limit, cursor }, + ); + + if (result.items.length) { + if (writeLock) { + await writeLock; + } + + signal.throwIfAborted(); + + expandLegacyCompoundRelationRefsInResponse(result.items); + if (!responseStream.send(result.items)) { + // The kernel buffer is full. Create the lock but do not await it + // yet - we can better spend our time going to the next round of + // the loop and read from the database while we wait for it to + // drain. + writeLock = createDeferred(); } } - // Stringify - let data: string; - if (prettyPrint) { - data = JSON.stringify(result.items, null, 2).slice(2, -2); - if (!firstSend) { - data = `,\n${data}`; - } - } else { - data = JSON.stringify(result.items).slice(1, -1); - if (!firstSend) { - data = `,${data}`; - } - } + signal.throwIfAborted(); - if (writeLock) { - await writeLock; - } + cursor = result.pageInfo?.nextCursor; + } while (cursor); - if (controller.signal.aborted) { - res.end(); - return; - } - - if (!res.write(data)) { - // The kernel buffer is full. Create the lock but do not await it - // yet - we can better spend our time going to the next round of - // the loop and read from the database while we wait for it to - // drain. - writeLock = createDeferred(); - } - - firstSend = false; - } - - if (controller.signal.aborted) { - res.end(); - return; - } - - cursor = result.pageInfo?.nextCursor; - } while (cursor); - - res.end(prettyPrint && !firstSend ? '\n]' : ']'); + responseStream.complete(); + } finally { + responseStream.close(); + } }) .get('/entities/by-query', async (req, res) => { const { items, pageInfo, totalItems } = diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 4b041d1a70..c05fb14558 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -15,7 +15,7 @@ */ import { InputError, NotAllowedError } from '@backstage/errors'; -import { Request } from 'express'; +import { Request, Response } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; import { @@ -25,6 +25,11 @@ import { QueryEntitiesRequest, } from '../catalog/types'; import { EntityFilter } from '@backstage/plugin-catalog-node'; +import { + Entity, + parseEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; export async function requireRequestBody(req: Request): Promise { const contentType = req.header('content-type'); @@ -138,3 +143,80 @@ export function decodeCursor(encodedCursor: string) { throw new InputError(`Malformed cursor: ${e}`); } } + +// TODO(freben): This is added as a compatibility guarantee, until we can be +// sure that all adopters have re-stitched their entities so that the new +// targetRef field is present on them, and that they have stopped consuming +// the now-removed old field +// TODO(jhaals): Remove this in April 2022 +export function expandLegacyCompoundRelationRefsInResponse( + entities: Entity[], +): void { + for (const entity of entities) { + if (entity.relations) { + for (const relation of entity.relations as any) { + if (!relation.targetRef && relation.target) { + // This is the case where an old-form entity, not yet stitched with + // the updated code, was in the database + relation.targetRef = stringifyEntityRef(relation.target); + } else if (!relation.target && relation.targetRef) { + // This is the case where a new-form entity, stitched with the + // updated code, was in the database but we still want to produce + // the old data shape as well for compatibility reasons + relation.target = parseEntityRef(relation.targetRef); + } + } + } + } +} + +export interface EntityArrayJsonStream { + send(entities: Entity[]): boolean; + complete(): void; + close(): void; +} + +// Helps stream Entity[] as a JSON response stream to avoid performance issues +export function createEntityArrayJsonStream( + res: Response, +): EntityArrayJsonStream { + // Imitate the httpRouter behavior of pretty-printing in development + const prettyPrint = process.env.NODE_ENV === 'development'; + let firstSend = true; + let completed = false; + + return { + send(entities) { + if (firstSend) { + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.status(200); + res.flushHeaders(); + } + + let data: string; + if (prettyPrint) { + data = JSON.stringify(entities, null, 2); + data = firstSend ? data.slice(0, -2) : `,\n${data.slice(2, -2)}`; + } else { + data = JSON.stringify(entities); + data = firstSend ? data.slice(0, -1) : `,${data.slice(1, -1)}`; + } + + firstSend = false; + return res.write(data); + }, + complete() { + if (firstSend) { + res.json([]); + } else { + res.end(prettyPrint ? '\n]' : ']'); + } + completed = true; + }, + close() { + if (!completed) { + res.end(); + } + }, + }; +} From 5e5cc8f5b758a0d97305154127a16fe2fcb80d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 4 Dec 2024 11:45:50 +0100 Subject: [PATCH 3/4] drain once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/service/createRouter.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index aa7dd1b755..196685337c 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -175,17 +175,13 @@ export async function createRouter( // from forcing our read loop to pile up response data in userspace // buffers faster than the kernel buffer is emptied. // https://nodejs.org/api/http.html#http_response_write_chunk_encoding_callback - let writeLock: DeferredPromise | undefined; + const locks: { writeLock?: DeferredPromise } = {}; const controller = new AbortController(); const signal = controller.signal; req.on('end', () => { controller.abort(new Error('Client closed connection')); - writeLock?.resolve(); - writeLock = undefined; - }); - res.on('drain', () => { - writeLock?.resolve(); - writeLock = undefined; + locks.writeLock?.resolve(); + delete locks.writeLock; }); const responseStream = createEntityArrayJsonStream(res); @@ -201,9 +197,7 @@ export async function createRouter( ); if (result.items.length) { - if (writeLock) { - await writeLock; - } + await locks?.writeLock; signal.throwIfAborted(); @@ -213,7 +207,11 @@ export async function createRouter( // yet - we can better spend our time going to the next round of // the loop and read from the database while we wait for it to // drain. - writeLock = createDeferred(); + locks.writeLock = createDeferred(); + res.once('drain', () => { + locks.writeLock?.resolve(); + delete locks.writeLock; + }); } } From 6df4a705a7d1cd864a436ece576b8472389c5950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 6 Dec 2024 10:05:28 +0100 Subject: [PATCH 4/4] skip counting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- plugins/catalog-backend/src/catalog/types.ts | 1 + .../service/DefaultEntitiesCatalog.test.ts | 46 +++++++++++++++++++ .../src/service/DefaultEntitiesCatalog.ts | 37 +++++++++++---- .../src/service/createRouter.test.ts | 1 + .../src/service/createRouter.ts | 9 +++- 5 files changed, 85 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index a02c74c0c1..b3e52fb480 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -203,6 +203,7 @@ export interface QueryEntitiesInitialRequest { term: string; fields?: string[]; }; + skipTotalItems?: boolean; } /** diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 418f503739..bb2ba3d587 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1478,6 +1478,52 @@ describe('DefaultEntitiesCatalog', () => { }, ); + it.each(databases.eachSupportedId())( + 'can skip totalItems, %p', + async databaseId => { + await createDatabase(databaseId); + + await Promise.all( + Array(15) + .fill(0) + .map(() => + addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: v4() }, + }), + ), + ); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: mockServices.logger.mock(), + stitcher, + }); + + const request: QueryEntitiesInitialRequest = { + limit: 10, + credentials: mockCredentials.none(), + skipTotalItems: true, + }; + let response = await catalog.queryEntities(request); + expect(response).toEqual({ + totalItems: 0, + items: expect.objectContaining({ length: 10 }), + pageInfo: { nextCursor: expect.anything() }, + }); + response = await catalog.queryEntities({ + ...request, + cursor: response.pageInfo.nextCursor!, + }); + expect(response).toEqual({ + totalItems: 0, + items: expect.objectContaining({ length: 5 }), + pageInfo: { prevCursor: expect.anything() }, + }); + }, + ); + it.each(databases.eachSupportedId())( 'should paginate results accordingly in case of clashing items, %p', async databaseId => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 2540fc1725..fa43eb79d2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -331,6 +331,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const cursor: Omit & { orderFieldValues?: (string | null)[]; + skipTotalItems: boolean; } = { orderFields: [], isPrevious: false, @@ -341,7 +342,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { // request. The result is then embedded into the cursor for subsequent // requests. Threfore this can be undefined here, but will then get // populated further down. - const shouldComputeTotalItems = cursor.totalItems === undefined; + const shouldComputeTotalItems = + cursor.totalItems === undefined && !cursor.skipTotalItems; const isFetchingBackwards = cursor.isPrevious; if (cursor.orderFields.length > 1) { @@ -532,8 +534,16 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const rows = shouldComputeTotalItems || limit > 0 ? await dbQuery : []; - const totalItems = - cursor.totalItems ?? (rows.length ? Number(rows[0].count) : 0); + let totalItems: number; + if (cursor.totalItems !== undefined) { + totalItems = cursor.totalItems; + } else if (cursor.skipTotalItems) { + totalItems = 0; + } else if (rows.length) { + totalItems = Number(rows[0].count); + } else { + totalItems = 0; + } if (isFetchingBackwards) { rows.reverse(); @@ -809,15 +819,26 @@ export const cursorParser: z.ZodSchema = z.object({ function parseCursorFromRequest( request?: QueryEntitiesRequest, -): Partial { +): Partial & { skipTotalItems: boolean } { if (isQueryEntitiesInitialRequest(request)) { - const { filter, orderFields: sortFields = [], fullTextFilter } = request; - return { filter, orderFields: sortFields, fullTextFilter }; + const { + filter, + orderFields: sortFields = [], + fullTextFilter, + skipTotalItems = false, + } = request; + return { filter, orderFields: sortFields, fullTextFilter, skipTotalItems }; } if (isQueryEntitiesCursorRequest(request)) { - return request.cursor; + return { + ...request.cursor, + // Doesn't matter here + skipTotalItems: false, + }; } - return {}; + return { + skipTotalItems: false, + }; } function invertOrder(order: EntityOrder['order']) { diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 8c4414984e..0c1d16b525 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -173,6 +173,7 @@ describe('createRouter readonly disabled', () => { }, limit: 10000, credentials: mockCredentials.user(), + skipTotalItems: true, }); }); }); diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 196685337c..3a6dc82586 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -192,7 +192,14 @@ export async function createRouter( do { const result = await entitiesCatalog.queryEntities( !cursor - ? { credentials, fields, limit, filter, orderFields: order } + ? { + credentials, + fields, + limit, + filter, + orderFields: order, + skipTotalItems: true, + } : { credentials, fields, limit, cursor }, );