From 31c4fe0c1754da62f2e1ece559c7e2300ff19b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 16 Oct 2024 09:52:03 +0200 Subject: [PATCH] chunk up refs in getEntitiesByRefs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/young-penguins-act.md | 5 + packages/catalog-client/src/CatalogClient.ts | 46 ++++--- packages/catalog-client/src/utils.test.ts | 124 +++++++++++++++++++ packages/catalog-client/src/utils.ts | 64 ++++++++++ 4 files changed, 219 insertions(+), 20 deletions(-) create mode 100644 .changeset/young-penguins-act.md create mode 100644 packages/catalog-client/src/utils.test.ts diff --git a/.changeset/young-penguins-act.md b/.changeset/young-penguins-act.md new file mode 100644 index 0000000000..bae2fe089f --- /dev/null +++ b/.changeset/young-penguins-act.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +The client now automatically splits up very large `getEntitiesByRefs` calls into several smaller requests behind the scenes when needed. This ensures that each individual request does not exceed common Express.js request body limits or overload the server. diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 4de314c1e8..ff3429f81b 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -41,7 +41,7 @@ import { QueryEntitiesResponse, ValidateEntityResponse, } from './types/api'; -import { isQueryEntitiesInitialRequest } from './utils'; +import { isQueryEntitiesInitialRequest, splitRefsIntoChunks } from './utils'; import { DefaultApiClient, TypedResponse } from './generated'; /** @@ -151,28 +151,34 @@ export class CatalogClient implements CatalogApi { request: GetEntitiesByRefsRequest, options?: CatalogRequestOptions, ): Promise { - const response = await this.apiClient.getEntitiesByRefs( - { - body: { - entityRefs: request.entityRefs, - fields: request.fields, + const getOneChunk = async (refs: string[]) => { + const response = await this.apiClient.getEntitiesByRefs( + { + body: { entityRefs: refs, fields: request.fields }, + query: { filter: this.getFilterValue(request.filter) }, }, - query: { - filter: this.getFilterValue(request.filter), - }, - }, - options, - ); - - if (!response.ok) { - throw await ResponseError.fromResponse(response); - } - - const { items } = (await response.json()) as { - items: Array; + options, + ); + if (!response.ok) { + throw await ResponseError.fromResponse(response); + } + const body = (await response.json()) as { + items: Array; + }; + return body.items.map(i => i ?? undefined); }; - return { items: items.map(i => i ?? undefined) }; + let result: Array | undefined; + for (const refs of splitRefsIntoChunks(request.entityRefs)) { + const entities = await getOneChunk(refs); + if (!result) { + result = entities; + } else { + result.push(...entities); + } + } + + return { items: result ?? [] }; } /** diff --git a/packages/catalog-client/src/utils.test.ts b/packages/catalog-client/src/utils.test.ts new file mode 100644 index 0000000000..2f00859985 --- /dev/null +++ b/packages/catalog-client/src/utils.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { splitRefsIntoChunks } from './utils'; + +describe('splitRefsIntoChunks', () => { + it('splits by count limit', () => { + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 0, + }), + ).toEqual([['a'], ['b'], ['c'], ['d']]); + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 1, + }), + ).toEqual([['a'], ['b'], ['c'], ['d']]); + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 2, + }), + ).toEqual([ + ['a', 'b'], + ['c', 'd'], + ]); + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 3, + }), + ).toEqual([['a', 'b', 'c'], ['d']]); + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 4, + }), + ).toEqual([['a', 'b', 'c', 'd']]); + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 5, + }), + ).toEqual([['a', 'b', 'c', 'd']]); + expect( + splitRefsIntoChunks(['a', 'b', 'c', 'd'], { + maxCountPerChunk: 5, + maxStringLengthPerChunk: 3, // the stricter limit now + extraStringLengthPerRef: 0, + }), + ).toEqual([['a', 'b', 'c'], ['d']]); + }); + + it('splits by length limit', () => { + expect( + splitRefsIntoChunks(['aa', 'b', 'c'], { + maxStringLengthPerChunk: 0, + extraStringLengthPerRef: 0, + }), + ).toEqual([['aa'], ['b'], ['c']]); + expect( + splitRefsIntoChunks(['aa', 'b', 'c'], { + maxStringLengthPerChunk: 1, + extraStringLengthPerRef: 0, + }), + ).toEqual([['aa'], ['b'], ['c']]); + expect( + splitRefsIntoChunks(['aa', 'b', 'c'], { + maxStringLengthPerChunk: 2, + extraStringLengthPerRef: 0, + }), + ).toEqual([['aa'], ['b', 'c']]); + expect( + splitRefsIntoChunks(['aa', 'b', 'c'], { + maxStringLengthPerChunk: 3, + extraStringLengthPerRef: 0, + }), + ).toEqual([['aa', 'b'], ['c']]); + expect( + splitRefsIntoChunks(['aa', 'b', 'c'], { + maxStringLengthPerChunk: 3, + extraStringLengthPerRef: 0, + maxCountPerChunk: 1, // the stricter limit now + }), + ).toEqual([['aa'], ['b'], ['c']]); + }); + + it('splits while the extra length is taken into account', () => { + expect( + splitRefsIntoChunks(['aaa', 'bbb', 'ccc'], { + maxStringLengthPerChunk: 9, + extraStringLengthPerRef: 0, + }), + ).toEqual([['aaa', 'bbb', 'ccc']]); + expect( + splitRefsIntoChunks(['aaa', 'bbb', 'ccc'], { + maxStringLengthPerChunk: 9, + extraStringLengthPerRef: 1, + }), + ).toEqual([['aaa', 'bbb'], ['ccc']]); + expect( + splitRefsIntoChunks(['aaa', 'bbb', 'ccc'], { + maxStringLengthPerChunk: 9, + extraStringLengthPerRef: 2, + }), + ).toEqual([['aaa'], ['bbb'], ['ccc']]); + expect( + splitRefsIntoChunks(['aaa', 'bbb', 'ccc'], { + maxStringLengthPerChunk: 9, + extraStringLengthPerRef: 0, + maxCountPerChunk: 2, // the stricter limit now + }), + ).toEqual([['aaa', 'bbb'], ['ccc']]); + }); +}); diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts index 9d51c34e71..63afec668a 100644 --- a/packages/catalog-client/src/utils.ts +++ b/packages/catalog-client/src/utils.ts @@ -24,3 +24,67 @@ export function isQueryEntitiesInitialRequest( ): request is QueryEntitiesInitialRequest { return !(request as QueryEntitiesCursorRequest).cursor; } + +/** + * Takes a set of entity refs, and splits them into chunks (groups) such that + * the total string length in each chunk does not exceed the default Express.js + * request body limit of 100 kB (with some margin) when JSON encoded as an + * array. + */ +export function splitRefsIntoChunks( + refs: string[], + options?: { + // No chunk has more than this many refs, no matter what + maxCountPerChunk?: number; + // The total string length (taking the extraStringLengthPerRef into account) + // of each chunk never exceeds this many characters, no matter what + maxStringLengthPerChunk?: number; + // Add this many characters to the length of each ref when calculating + // (default is 3, since eacn array entry is surrounded by quotes and a + // comma) + extraStringLengthPerRef?: number; + }, +): string[][] { + if (!refs.length) { + return []; + } + + const { + maxCountPerChunk = 1000, + maxStringLengthPerChunk = 90 * 2 ** 10, + extraStringLengthPerRef = 3, + } = options ?? {}; + + const chunks: string[][] = []; + + let currentChunkStart = 0; + let currentChunkStringLength = 0; + let currentChunkSize = 0; + + for (let i = 0; i < refs.length; ++i) { + const refLength = refs[i].length + extraStringLengthPerRef; + + // always allow at least one element per chunk even in abnormal situations + if (currentChunkSize > 0) { + // emit chunk and start over if either the string length or the count + // limit would be reached + if ( + currentChunkStringLength + refLength > maxStringLengthPerChunk || + currentChunkSize + 1 > maxCountPerChunk + ) { + chunks.push(refs.slice(currentChunkStart, i)); + currentChunkStart = i; + currentChunkStringLength = 0; + currentChunkSize = 0; + } + } + + currentChunkStringLength += refLength; + currentChunkSize += 1; + } + + // emit whatever is left as the last chunk + chunks.push(refs.slice(currentChunkStart, refs.length)); + + return chunks; +}