From ebeebd9012886f139409fb37de50997b0c45826d Mon Sep 17 00:00:00 2001 From: Nilay1999 Date: Thu, 22 Jan 2026 22:39:07 +0530 Subject: [PATCH] refactor(client): :recycle: update queryEntities to support query payload - add query support to do advanced search using queryEntities - update core query of queryEntitiesByPredicate to handle cursor based pagination Signed-off-by: Nilay1999 --- .../catalog-client/src/CatalogClient.test.ts | 305 ++++++++++++++++++ packages/catalog-client/src/CatalogClient.ts | 96 +++++- packages/catalog-client/src/utils.ts | 13 + plugins/catalog-backend/src/catalog/types.ts | 12 +- .../src/service/DefaultEntitiesCatalog.ts | 138 ++++++-- plugins/catalog-backend/src/service/util.ts | 2 +- 6 files changed, 530 insertions(+), 36 deletions(-) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index d62323852e..31c5d62046 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -540,6 +540,311 @@ describe('CatalogClient', () => { }); }); + describe('queryEntities with predicate-based queries (POST endpoint)', () => { + const defaultResponse = { + items: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'service-1', + namespace: 'default', + }, + spec: { + type: 'service', + owner: 'team-a', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'service-2', + namespace: 'default', + }, + spec: { + type: 'service', + owner: 'team-b', + }, + }, + ], + pageInfo: {}, + }; + + it('should use POST endpoint when query is provided', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.method).toBe('POST'); + expect(req.body).toMatchObject({ + query: { + kind: 'component', + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + const response = await client.queryEntities({ + query: { kind: 'component' }, + limit: 20, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + expect(response.items).toEqual(defaultResponse.items); + expect(response.totalItems).toBe(defaultResponse.items.length); + }); + + it('should throw error when both filter and query are provided', async () => { + await expect( + client.queryEntities({ + filter: { kind: 'component' }, + query: { kind: 'component' }, + } as any), + ).rejects.toThrow( + 'Cannot specify both "filter" and "query" in the same request', + ); + }); + + it('should support $all operator', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.body).toMatchObject({ + query: { + $all: [{ kind: 'component' }, { 'spec.type': 'service' }], + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { + $all: [{ kind: 'component' }, { 'spec.type': 'service' }], + }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should support $any operator', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.body).toMatchObject({ + query: { + $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }], + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { + $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }], + }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should support $not operator', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.body).toMatchObject({ + query: { + $not: { 'spec.lifecycle': 'experimental' }, + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { + $not: { 'spec.lifecycle': 'experimental' }, + }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should support $exists operator', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.body).toMatchObject({ + query: { + 'spec.owner': { $exists: true }, + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { + 'spec.owner': { $exists: true }, + }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should support $in operator', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.body).toMatchObject({ + query: { + 'spec.owner': { $in: ['team-a', 'team-b', 'team-c'] }, + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { + 'spec.owner': { $in: ['team-a', 'team-b', 'team-c'] }, + }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should support complex nested predicates', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + expect(req.body).toMatchObject({ + query: { + $all: [ + { kind: 'component' }, + { + $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }], + }, + { + $not: { + 'spec.lifecycle': 'experimental', + }, + }, + ], + }, + }); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { + $all: [ + { kind: 'component' }, + { + $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }], + }, + { + $not: { + 'spec.lifecycle': 'experimental', + }, + }, + ], + }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should send orderFields with correct format (field,order)', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + const url = new URL(req.url); + expect(url.searchParams.getAll('orderField')).toEqual([ + 'metadata.name,asc', + ]); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { kind: 'component' }, + orderFields: { field: 'metadata.name', order: 'asc' }, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should send multiple orderFields with correct format', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + const url = new URL(req.url); + expect(url.searchParams.getAll('orderField')).toEqual([ + 'metadata.name,asc', + 'spec.type,desc', + ]); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { kind: 'component' }, + orderFields: [ + { field: 'metadata.name', order: 'asc' }, + { field: 'spec.type', order: 'desc' }, + ], + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should send limit and offset parameters', async () => { + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + const url = new URL(req.url); + expect(url.searchParams.get('limit')).toBe('50'); + expect(url.searchParams.get('offset')).toBe('10'); + return res(ctx.json(defaultResponse)); + }); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + query: { kind: 'component' }, + limit: 50, + offset: 10, + }); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should not allow cursor with query (cursor takes precedence)', async () => { + // When cursor is provided, it's not an initial request, so the query + // parameter is ignored and it goes to GET endpoint + const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => { + // Should use GET endpoint, not POST + expect(req.method).toBe('GET'); + return res(ctx.json({ items: [], totalItems: 0, pageInfo: {} })); + }); + + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + // This will use GET endpoint with cursor, ignoring the query parameter + await client.queryEntities({ + cursor: 'some-cursor', + query: { kind: 'component' }, + } as any); + + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + }); + + it('should handle errors from POST endpoint', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => res(ctx.status(400))); + + server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await expect(() => + client.queryEntities({ query: { kind: 'component' } }), + ).rejects.toThrow(/Request failed with 400/); + }); + }); + describe('streamEntities', () => { const defaultResponse: QueryEntitiesResponse = { items: [ diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index a537e8b3a5..2d7fa7d957 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -46,7 +46,11 @@ import { StreamEntitiesRequest, ValidateEntityResponse, } from './types/api'; -import { isQueryEntitiesInitialRequest, splitRefsIntoChunks } from './utils'; +import { + isQueryEntitiesInitialRequest, + splitRefsIntoChunks, + cursorContainsQuery, +} from './utils'; import { DefaultApiClient, GetLocationsByQueryRequest, @@ -266,6 +270,30 @@ export class CatalogClient implements CatalogApi { request: QueryEntitiesRequest = {}, options?: CatalogRequestOptions, ): Promise { + // Validate that filter and query are mutually exclusive + if ( + isQueryEntitiesInitialRequest(request) && + request.filter && + request.query + ) { + throw new Error( + 'Cannot specify both "filter" and "query" in the same request. Use "filter" for traditional key-value filtering or "query" for predicate-based filtering.', + ); + } + + // Route to POST endpoint if query predicate is provided (initial request) + if (isQueryEntitiesInitialRequest(request) && request.query) { + return this.queryEntitiesByPredicate(request, options); + } + + // Route to POST endpoint if cursor contains a query predicate (pagination) + if ( + !isQueryEntitiesInitialRequest(request) && + cursorContainsQuery(request.cursor) + ) { + return this.queryEntitiesByPredicate(request, options); + } + const params: Partial< Parameters[0]['query'] > = {}; @@ -320,6 +348,72 @@ export class CatalogClient implements CatalogApi { ); } + /** + * Query entities using predicate-based filters (POST endpoint). + * @internal + */ + private async queryEntitiesByPredicate( + request: QueryEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise { + const params: { + limit?: number; + offset?: number; + after?: string; + orderField?: string[]; + cursor?: string; + fields?: string[]; + } = {}; + + let query; + + if (isQueryEntitiesInitialRequest(request)) { + // Initial request with query predicate + const { query: requestQuery, limit, offset, orderFields } = request; + query = requestQuery; + + if (limit !== undefined) { + params.limit = limit; + } + if (offset !== undefined) { + params.offset = offset; + } + if (orderFields !== undefined) { + params.orderField = ( + Array.isArray(orderFields) ? orderFields : [orderFields] + ).map(({ field, order }) => `${field},${order}`); + } + } else { + // Cursor-based pagination request + const { cursor, limit } = request; + + params.after = cursor; + if (limit !== undefined) { + params.limit = limit; + } + // Query will be extracted from cursor on the backend + query = undefined; + } + + const response = await this.apiClient.queryEntitiesByPredicate( + { + body: query ? { query } : {}, + query: params, + }, + options, + ); + + const result = await this.requestRequired(response); + + return { + items: result.items, + totalItems: result.items.length, + pageInfo: { + nextCursor: result.pageInfo?.nextCursor, + }, + }; + } + /** * {@inheritdoc CatalogApi.getEntityByRef} */ diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts index a16cf53db1..e7713623b9 100644 --- a/packages/catalog-client/src/utils.ts +++ b/packages/catalog-client/src/utils.ts @@ -26,6 +26,19 @@ export function isQueryEntitiesInitialRequest( return !(request as QueryEntitiesCursorRequest).cursor; } +/** + * Check if a cursor contains a predicate query by attempting to decode it. + * @internal + */ +export function cursorContainsQuery(cursor: string): boolean { + try { + const decoded = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8')); + return !!decoded.query; + } catch { + return false; + } +} + /** * Takes a set of entity refs, and splits them into chunks (groups) such that * the total string length in each chunk does not exceed the default Express.js diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 1b87352318..aad176e390 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -53,7 +53,7 @@ export type EntitiesRequest = { }; export type EntityPredicateRequest = { - filter?: EntityPredicate; + query?: EntityPredicate; order?: EntityOrder[]; pagination?: EntityPagination; credentials: BackstageCredentials; @@ -228,6 +228,11 @@ export interface QueryEntitiesInitialRequest { limit?: number; offset?: number; filter?: EntityFilter; + /** + * Predicate-based query for filtering entities. + * Mutually exclusive with filter. + */ + query?: EntityPredicate; orderFields?: EntityOrder[]; fullTextFilter?: { term: string; @@ -289,6 +294,11 @@ export type Cursor = { * A filter to be applied to the full list of entities. */ filter?: EntityFilter; + /** + * A predicate-based query to be applied to the full list of entities. + * Mutually exclusive with filter. + */ + query?: EntityPredicate; /** * true if the cursor is a previous cursor. */ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index c7b14282ef..151e345f5b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -46,6 +46,8 @@ import { import { Stitcher } from '../stitching/types'; import { + decodeCursor, + encodeCursor, expandLegacyCompoundRelationsInEntity, isQueryEntitiesCursorRequest, isQueryEntitiesInitialRequest, @@ -221,70 +223,133 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const db = this.database; const { limit, offset } = parsePagination(request?.pagination); - let entitiesQuery = - db('final_entities').select('final_entities.*'); + const cursor = request?.pagination?.after + ? decodeCursor(request.pagination.after) + : { + query: request?.query, + orderFields: request?.order || [], + isPrevious: false, + orderFieldValues: undefined, + firstSortFieldValues: undefined, + }; - request?.order?.forEach(({ field }, index) => { - const alias = `order_${index}`; - entitiesQuery = entitiesQuery.leftOuterJoin( - { [alias]: 'search' }, - function search(inner) { + // Use query from cursor if not provided in request (pagination case) + const effectiveQuery = request?.query ?? cursor.query; + const sortField = cursor.orderFields?.[0]; + + let entitiesQuery = db('final_entities'); + + // Join with search table if we have a sort field + if (sortField) { + entitiesQuery = entitiesQuery + .distinct() + .leftOuterJoin({ order_0: 'search' }, function search(inner) { inner - .on(`${alias}.entity_id`, 'final_entities.entity_id') - .andOn(`${alias}.key`, db.raw('?', [field])); - }, - ); - }); + .on('order_0.entity_id', 'final_entities.entity_id') + .andOn('order_0.key', db.raw('?', [sortField.field])); + }) + .select({ + entity_id: 'final_entities.entity_id', + final_entity: 'final_entities.final_entity', + value: 'order_0.value', + }); + } else { + entitiesQuery = entitiesQuery.select({ + entity_id: 'final_entities.entity_id', + final_entity: 'final_entities.final_entity', + }); + } entitiesQuery = entitiesQuery.whereNotNull('final_entities.final_entity'); - if (request?.filter) { + // Apply predicate filter from cursor + if (effectiveQuery) { entitiesQuery = applyPredicateEntityFilterToQuery({ - filter: request.filter, + filter: effectiveQuery, targetQuery: entitiesQuery, onEntityIdField: 'final_entities.entity_id', knex: db, }); } - request?.order?.forEach(({ order }, index) => { + // Apply cursor-based pagination (keyset pagination) + if (cursor.orderFieldValues) { + if (cursor.orderFieldValues.length === 2) { + const [sortValue, entityId] = cursor.orderFieldValues; + const isOrderingDescending = sortField?.order === 'desc'; + entitiesQuery = entitiesQuery.andWhere(function nested() { + this.where( + 'order_0.value', + isOrderingDescending ? '<' : '>', + sortValue, + ) + .orWhere('order_0.value', '=', sortValue) + .andWhere('final_entities.entity_id', '>', entityId); + }); + } else if (cursor.orderFieldValues.length === 1) { + const [entityId] = cursor.orderFieldValues; + entitiesQuery = entitiesQuery.andWhere( + 'final_entities.entity_id', + '>', + entityId, + ); + } + } + + if (sortField) { if (db.client.config.client === 'pg') { entitiesQuery = entitiesQuery.orderBy([ - { column: `order_${index}.value`, order, nulls: 'last' }, + { column: 'order_0.value', order: sortField.order, nulls: 'last' }, + { column: 'final_entities.entity_id', order: 'asc' }, ]); } else { entitiesQuery = entitiesQuery.orderBy([ - { column: `order_${index}.value`, order: undefined, nulls: 'last' }, - { column: `order_${index}.value`, order }, + { column: 'order_0.value', order: undefined, nulls: 'last' }, + { column: 'order_0.value', order: sortField.order }, + { column: 'final_entities.entity_id', order: 'asc' }, ]); } - }); - - if (!request?.order) { - entitiesQuery = entitiesQuery.orderBy('final_entities.entity_ref', 'asc'); } else { - entitiesQuery.orderBy('final_entities.entity_id', 'asc'); + entitiesQuery = entitiesQuery.orderBy('final_entities.entity_id', 'asc'); } - if (limit !== undefined) { - entitiesQuery = entitiesQuery.limit(limit + 1); - } - if (offset !== undefined) { + // Apply a manually set initial offset (only when not using cursor pagination) + if (!request?.pagination?.after && offset !== undefined) { entitiesQuery = entitiesQuery.offset(offset); } + const effectiveLimit = limit ?? DEFAULT_LIMIT; + entitiesQuery = entitiesQuery.limit(effectiveLimit + 1); let rows = await entitiesQuery; let pageInfo: DbPageInfo; - if (limit === undefined || rows.length <= limit) { + + if (rows.length <= effectiveLimit) { pageInfo = { hasNextPage: false }; } else { + // Remove the extra row rows = rows.slice(0, -1); + + const lastRow = rows[rows.length - 1]; + const firstRow = rows[0]; + + // Create proper cursor with query field + const nextCursor: Cursor = { + query: effectiveQuery, + orderFields: cursor.orderFields || [], + orderFieldValues: sortField + ? [(lastRow as any).value, lastRow.entity_id] + : [lastRow.entity_id], + isPrevious: false, + firstSortFieldValues: + cursor.firstSortFieldValues || + (sortField + ? [(firstRow as any).value, firstRow.entity_id] + : [firstRow.entity_id]), + }; + pageInfo = { hasNextPage: true, - endCursor: stringifyPagination({ - limit, - offset: (offset ?? 0) + limit, - }), + endCursor: encodeCursor(nextCursor), }; } @@ -831,11 +896,18 @@ function parseCursorFromRequest( if (isQueryEntitiesInitialRequest(request)) { const { filter, + query, orderFields: sortFields = [], fullTextFilter, skipTotalItems = false, } = request; - return { filter, orderFields: sortFields, fullTextFilter, skipTotalItems }; + return { + filter, + query, + orderFields: sortFields, + fullTextFilter, + skipTotalItems, + }; } if (isQueryEntitiesCursorRequest(request)) { return { diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 7e0e8b42f9..dbdbd374cf 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -122,7 +122,7 @@ export const cursorParser: z.ZodSchema = z.object({ orderFieldValues: z.array(z.string().or(z.null())), filter: entityFilterParser.optional(), isPrevious: z.boolean(), - query: z.string().optional(), + query: z.any().optional(), firstSortFieldValues: z.array(z.string().or(z.null())).optional(), totalItems: z.number().optional(), });