refactor(client): ♻️ update queryEntities to support query payload

- add query support to do advanced search using queryEntities
- update core query of queryEntitiesByPredicate to handle cursor based pagination

Signed-off-by: Nilay1999 <nilayparmar19@gmail.com>

packages/catalog-client/src/CatalogClient.test.ts

@@ -540,6 +540,311 @@ describe('CatalogClient', () => {
     });
   });
 
+  describe('queryEntities with predicate-based queries (POST endpoint)', () => {
+    const defaultResponse = {
+      items: [
+        {
+          apiVersion: '1',
+          kind: 'Component',
+          metadata: {
+            name: 'service-1',
+            namespace: 'default',
+          },
+          spec: {
+            type: 'service',
+            owner: 'team-a',
+          },
+        },
+        {
+          apiVersion: '1',
+          kind: 'Component',
+          metadata: {
+            name: 'service-2',
+            namespace: 'default',
+          },
+          spec: {
+            type: 'service',
+            owner: 'team-b',
+          },
+        },
+      ],
+      pageInfo: {},
+    };
+
+    it('should use POST endpoint when query is provided', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.method).toBe('POST');
+        expect(req.body).toMatchObject({
+          query: {
+            kind: 'component',
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      const response = await client.queryEntities({
+        query: { kind: 'component' },
+        limit: 20,
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+      expect(response.items).toEqual(defaultResponse.items);
+      expect(response.totalItems).toBe(defaultResponse.items.length);
+    });
+
+    it('should throw error when both filter and query are provided', async () => {
+      await expect(
+        client.queryEntities({
+          filter: { kind: 'component' },
+          query: { kind: 'component' },
+        } as any),
+      ).rejects.toThrow(
+        'Cannot specify both "filter" and "query" in the same request',
+      );
+    });
+
+    it('should support $all operator', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.body).toMatchObject({
+          query: {
+            $all: [{ kind: 'component' }, { 'spec.type': 'service' }],
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: {
+          $all: [{ kind: 'component' }, { 'spec.type': 'service' }],
+        },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support $any operator', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.body).toMatchObject({
+          query: {
+            $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }],
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: {
+          $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }],
+        },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support $not operator', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.body).toMatchObject({
+          query: {
+            $not: { 'spec.lifecycle': 'experimental' },
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: {
+          $not: { 'spec.lifecycle': 'experimental' },
+        },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support $exists operator', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.body).toMatchObject({
+          query: {
+            'spec.owner': { $exists: true },
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: {
+          'spec.owner': { $exists: true },
+        },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support $in operator', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.body).toMatchObject({
+          query: {
+            'spec.owner': { $in: ['team-a', 'team-b', 'team-c'] },
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: {
+          'spec.owner': { $in: ['team-a', 'team-b', 'team-c'] },
+        },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should support complex nested predicates', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        expect(req.body).toMatchObject({
+          query: {
+            $all: [
+              { kind: 'component' },
+              {
+                $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }],
+              },
+              {
+                $not: {
+                  'spec.lifecycle': 'experimental',
+                },
+              },
+            ],
+          },
+        });
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: {
+          $all: [
+            { kind: 'component' },
+            {
+              $any: [{ 'spec.type': 'service' }, { 'spec.type': 'website' }],
+            },
+            {
+              $not: {
+                'spec.lifecycle': 'experimental',
+              },
+            },
+          ],
+        },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should send orderFields with correct format (field,order)', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        const url = new URL(req.url);
+        expect(url.searchParams.getAll('orderField')).toEqual([
+          'metadata.name,asc',
+        ]);
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: { kind: 'component' },
+        orderFields: { field: 'metadata.name', order: 'asc' },
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should send multiple orderFields with correct format', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        const url = new URL(req.url);
+        expect(url.searchParams.getAll('orderField')).toEqual([
+          'metadata.name,asc',
+          'spec.type,desc',
+        ]);
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: { kind: 'component' },
+        orderFields: [
+          { field: 'metadata.name', order: 'asc' },
+          { field: 'spec.type', order: 'desc' },
+        ],
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should send limit and offset parameters', async () => {
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        const url = new URL(req.url);
+        expect(url.searchParams.get('limit')).toBe('50');
+        expect(url.searchParams.get('offset')).toBe('10');
+        return res(ctx.json(defaultResponse));
+      });
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await client.queryEntities({
+        query: { kind: 'component' },
+        limit: 50,
+        offset: 10,
+      });
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should not allow cursor with query (cursor takes precedence)', async () => {
+      // When cursor is provided, it's not an initial request, so the query
+      // parameter is ignored and it goes to GET endpoint
+      const mockedEndpoint = jest.fn().mockImplementation((req, res, ctx) => {
+        // Should use GET endpoint, not POST
+        expect(req.method).toBe('GET');
+        return res(ctx.json({ items: [], totalItems: 0, pageInfo: {} }));
+      });
+
+      server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      // This will use GET endpoint with cursor, ignoring the query parameter
+      await client.queryEntities({
+        cursor: 'some-cursor',
+        query: { kind: 'component' },
+      } as any);
+
+      expect(mockedEndpoint).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle errors from POST endpoint', async () => {
+      const mockedEndpoint = jest
+        .fn()
+        .mockImplementation((_req, res, ctx) => res(ctx.status(400)));
+
+      server.use(rest.post(`${mockBaseUrl}/entities/by-query`, mockedEndpoint));
+
+      await expect(() =>
+        client.queryEntities({ query: { kind: 'component' } }),
+      ).rejects.toThrow(/Request failed with 400/);
+    });
+  });
+
   describe('streamEntities', () => {
     const defaultResponse: QueryEntitiesResponse = {
       items: [

packages/catalog-client/src/CatalogClient.ts

@@ -46,7 +46,11 @@ import {
   StreamEntitiesRequest,
   ValidateEntityResponse,
 } from './types/api';
-import { isQueryEntitiesInitialRequest, splitRefsIntoChunks } from './utils';
+import {
+  isQueryEntitiesInitialRequest,
+  splitRefsIntoChunks,
+  cursorContainsQuery,
+} from './utils';
 import {
   DefaultApiClient,
   GetLocationsByQueryRequest,
@@ -266,6 +270,30 @@ export class CatalogClient implements CatalogApi {
     request: QueryEntitiesRequest = {},
     options?: CatalogRequestOptions,
   ): Promise<QueryEntitiesResponse> {
+    // Validate that filter and query are mutually exclusive
+    if (
+      isQueryEntitiesInitialRequest(request) &&
+      request.filter &&
+      request.query
+    ) {
+      throw new Error(
+        'Cannot specify both "filter" and "query" in the same request. Use "filter" for traditional key-value filtering or "query" for predicate-based filtering.',
+      );
+    }
+
+    // Route to POST endpoint if query predicate is provided (initial request)
+    if (isQueryEntitiesInitialRequest(request) && request.query) {
+      return this.queryEntitiesByPredicate(request, options);
+    }
+
+    // Route to POST endpoint if cursor contains a query predicate (pagination)
+    if (
+      !isQueryEntitiesInitialRequest(request) &&
+      cursorContainsQuery(request.cursor)
+    ) {
+      return this.queryEntitiesByPredicate(request, options);
+    }
+
     const params: Partial<
       Parameters<typeof this.apiClient.getEntitiesByQuery>[0]['query']
     > = {};
@@ -320,6 +348,72 @@ export class CatalogClient implements CatalogApi {
     );
   }
 
+  /**
+   * Query entities using predicate-based filters (POST endpoint).
+   * @internal
+   */
+  private async queryEntitiesByPredicate(
+    request: QueryEntitiesRequest,
+    options?: CatalogRequestOptions,
+  ): Promise<QueryEntitiesResponse> {
+    const params: {
+      limit?: number;
+      offset?: number;
+      after?: string;
+      orderField?: string[];
+      cursor?: string;
+      fields?: string[];
+    } = {};
+
+    let query;
+
+    if (isQueryEntitiesInitialRequest(request)) {
+      // Initial request with query predicate
+      const { query: requestQuery, limit, offset, orderFields } = request;
+      query = requestQuery;
+
+      if (limit !== undefined) {
+        params.limit = limit;
+      }
+      if (offset !== undefined) {
+        params.offset = offset;
+      }
+      if (orderFields !== undefined) {
+        params.orderField = (
+          Array.isArray(orderFields) ? orderFields : [orderFields]
+        ).map(({ field, order }) => `${field},${order}`);
+      }
+    } else {
+      // Cursor-based pagination request
+      const { cursor, limit } = request;
+
+      params.after = cursor;
+      if (limit !== undefined) {
+        params.limit = limit;
+      }
+      // Query will be extracted from cursor on the backend
+      query = undefined;
+    }
+
+    const response = await this.apiClient.queryEntitiesByPredicate(
+      {
+        body: query ? { query } : {},
+        query: params,
+      },
+      options,
+    );
+
+    const result = await this.requestRequired(response);
+
+    return {
+      items: result.items,
+      totalItems: result.items.length,
+      pageInfo: {
+        nextCursor: result.pageInfo?.nextCursor,
+      },
+    };
+  }
+
   /**
    * {@inheritdoc CatalogApi.getEntityByRef}
    */

packages/catalog-client/src/utils.ts

@@ -26,6 +26,19 @@ export function isQueryEntitiesInitialRequest(
   return !(request as QueryEntitiesCursorRequest).cursor;
 }
 
+/**
+ * Check if a cursor contains a predicate query by attempting to decode it.
+ * @internal
+ */
+export function cursorContainsQuery(cursor: string): boolean {
+  try {
+    const decoded = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8'));
+    return !!decoded.query;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Takes a set of entity refs, and splits them into chunks (groups) such that
  * the total string length in each chunk does not exceed the default Express.js

plugins/catalog-backend/src/catalog/types.ts

@@ -53,7 +53,7 @@ export type EntitiesRequest = {
 };
 
 export type EntityPredicateRequest = {
-  filter?: EntityPredicate;
+  query?: EntityPredicate;
   order?: EntityOrder[];
   pagination?: EntityPagination;
   credentials: BackstageCredentials;
@@ -228,6 +228,11 @@ export interface QueryEntitiesInitialRequest {
   limit?: number;
   offset?: number;
   filter?: EntityFilter;
+  /**
+   * Predicate-based query for filtering entities.
+   * Mutually exclusive with filter.
+   */
+  query?: EntityPredicate;
   orderFields?: EntityOrder[];
   fullTextFilter?: {
     term: string;
@@ -289,6 +294,11 @@ export type Cursor = {
    * A filter to be applied to the full list of entities.
    */
   filter?: EntityFilter;
+  /**
+   * A predicate-based query to be applied to the full list of entities.
+   * Mutually exclusive with filter.
+   */
+  query?: EntityPredicate;
   /**
    * true if the cursor is a previous cursor.
    */

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -46,6 +46,8 @@ import {
 import { Stitcher } from '../stitching/types';
 
 import {
+  decodeCursor,
+  encodeCursor,
   expandLegacyCompoundRelationsInEntity,
   isQueryEntitiesCursorRequest,
   isQueryEntitiesInitialRequest,
@@ -221,70 +223,133 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     const db = this.database;
     const { limit, offset } = parsePagination(request?.pagination);
 
-    let entitiesQuery =
-      db<DbFinalEntitiesRow>('final_entities').select('final_entities.*');
+    const cursor = request?.pagination?.after
+      ? decodeCursor(request.pagination.after)
+      : {
+          query: request?.query,
+          orderFields: request?.order || [],
+          isPrevious: false,
+          orderFieldValues: undefined,
+          firstSortFieldValues: undefined,
+        };
 
-    request?.order?.forEach(({ field }, index) => {
-      const alias = `order_${index}`;
-      entitiesQuery = entitiesQuery.leftOuterJoin(
-        { [alias]: 'search' },
-        function search(inner) {
+    // Use query from cursor if not provided in request (pagination case)
+    const effectiveQuery = request?.query ?? cursor.query;
+    const sortField = cursor.orderFields?.[0];
+
+    let entitiesQuery = db<DbFinalEntitiesRow>('final_entities');
+
+    // Join with search table if we have a sort field
+    if (sortField) {
+      entitiesQuery = entitiesQuery
+        .distinct()
+        .leftOuterJoin({ order_0: 'search' }, function search(inner) {
           inner
-            .on(`${alias}.entity_id`, 'final_entities.entity_id')
-            .andOn(`${alias}.key`, db.raw('?', [field]));
-        },
-      );
-    });
+            .on('order_0.entity_id', 'final_entities.entity_id')
+            .andOn('order_0.key', db.raw('?', [sortField.field]));
+        })
+        .select({
+          entity_id: 'final_entities.entity_id',
+          final_entity: 'final_entities.final_entity',
+          value: 'order_0.value',
+        });
+    } else {
+      entitiesQuery = entitiesQuery.select({
+        entity_id: 'final_entities.entity_id',
+        final_entity: 'final_entities.final_entity',
+      });
+    }
 
     entitiesQuery = entitiesQuery.whereNotNull('final_entities.final_entity');
 
-    if (request?.filter) {
+    // Apply predicate filter from cursor
+    if (effectiveQuery) {
       entitiesQuery = applyPredicateEntityFilterToQuery({
-        filter: request.filter,
+        filter: effectiveQuery,
         targetQuery: entitiesQuery,
         onEntityIdField: 'final_entities.entity_id',
         knex: db,
       });
     }
 
-    request?.order?.forEach(({ order }, index) => {
+    // Apply cursor-based pagination (keyset pagination)
+    if (cursor.orderFieldValues) {
+      if (cursor.orderFieldValues.length === 2) {
+        const [sortValue, entityId] = cursor.orderFieldValues;
+        const isOrderingDescending = sortField?.order === 'desc';
+        entitiesQuery = entitiesQuery.andWhere(function nested() {
+          this.where(
+            'order_0.value',
+            isOrderingDescending ? '<' : '>',
+            sortValue,
+          )
+            .orWhere('order_0.value', '=', sortValue)
+            .andWhere('final_entities.entity_id', '>', entityId);
+        });
+      } else if (cursor.orderFieldValues.length === 1) {
+        const [entityId] = cursor.orderFieldValues;
+        entitiesQuery = entitiesQuery.andWhere(
+          'final_entities.entity_id',
+          '>',
+          entityId,
+        );
+      }
+    }
+
+    if (sortField) {
       if (db.client.config.client === 'pg') {
         entitiesQuery = entitiesQuery.orderBy([
-          { column: `order_${index}.value`, order, nulls: 'last' },
+          { column: 'order_0.value', order: sortField.order, nulls: 'last' },
+          { column: 'final_entities.entity_id', order: 'asc' },
         ]);
       } else {
         entitiesQuery = entitiesQuery.orderBy([
-          { column: `order_${index}.value`, order: undefined, nulls: 'last' },
-          { column: `order_${index}.value`, order },
+          { column: 'order_0.value', order: undefined, nulls: 'last' },
+          { column: 'order_0.value', order: sortField.order },
+          { column: 'final_entities.entity_id', order: 'asc' },
         ]);
       }
-    });
-
-    if (!request?.order) {
-      entitiesQuery = entitiesQuery.orderBy('final_entities.entity_ref', 'asc');
     } else {
-      entitiesQuery.orderBy('final_entities.entity_id', 'asc');
+      entitiesQuery = entitiesQuery.orderBy('final_entities.entity_id', 'asc');
     }
 
-    if (limit !== undefined) {
-      entitiesQuery = entitiesQuery.limit(limit + 1);
-    }
-    if (offset !== undefined) {
+    // Apply a manually set initial offset (only when not using cursor pagination)
+    if (!request?.pagination?.after && offset !== undefined) {
       entitiesQuery = entitiesQuery.offset(offset);
     }
+    const effectiveLimit = limit ?? DEFAULT_LIMIT;
+    entitiesQuery = entitiesQuery.limit(effectiveLimit + 1);
 
     let rows = await entitiesQuery;
     let pageInfo: DbPageInfo;
-    if (limit === undefined || rows.length <= limit) {
+
+    if (rows.length <= effectiveLimit) {
       pageInfo = { hasNextPage: false };
     } else {
+      // Remove the extra row
       rows = rows.slice(0, -1);
+
+      const lastRow = rows[rows.length - 1];
+      const firstRow = rows[0];
+
+      // Create proper cursor with query field
+      const nextCursor: Cursor = {
+        query: effectiveQuery,
+        orderFields: cursor.orderFields || [],
+        orderFieldValues: sortField
+          ? [(lastRow as any).value, lastRow.entity_id]
+          : [lastRow.entity_id],
+        isPrevious: false,
+        firstSortFieldValues:
+          cursor.firstSortFieldValues ||
+          (sortField
+            ? [(firstRow as any).value, firstRow.entity_id]
+            : [firstRow.entity_id]),
+      };
+
       pageInfo = {
         hasNextPage: true,
-        endCursor: stringifyPagination({
-          limit,
-          offset: (offset ?? 0) + limit,
-        }),
+        endCursor: encodeCursor(nextCursor),
       };
     }
 
@@ -831,11 +896,18 @@ function parseCursorFromRequest(
   if (isQueryEntitiesInitialRequest(request)) {
     const {
       filter,
+      query,
       orderFields: sortFields = [],
       fullTextFilter,
       skipTotalItems = false,
     } = request;
-    return { filter, orderFields: sortFields, fullTextFilter, skipTotalItems };
+    return {
+      filter,
+      query,
+      orderFields: sortFields,
+      fullTextFilter,
+      skipTotalItems,
+    };
   }
   if (isQueryEntitiesCursorRequest(request)) {
     return {

plugins/catalog-backend/src/service/util.ts

@@ -122,7 +122,7 @@ export const cursorParser: z.ZodSchema<Cursor> = z.object({
   orderFieldValues: z.array(z.string().or(z.null())),
   filter: entityFilterParser.optional(),
   isPrevious: z.boolean(),
-  query: z.string().optional(),
+  query: z.any().optional(),
   firstSortFieldValues: z.array(z.string().or(z.null())).optional(),
   totalItems: z.number().optional(),
 });