Merge pull request #33037 from backstage/freben/by-refs-predicate-query

feat(catalog): Add predicate-based filtering to the by-refs endpoint

.changeset/by-refs-predicate-backend.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Added support for predicate-based filtering on the `/entities/by-refs` endpoint via the `query` field in the request body. Supports `$all`, `$any`, `$not`, `$exists`, `$in`, `$contains`, and `$hasPrefix` operators.

.changeset/by-refs-predicate-client.md

@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-client': minor
+---
+
+Added support for the `query` field in `getEntitiesByRefs` requests, enabling predicate-based filtering with `$all`, `$any`, `$not`, `$exists`, `$in`, `$contains`, and `$hasPrefix` operators.

packages/catalog-client/report.api.md

@@ -236,6 +236,7 @@ export interface GetEntitiesByRefsRequest {
   entityRefs: string[];
   fields?: EntityFieldsQuery | undefined;
   filter?: EntityFilterQuery;
+  query?: FilterPredicate;
 }
 
 // @public

packages/catalog-client/src/CatalogClient.test.ts

@@ -302,6 +302,103 @@ describe('CatalogClient', () => {
 
       expect(response).toEqual({ items: [entity, undefined] });
     });
+
+    it('sends only query predicate in the body when query is provided without filter', async () => {
+      expect.assertions(3);
+      const entity = {
+        apiVersion: '1',
+        kind: 'Component',
+        metadata: {
+          name: 'Test2',
+          namespace: 'test1',
+        },
+      };
+      server.use(
+        rest.post(`${mockBaseUrl}/entities/by-refs`, async (req, res, ctx) => {
+          expect(req.url.search).toBe('');
+          await expect(req.json()).resolves.toEqual({
+            entityRefs: ['k:n/a'],
+            query: { kind: 'Component' },
+          });
+          return res(ctx.json({ items: [entity] }));
+        }),
+      );
+
+      const response = await client.getEntitiesByRefs(
+        {
+          entityRefs: ['k:n/a'],
+          query: { kind: 'Component' },
+        },
+        { token },
+      );
+
+      expect(response).toEqual({ items: [entity] });
+    });
+
+    it('merges filter and query into $all predicate when both are provided', async () => {
+      expect.assertions(4);
+      const entity = {
+        apiVersion: '1',
+        kind: 'Component',
+        metadata: {
+          name: 'Test2',
+          namespace: 'test1',
+        },
+      };
+      server.use(
+        rest.post(`${mockBaseUrl}/entities/by-refs`, async (req, res, ctx) => {
+          expect(req.url.search).toBe('');
+          const body = await req.json();
+          expect(body.entityRefs).toEqual(['k:n/a']);
+          expect(body.query).toEqual({
+            $all: [{ kind: 'Component' }, { kind: 'API' }],
+          });
+          return res(ctx.json({ items: [entity] }));
+        }),
+      );
+
+      const response = await client.getEntitiesByRefs(
+        {
+          entityRefs: ['k:n/a'],
+          query: { kind: 'Component' },
+          filter: { kind: ['API'] },
+        },
+        { token },
+      );
+
+      expect(response).toEqual({ items: [entity] });
+    });
+
+    it('sends filter as query parameter when only filter is provided (backward compat)', async () => {
+      expect.assertions(4);
+      const entity = {
+        apiVersion: '1',
+        kind: 'Component',
+        metadata: {
+          name: 'Test2',
+          namespace: 'test1',
+        },
+      };
+      server.use(
+        rest.post(`${mockBaseUrl}/entities/by-refs`, async (req, res, ctx) => {
+          expect(req.url.search).toBe('?filter=kind%3DAPI');
+          const body = await req.json();
+          expect(body).toEqual({ entityRefs: ['k:n/a'] });
+          expect(body.query).toBeUndefined();
+          return res(ctx.json({ items: [entity] }));
+        }),
+      );
+
+      const response = await client.getEntitiesByRefs(
+        {
+          entityRefs: ['k:n/a'],
+          filter: { kind: ['API'] },
+        },
+        { token },
+      );
+
+      expect(response).toEqual({ items: [entity] });
+    });
   });
 
   describe('queryEntities', () => {

packages/catalog-client/src/CatalogClient.ts

@@ -237,11 +237,36 @@ export class CatalogClient implements CatalogApi {
     request: GetEntitiesByRefsRequest,
     options?: CatalogRequestOptions,
   ): Promise<GetEntitiesByRefsResponse> {
+    const { filter, query } = request;
+
+    // Only convert and merge if both filter and query are provided, or if
+    // query alone is provided. When only filter is given, preserve the old
+    // query-parameter behavior for backward compatibility.
+    let filterPredicate: FilterPredicate | undefined;
+    if (query !== undefined) {
+      if (typeof query !== 'object' || query === null || Array.isArray(query)) {
+        throw new InputError('Query must be an object');
+      }
+      filterPredicate = query;
+      if (filter !== undefined) {
+        const converted = convertFilterToPredicate(filter);
+        filterPredicate = { $all: [filterPredicate, converted] };
+      }
+    }
+
     const getOneChunk = async (refs: string[]) => {
       const response = await this.apiClient.getEntitiesByRefs(
         {
-          body: { entityRefs: refs, fields: request.fields },
-          query: { filter: this.getFilterValue(request.filter) },
+          body: {
+            entityRefs: refs,
+            fields: request.fields,
+            ...(filterPredicate && {
+              query: filterPredicate as unknown as { [key: string]: any },
+            }),
+          },
+          query: filterPredicate
+            ? {}
+            : { filter: this.getFilterValue(request.filter) },
         },
         options,
       );

packages/catalog-client/src/schema/openapi/generated/models/GetEntitiesByRefsRequest.model.ts

@@ -24,4 +24,8 @@
 export interface GetEntitiesByRefsRequest {
   entityRefs: Array<string>;
   fields?: Array<string>;
+  /**
+   * A type representing all allowed JSON object values.
+   */
+  query?: { [key: string]: any };
 }

packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts

@@ -370,6 +370,25 @@ describe('InMemoryCatalogClient', () => {
         { kind: 'CustomKind', metadata: { name: 'e1' } },
       ]);
     });
+
+    it('supports query predicate filter', async () => {
+      const client = new InMemoryCatalogClient({ entities });
+      const result = await client.getEntitiesByRefs({
+        entityRefs: ['secondcustomkind:default/e2', 'customkind:default/e1'],
+        query: { kind: 'CustomKind' },
+      });
+      expect(result.items).toEqual([undefined, entity1]);
+    });
+
+    it('supports both filter and query predicate together', async () => {
+      const client = new InMemoryCatalogClient({ entities });
+      const result = await client.getEntitiesByRefs({
+        entityRefs: ['customkind:default/e1', 'customkind:other/e3'],
+        filter: { kind: 'CustomKind' },
+        query: { 'metadata.namespace': 'other' },
+      });
+      expect(result.items).toEqual([undefined, entity3]);
+    });
   });
 
   describe('queryEntities', () => {
@@ -975,6 +994,29 @@ describe('InMemoryCatalogClient', () => {
       });
       expect(result.facets['spec.nonexistent']).toEqual([]);
     });
+
+    it('supports query predicate filter', async () => {
+      const client = new InMemoryCatalogClient({ entities });
+      const result = await client.getEntityFacets({
+        facets: ['spec.type'],
+        query: { kind: 'CustomKind' },
+      });
+      expect(result.facets['spec.type']).toEqual([
+        { value: 'service', count: 2 },
+      ]);
+    });
+
+    it('supports both filter and query predicate together', async () => {
+      const client = new InMemoryCatalogClient({ entities });
+      const result = await client.getEntityFacets({
+        facets: ['spec.type'],
+        filter: { kind: 'CustomKind' },
+        query: { 'metadata.namespace': 'default' },
+      });
+      expect(result.facets['spec.type']).toEqual([
+        { value: 'service', count: 1 },
+      ]);
+    });
   });
 
   describe('not implemented methods', () => {

packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts

@@ -361,10 +361,15 @@ export class InMemoryCatalogClient implements CatalogApi {
     request: GetEntitiesByRefsRequest,
   ): Promise<GetEntitiesByRefsResponse> {
     const filter = createFilter(request.filter);
+    const queryFilter = request.query
+      ? filterPredicateToFilterFunction(request.query)
+      : undefined;
     const refMap = this.#createEntityRefMap();
     const items = request.entityRefs
       .map(ref => refMap.get(ref))
-      .map(e => (e && filter(e) ? e : undefined));
+      .map(e =>
+        e && filter(e) && (!queryFilter || queryFilter(e)) ? e : undefined,
+      );
     return {
       items: request.fields
         ? items.map(e => (e ? applyFieldsFilter(e, request.fields) : undefined))
@@ -506,7 +511,12 @@ export class InMemoryCatalogClient implements CatalogApi {
     request: GetEntityFacetsRequest,
   ): Promise<GetEntityFacetsResponse> {
     const filter = createFilter(request.filter);
-    const filteredEntities = this.#entities.filter(filter);
+    let filteredEntities = this.#entities.filter(filter);
+    if (request.query) {
+      filteredEntities = filteredEntities.filter(
+        filterPredicateToFilterFunction(request.query),
+      );
+    }
     const facets = Object.fromEntries(
       request.facets.map(facet => {
         const facetValues = new Map<string, number>();

packages/catalog-client/src/types/api.ts

@@ -212,6 +212,16 @@ export interface GetEntitiesByRefsRequest {
    * If given, return only entities that match the given filter.
    */
   filter?: EntityFilterQuery;
+  /**
+   * If given, return only entities that match the given predicate query.
+   *
+   * @remarks
+   *
+   * Supports operators like `$all`, `$any`, `$not`, `$exists`, `$in`,
+   * `$contains`, and `$hasPrefix`. When both `filter` and `query` are
+   * provided, they are combined with `$all`.
+   */
+  query?: FilterPredicate;
 }
 
 /**

plugins/catalog-backend/src/catalog/types.ts

@@ -86,6 +86,10 @@ export interface EntitiesBatchRequest {
    * they did not exist.
    */
   filter?: EntityFilter;
+  /**
+   * Predicate-based query for filtering entities.
+   */
+  query?: FilterPredicate;
   /**
    * Strips out only the parts of the entity bodies to include in the response.
    */

plugins/catalog-backend/src/schema/openapi.yaml

@@ -1036,6 +1036,8 @@ paths:
                   type: array
                   items:
                     type: string
+                query:
+                  $ref: '#/components/schemas/JsonObject'
             examples:
               Fetch Backstage entities:
                 value:

plugins/catalog-backend/src/schema/openapi/generated/models/GetEntitiesByRefsRequest.model.ts

@@ -24,4 +24,8 @@
 export interface GetEntitiesByRefsRequest {
   entityRefs: Array<string>;
   fields?: Array<string>;
+  /**
+   * A type representing all allowed JSON object values.
+   */
+  query?: { [key: string]: any };
 }

plugins/catalog-backend/src/schema/openapi/generated/router.ts

@@ -1124,6 +1124,9 @@ export const spec = {
                       type: 'string',
                     },
                   },
+                  query: {
+                    $ref: '#/components/schemas/JsonObject',
+                  },
                 },
               },
               examples: {

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -224,9 +224,10 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
         })
         .whereIn('final_entities.entity_ref', chunk);
 
-      if (request?.filter) {
+      if (request?.filter || request?.query) {
         query = applyEntityFilterToQuery({
           filter: request.filter,
+          query: request.query,
           targetQuery: query,
           onEntityIdField: 'final_entities.entity_id',
           knex: this.database,

plugins/catalog-backend/src/service/createRouter.test.ts

@@ -725,6 +725,85 @@ describe('createRouter readonly disabled', () => {
     });
   });
 
+  describe('POST /entities/by-refs with query predicate', () => {
+    it('can fetch entities by refs with a predicate query', async () => {
+      const entity: Entity = {
+        apiVersion: 'a',
+        kind: 'component',
+        metadata: {
+          name: 'a',
+        },
+      };
+      const entityRef = stringifyEntityRef(entity);
+      entitiesCatalog.entitiesBatch.mockResolvedValue({
+        items: { type: 'object', entities: [entity] },
+      });
+      const response = await request(app)
+        .post('/entities/by-refs')
+        .set('Content-Type', 'application/json')
+        .send(
+          JSON.stringify({
+            entityRefs: [entityRef],
+            query: { kind: 'Component' },
+          }),
+        );
+      expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledTimes(1);
+      expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledWith({
+        entityRefs: [entityRef],
+        fields: undefined,
+        credentials: mockCredentials.user(),
+        filter: undefined,
+        query: { kind: 'Component' },
+      });
+      expect(response.status).toEqual(200);
+      expect(response.body).toEqual({ items: [entity] });
+    });
+
+    it('forwards both filter query param and body query predicate independently', async () => {
+      const entity: Entity = {
+        apiVersion: 'a',
+        kind: 'component',
+        metadata: {
+          name: 'a',
+        },
+      };
+      const entityRef = stringifyEntityRef(entity);
+      entitiesCatalog.entitiesBatch.mockResolvedValue({
+        items: { type: 'object', entities: [entity] },
+      });
+      const response = await request(app)
+        .post('/entities/by-refs?filter=kind=Component')
+        .set('Content-Type', 'application/json')
+        .send(
+          JSON.stringify({
+            entityRefs: [entityRef],
+            query: { 'metadata.namespace': 'default' },
+          }),
+        );
+      expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledWith({
+        entityRefs: [entityRef],
+        fields: undefined,
+        credentials: mockCredentials.user(),
+        filter: { key: 'kind', values: ['Component'] },
+        query: { 'metadata.namespace': 'default' },
+      });
+      expect(response.status).toEqual(200);
+    });
+
+    it('rejects invalid query predicate', async () => {
+      const response = await request(app)
+        .post('/entities/by-refs')
+        .set('Content-Type', 'application/json')
+        .send(
+          JSON.stringify({
+            entityRefs: ['component:default/a'],
+            query: { $invalid: 'bad' },
+          }),
+        );
+      expect(response.status).toEqual(400);
+    });
+  });
+
   describe('GET /locations', () => {
     it('happy path: lists locations', async () => {
       const locations: Location[] = [

plugins/catalog-backend/src/service/createRouter.ts

@@ -525,6 +525,7 @@ export async function createRouter(
           const { items } = await entitiesCatalog.entitiesBatch({
             entityRefs: request.entityRefs,
             filter: parseEntityFilterParams(req.query),
+            query: request.query,
             fields: parseEntityTransformParams(req.query, request.fields),
             credentials: await httpAuth.credentials(req),
           });

plugins/catalog-backend/src/service/request/entitiesBatchRequest.ts

@@ -15,20 +15,36 @@
  */
 
 import { InputError } from '@backstage/errors';
+import {
+  createZodV3FilterPredicateSchema,
+  FilterPredicate,
+} from '@backstage/filter-predicates';
 import { Request } from 'express';
-import { z } from 'zod';
+import { z } from 'zod/v3';
+import { fromZodError } from 'zod-validation-error/v3';
+
+const filterPredicateSchema = createZodV3FilterPredicateSchema(z);
 
 const schema = z.object({
   entityRefs: z.array(z.string()),
   fields: z.array(z.string()).optional(),
+  query: filterPredicateSchema.optional(),
 });
 
-export function entitiesBatchRequest(req: Request): z.infer<typeof schema> {
-  try {
-    return schema.parse(req.body);
-  } catch (error) {
+export interface ParsedEntitiesBatchRequest {
+  entityRefs: string[];
+  fields?: string[];
+  query?: FilterPredicate;
+}
+
+export function entitiesBatchRequest(req: Request): ParsedEntitiesBatchRequest {
+  const result = schema.safeParse(req.body);
+  if (!result.success) {
     throw new InputError(
-      `Malformed request body (did you remember to specify an application/json content type?), ${error.message}`,
+      `Malformed request body (did you remember to specify an application/json content type?), ${fromZodError(
+        result.error,
+      )}`,
     );
   }
+  return result.data;
 }