feat(catalog-backend): ✨ add predicate-based entity filtering

- Add EntityPredicate and EntityPredicateRequest types to catalog types
- Implement queryEntitiesByPredicate method in DefaultEntitiesCatalog for predicate-based filtering
- Create applyPredicateEntityFilterToQuery utility to convert predicates to database queries

Signed-off-by: Nilay1999 <nilayparmar19@gmail.com>

plugins/catalog-backend/src/catalog/types.ts

@@ -16,7 +16,7 @@
 
 import { BackstageCredentials } from '@backstage/backend-plugin-api';
 import { Entity } from '@backstage/catalog-model';
-import { EntityFilter } from '@backstage/plugin-catalog-node';
+import { EntityFilter, EntityPredicate } from '@backstage/plugin-catalog-node';
 
 /**
  * A pagination rule for entities.
@@ -52,6 +52,13 @@ export type EntitiesRequest = {
   credentials: BackstageCredentials;
 };
 
+export type EntityPredicateRequest = {
+  filter?: EntityPredicate;
+  order?: EntityOrder[];
+  pagination?: EntityPagination;
+  credentials: BackstageCredentials;
+};
+
 /**
  * Encapsulates either a deserialized or serialized entities to be sent in a response.
  * @internal
@@ -165,6 +172,15 @@ export interface EntitiesCatalog {
    */
   queryEntities(request: QueryEntitiesRequest): Promise<QueryEntitiesResponse>;
 
+  /**
+   * Fetch entities using predicate-based filters.
+   *
+   * @param request - Request options with predicate filter
+   */
+  queryEntitiesByPredicate(
+    request?: EntityPredicateRequest,
+  ): Promise<EntitiesResponse>;
+
   /**
    * Removes a single entity.
    *

plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts

@@ -31,6 +31,7 @@ import {
   EntityFacetsResponse,
   EntityOrder,
   EntityPagination,
+  EntityPredicateRequest,
   QueryEntitiesRequest,
   QueryEntitiesResponse,
 } from '../catalog/types';
@@ -52,6 +53,7 @@ import {
 import { EntityFilter } from '@backstage/plugin-catalog-node';
 import { LoggerService } from '@backstage/backend-plugin-api';
 import { applyEntityFilterToQuery } from './request/applyEntityFilterToQuery';
+import { applyPredicateEntityFilterToQuery } from './request/applyPredicateEntityFilterToQuery';
 import { processRawEntitiesResult } from './response';
 
 const DEFAULT_LIMIT = 200;
@@ -213,6 +215,93 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
     };
   }
 
+  async queryEntitiesByPredicate(
+    request?: EntityPredicateRequest,
+  ): Promise<EntitiesResponse> {
+    const db = this.database;
+    const { limit, offset } = parsePagination(request?.pagination);
+
+    let entitiesQuery =
+      db<DbFinalEntitiesRow>('final_entities').select('final_entities.*');
+
+    request?.order?.forEach(({ field }, index) => {
+      const alias = `order_${index}`;
+      entitiesQuery = entitiesQuery.leftOuterJoin(
+        { [alias]: 'search' },
+        function search(inner) {
+          inner
+            .on(`${alias}.entity_id`, 'final_entities.entity_id')
+            .andOn(`${alias}.key`, db.raw('?', [field]));
+        },
+      );
+    });
+
+    entitiesQuery = entitiesQuery.whereNotNull('final_entities.final_entity');
+
+    if (request?.filter) {
+      entitiesQuery = applyPredicateEntityFilterToQuery({
+        filter: request.filter,
+        targetQuery: entitiesQuery,
+        onEntityIdField: 'final_entities.entity_id',
+        knex: db,
+      });
+    }
+
+    request?.order?.forEach(({ order }, index) => {
+      if (db.client.config.client === 'pg') {
+        entitiesQuery = entitiesQuery.orderBy([
+          { column: `order_${index}.value`, order, nulls: 'last' },
+        ]);
+      } else {
+        entitiesQuery = entitiesQuery.orderBy([
+          { column: `order_${index}.value`, order: undefined, nulls: 'last' },
+          { column: `order_${index}.value`, order },
+        ]);
+      }
+    });
+
+    if (!request?.order) {
+      entitiesQuery = entitiesQuery.orderBy('final_entities.entity_ref', 'asc');
+    } else {
+      entitiesQuery.orderBy('final_entities.entity_id', 'asc');
+    }
+
+    if (limit !== undefined) {
+      entitiesQuery = entitiesQuery.limit(limit + 1);
+    }
+    if (offset !== undefined) {
+      entitiesQuery = entitiesQuery.offset(offset);
+    }
+
+    let rows = await entitiesQuery;
+    let pageInfo: DbPageInfo;
+    if (limit === undefined || rows.length <= limit) {
+      pageInfo = { hasNextPage: false };
+    } else {
+      rows = rows.slice(0, -1);
+      pageInfo = {
+        hasNextPage: true,
+        endCursor: stringifyPagination({
+          limit,
+          offset: (offset ?? 0) + limit,
+        }),
+      };
+    }
+
+    return {
+      entities: processRawEntitiesResult(
+        rows.map(r => r.final_entity!),
+        this.enableRelationsCompatibility
+          ? e => {
+              expandLegacyCompoundRelationsInEntity(e);
+              return e;
+            }
+          : undefined,
+      ),
+      pageInfo,
+    };
+  }
+
   async entitiesBatch(
     request: EntitiesBatchRequest,
   ): Promise<EntitiesBatchResponse> {

plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts

@@ -0,0 +1,239 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  EntityPredicate,
+  EntityPredicatePrimitive,
+  EntityPredicateValue,
+} from '@backstage/plugin-catalog-node';
+import { Knex } from 'knex';
+import { DbSearchRow } from '../../database/tables';
+
+function isAllPredicate(
+  filter: EntityPredicate,
+): filter is { $all: EntityPredicate[] } {
+  return typeof filter === 'object' && filter !== null && '$all' in filter;
+}
+
+function isAnyPredicate(
+  filter: EntityPredicate,
+): filter is { $any: EntityPredicate[] } {
+  return typeof filter === 'object' && filter !== null && '$any' in filter;
+}
+
+function isNotPredicate(
+  filter: EntityPredicate,
+): filter is { $not: EntityPredicate } {
+  return typeof filter === 'object' && filter !== null && '$not' in filter;
+}
+
+function isPrimitive(value: unknown): value is EntityPredicatePrimitive {
+  return (
+    typeof value === 'string' ||
+    typeof value === 'number' ||
+    typeof value === 'boolean'
+  );
+}
+
+function isExistsValue(
+  value: EntityPredicateValue,
+): value is { $exists: boolean } {
+  return typeof value === 'object' && value !== null && '$exists' in value;
+}
+
+function isInValue(
+  value: EntityPredicateValue,
+): value is { $in: EntityPredicatePrimitive[] } {
+  return typeof value === 'object' && value !== null && '$in' in value;
+}
+
+function isFieldExpression(filter: EntityPredicate): boolean {
+  if (typeof filter !== 'object' || filter === null) {
+    return false;
+  }
+  // Not a logical operator
+  return !('$all' in filter || '$any' in filter || '$not' in filter);
+}
+
+/**
+ * example generated query
+ * Selects all non-null final entities that:
+ *
+ * - Are of kind "component"
+ * - Have spec.type = "service"
+ * - Are owned by either:
+ *   - backend-team
+ *   - platform-team
+ * - Are NOT in experimental lifecycle
+ *
+ * Results are ordered by entity_ref in ascending order.
+ *
+ * SQL Reference:
+ * ```
+ * SELECT final_entities.*
+ * FROM final_entities
+ * WHERE final_entities.final_entity IS NOT NULL
+ *   AND final_entities.entity_id IN (
+ *     SELECT entity_id FROM search WHERE key = 'kind' AND value = 'component'
+ *   )
+ *   AND final_entities.entity_id IN (
+ *     SELECT entity_id FROM search WHERE key = 'spec.type' AND value = 'service'
+ *   )
+ *   AND (
+ *     final_entities.entity_id IN (
+ *       SELECT entity_id FROM search WHERE key = 'spec.owner' AND value = 'backend-team'
+ *     )
+ *     OR
+ *     final_entities.entity_id IN (
+ *       SELECT entity_id FROM search WHERE key = 'spec.owner' AND value = 'platform-team'
+ *     )
+ *   )
+ *   AND final_entities.entity_id NOT IN (
+ *     SELECT entity_id FROM search WHERE key = 'spec.lifecycle' AND value = 'experimental'
+ *   )
+ * ORDER BY final_entities.entity_ref ASC;
+ * ```
+ */
+
+function applyPredicateInStrategy(
+  filter: EntityPredicate,
+  targetQuery: Knex.QueryBuilder,
+  onEntityIdField: string,
+  knex: Knex,
+  negate: boolean,
+): Knex.QueryBuilder {
+  // Handle $not
+  if (isNotPredicate(filter)) {
+    return applyPredicateInStrategy(
+      filter.$not,
+      targetQuery,
+      onEntityIdField,
+      knex,
+      !negate,
+    );
+  }
+
+  // Handle $all (AND)
+  if (isAllPredicate(filter)) {
+    return targetQuery[negate ? 'andWhereNot' : 'andWhere'](
+      function allFilter() {
+        for (const subFilter of filter.$all) {
+          this.andWhere(subQuery =>
+            applyPredicateInStrategy(
+              subFilter,
+              subQuery,
+              onEntityIdField,
+              knex,
+              false,
+            ),
+          );
+        }
+      },
+    );
+  }
+
+  // Handle $any (OR)
+  if (isAnyPredicate(filter)) {
+    return targetQuery[negate ? 'andWhereNot' : 'andWhere'](
+      function anyFilter() {
+        for (const subFilter of filter.$any) {
+          this.orWhere(subQuery =>
+            applyPredicateInStrategy(
+              subFilter,
+              subQuery,
+              onEntityIdField,
+              knex,
+              false,
+            ),
+          );
+        }
+      },
+    );
+  }
+
+  // Handle primitive value at top level (e.g., "component" shorthand)
+  if (isPrimitive(filter)) {
+    const matchQuery = knex<DbSearchRow>('search')
+      .select('search.entity_id')
+      .where({ value: String(filter).toLowerCase() });
+    return targetQuery.andWhere(
+      onEntityIdField,
+      negate ? 'not in' : 'in',
+      matchQuery,
+    );
+  }
+
+  // Handle field expressions like { "kind": "component" } or { "spec.type": { "$in": ["service", "website"] } }
+  if (isFieldExpression(filter)) {
+    return targetQuery[negate ? 'andWhereNot' : 'andWhere'](
+      function fieldFilter() {
+        for (const [key, value] of Object.entries(filter)) {
+          const normalizedKey = key.toLowerCase();
+
+          if (isExistsValue(value)) {
+            // Handle $exists
+            const existsQuery = knex<DbSearchRow>('search')
+              .select('search.entity_id')
+              .where({ key: normalizedKey });
+
+            if (value.$exists) {
+              this.andWhere(onEntityIdField, 'in', existsQuery);
+            } else {
+              this.andWhere(onEntityIdField, 'not in', existsQuery);
+            }
+          } else if (isInValue(value)) {
+            // Handle $in
+            const values = value.$in.map(v => String(v).toLowerCase());
+            const matchQuery = knex<DbSearchRow>('search')
+              .select('search.entity_id')
+              .where({ key: normalizedKey })
+              .whereIn('value', values);
+            this.andWhere(onEntityIdField, 'in', matchQuery);
+          } else if (isPrimitive(value)) {
+            // Handle direct value match
+            const matchQuery = knex<DbSearchRow>('search')
+              .select('search.entity_id')
+              .where({
+                key: normalizedKey,
+                value: String(value).toLowerCase(),
+              });
+            this.andWhere(onEntityIdField, 'in', matchQuery);
+          }
+        }
+      },
+    );
+  }
+
+  return targetQuery;
+}
+
+export function applyPredicateEntityFilterToQuery(options: {
+  filter: EntityPredicate;
+  targetQuery: Knex.QueryBuilder;
+  onEntityIdField: string;
+  knex: Knex;
+  strategy?: 'in' | 'join';
+}): Knex.QueryBuilder {
+  const { filter, targetQuery, onEntityIdField, knex } = options;
+
+  return applyPredicateInStrategy(
+    filter,
+    targetQuery,
+    onEntityIdField,
+    knex,
+    false,
+  );
+}

plugins/catalog-node/src/api/common.ts

@@ -85,3 +85,69 @@ export type EntitiesSearchFilter = {
    */
   values?: string[];
 };
+
+/**
+ * Defines a predicate used to filter entities using logical and comparison operators.
+ *
+ * This type supports:
+ * - Primitive equality matching
+ * - Field existence checks
+ * - Inclusion checks
+ * - Logical composition using $all, $any, and $not
+ *
+ * @public
+ */
+export type EntityPredicate =
+  | EntityPredicateExpression
+  | EntityPredicatePrimitive
+  | { $all: EntityPredicate[] }
+  | { $any: EntityPredicate[] }
+  | { $not: EntityPredicate };
+
+/**
+ * Represents a field-based predicate expression.
+ *
+ * Each key is treated as a JSON path into the entity object.
+ * Keys starting with `$` are reserved for operators and are not allowed.
+ *
+ * Example:
+ * ```ts
+ * {
+ *   "spec.type": "service",
+ *   "metadata.name": { $in: ["payments", "orders"] }
+ * }
+ * ```
+ *
+ * @public
+ */
+export type EntityPredicateExpression = {
+  /** JSON path of the entity field */
+  [KPath in string]: EntityPredicateValue;
+} & {
+  /** Operator keys are not allowed as field paths */
+  [KPath in `$${string}`]: never;
+};
+
+/**
+ * Represents a value condition for a predicate field.
+ *
+ * Supported forms:
+ * - Primitive equality match
+ * - Existence check using `$exists`
+ * - Inclusion check using `$in`
+ *
+ * @public
+ */
+export type EntityPredicateValue =
+  | EntityPredicatePrimitive
+  | { $exists: boolean }
+  | { $in: EntityPredicatePrimitive[] };
+
+/**
+ * Represents a primitive predicate value.
+ *
+ * Used for direct equality comparison.
+ *
+ * @public
+ */
+export type EntityPredicatePrimitive = string | number | boolean;

plugins/catalog-node/src/api/index.ts

@@ -20,6 +20,10 @@ export type {
   LocationSpec,
   EntitiesSearchFilter,
   EntityFilter,
+  EntityPredicate,
+  EntityPredicateExpression,
+  EntityPredicateValue,
+  EntityPredicatePrimitive,
 } from './common';
 export type {
   CatalogProcessor,