From e5ee4d8da7f8315df8254274d12eed2eeedd75ba Mon Sep 17 00:00:00 2001 From: Nilay1999 Date: Mon, 12 Jan 2026 11:30:26 +0530 Subject: [PATCH] feat(catalog-backend): :sparkles: add predicate-based entity filtering - Add EntityPredicate and EntityPredicateRequest types to catalog types - Implement queryEntitiesByPredicate method in DefaultEntitiesCatalog for predicate-based filtering - Create applyPredicateEntityFilterToQuery utility to convert predicates to database queries Signed-off-by: Nilay1999 --- plugins/catalog-backend/src/catalog/types.ts | 18 +- .../src/service/DefaultEntitiesCatalog.ts | 89 +++++++ .../applyPredicateEntityFilterToQuery.ts | 239 ++++++++++++++++++ plugins/catalog-node/src/api/common.ts | 66 +++++ plugins/catalog-node/src/api/index.ts | 4 + 5 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 1fbf337986..1b87352318 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -16,7 +16,7 @@ import { BackstageCredentials } from '@backstage/backend-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityFilter } from '@backstage/plugin-catalog-node'; +import { EntityFilter, EntityPredicate } from '@backstage/plugin-catalog-node'; /** * A pagination rule for entities. @@ -52,6 +52,13 @@ export type EntitiesRequest = { credentials: BackstageCredentials; }; +export type EntityPredicateRequest = { + filter?: EntityPredicate; + order?: EntityOrder[]; + pagination?: EntityPagination; + credentials: BackstageCredentials; +}; + /** * Encapsulates either a deserialized or serialized entities to be sent in a response. * @internal @@ -165,6 +172,15 @@ export interface EntitiesCatalog { */ queryEntities(request: QueryEntitiesRequest): Promise; + /** + * Fetch entities using predicate-based filters. + * + * @param request - Request options with predicate filter + */ + queryEntitiesByPredicate( + request?: EntityPredicateRequest, + ): Promise; + /** * Removes a single entity. * diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 92c0bb0ecf..c7b14282ef 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -31,6 +31,7 @@ import { EntityFacetsResponse, EntityOrder, EntityPagination, + EntityPredicateRequest, QueryEntitiesRequest, QueryEntitiesResponse, } from '../catalog/types'; @@ -52,6 +53,7 @@ import { import { EntityFilter } from '@backstage/plugin-catalog-node'; import { LoggerService } from '@backstage/backend-plugin-api'; import { applyEntityFilterToQuery } from './request/applyEntityFilterToQuery'; +import { applyPredicateEntityFilterToQuery } from './request/applyPredicateEntityFilterToQuery'; import { processRawEntitiesResult } from './response'; const DEFAULT_LIMIT = 200; @@ -213,6 +215,93 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } + async queryEntitiesByPredicate( + request?: EntityPredicateRequest, + ): Promise { + const db = this.database; + const { limit, offset } = parsePagination(request?.pagination); + + let entitiesQuery = + db('final_entities').select('final_entities.*'); + + request?.order?.forEach(({ field }, index) => { + const alias = `order_${index}`; + entitiesQuery = entitiesQuery.leftOuterJoin( + { [alias]: 'search' }, + function search(inner) { + inner + .on(`${alias}.entity_id`, 'final_entities.entity_id') + .andOn(`${alias}.key`, db.raw('?', [field])); + }, + ); + }); + + entitiesQuery = entitiesQuery.whereNotNull('final_entities.final_entity'); + + if (request?.filter) { + entitiesQuery = applyPredicateEntityFilterToQuery({ + filter: request.filter, + targetQuery: entitiesQuery, + onEntityIdField: 'final_entities.entity_id', + knex: db, + }); + } + + request?.order?.forEach(({ order }, index) => { + if (db.client.config.client === 'pg') { + entitiesQuery = entitiesQuery.orderBy([ + { column: `order_${index}.value`, order, nulls: 'last' }, + ]); + } else { + entitiesQuery = entitiesQuery.orderBy([ + { column: `order_${index}.value`, order: undefined, nulls: 'last' }, + { column: `order_${index}.value`, order }, + ]); + } + }); + + if (!request?.order) { + entitiesQuery = entitiesQuery.orderBy('final_entities.entity_ref', 'asc'); + } else { + entitiesQuery.orderBy('final_entities.entity_id', 'asc'); + } + + if (limit !== undefined) { + entitiesQuery = entitiesQuery.limit(limit + 1); + } + if (offset !== undefined) { + entitiesQuery = entitiesQuery.offset(offset); + } + + let rows = await entitiesQuery; + let pageInfo: DbPageInfo; + if (limit === undefined || rows.length <= limit) { + pageInfo = { hasNextPage: false }; + } else { + rows = rows.slice(0, -1); + pageInfo = { + hasNextPage: true, + endCursor: stringifyPagination({ + limit, + offset: (offset ?? 0) + limit, + }), + }; + } + + return { + entities: processRawEntitiesResult( + rows.map(r => r.final_entity!), + this.enableRelationsCompatibility + ? e => { + expandLegacyCompoundRelationsInEntity(e); + return e; + } + : undefined, + ), + pageInfo, + }; + } + async entitiesBatch( request: EntitiesBatchRequest, ): Promise { diff --git a/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts new file mode 100644 index 0000000000..59e08e7d9e --- /dev/null +++ b/plugins/catalog-backend/src/service/request/applyPredicateEntityFilterToQuery.ts @@ -0,0 +1,239 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + EntityPredicate, + EntityPredicatePrimitive, + EntityPredicateValue, +} from '@backstage/plugin-catalog-node'; +import { Knex } from 'knex'; +import { DbSearchRow } from '../../database/tables'; + +function isAllPredicate( + filter: EntityPredicate, +): filter is { $all: EntityPredicate[] } { + return typeof filter === 'object' && filter !== null && '$all' in filter; +} + +function isAnyPredicate( + filter: EntityPredicate, +): filter is { $any: EntityPredicate[] } { + return typeof filter === 'object' && filter !== null && '$any' in filter; +} + +function isNotPredicate( + filter: EntityPredicate, +): filter is { $not: EntityPredicate } { + return typeof filter === 'object' && filter !== null && '$not' in filter; +} + +function isPrimitive(value: unknown): value is EntityPredicatePrimitive { + return ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ); +} + +function isExistsValue( + value: EntityPredicateValue, +): value is { $exists: boolean } { + return typeof value === 'object' && value !== null && '$exists' in value; +} + +function isInValue( + value: EntityPredicateValue, +): value is { $in: EntityPredicatePrimitive[] } { + return typeof value === 'object' && value !== null && '$in' in value; +} + +function isFieldExpression(filter: EntityPredicate): boolean { + if (typeof filter !== 'object' || filter === null) { + return false; + } + // Not a logical operator + return !('$all' in filter || '$any' in filter || '$not' in filter); +} + +/** + * example generated query + * Selects all non-null final entities that: + * + * - Are of kind "component" + * - Have spec.type = "service" + * - Are owned by either: + * - backend-team + * - platform-team + * - Are NOT in experimental lifecycle + * + * Results are ordered by entity_ref in ascending order. + * + * SQL Reference: + * ``` + * SELECT final_entities.* + * FROM final_entities + * WHERE final_entities.final_entity IS NOT NULL + * AND final_entities.entity_id IN ( + * SELECT entity_id FROM search WHERE key = 'kind' AND value = 'component' + * ) + * AND final_entities.entity_id IN ( + * SELECT entity_id FROM search WHERE key = 'spec.type' AND value = 'service' + * ) + * AND ( + * final_entities.entity_id IN ( + * SELECT entity_id FROM search WHERE key = 'spec.owner' AND value = 'backend-team' + * ) + * OR + * final_entities.entity_id IN ( + * SELECT entity_id FROM search WHERE key = 'spec.owner' AND value = 'platform-team' + * ) + * ) + * AND final_entities.entity_id NOT IN ( + * SELECT entity_id FROM search WHERE key = 'spec.lifecycle' AND value = 'experimental' + * ) + * ORDER BY final_entities.entity_ref ASC; + * ``` + */ + +function applyPredicateInStrategy( + filter: EntityPredicate, + targetQuery: Knex.QueryBuilder, + onEntityIdField: string, + knex: Knex, + negate: boolean, +): Knex.QueryBuilder { + // Handle $not + if (isNotPredicate(filter)) { + return applyPredicateInStrategy( + filter.$not, + targetQuery, + onEntityIdField, + knex, + !negate, + ); + } + + // Handle $all (AND) + if (isAllPredicate(filter)) { + return targetQuery[negate ? 'andWhereNot' : 'andWhere']( + function allFilter() { + for (const subFilter of filter.$all) { + this.andWhere(subQuery => + applyPredicateInStrategy( + subFilter, + subQuery, + onEntityIdField, + knex, + false, + ), + ); + } + }, + ); + } + + // Handle $any (OR) + if (isAnyPredicate(filter)) { + return targetQuery[negate ? 'andWhereNot' : 'andWhere']( + function anyFilter() { + for (const subFilter of filter.$any) { + this.orWhere(subQuery => + applyPredicateInStrategy( + subFilter, + subQuery, + onEntityIdField, + knex, + false, + ), + ); + } + }, + ); + } + + // Handle primitive value at top level (e.g., "component" shorthand) + if (isPrimitive(filter)) { + const matchQuery = knex('search') + .select('search.entity_id') + .where({ value: String(filter).toLowerCase() }); + return targetQuery.andWhere( + onEntityIdField, + negate ? 'not in' : 'in', + matchQuery, + ); + } + + // Handle field expressions like { "kind": "component" } or { "spec.type": { "$in": ["service", "website"] } } + if (isFieldExpression(filter)) { + return targetQuery[negate ? 'andWhereNot' : 'andWhere']( + function fieldFilter() { + for (const [key, value] of Object.entries(filter)) { + const normalizedKey = key.toLowerCase(); + + if (isExistsValue(value)) { + // Handle $exists + const existsQuery = knex('search') + .select('search.entity_id') + .where({ key: normalizedKey }); + + if (value.$exists) { + this.andWhere(onEntityIdField, 'in', existsQuery); + } else { + this.andWhere(onEntityIdField, 'not in', existsQuery); + } + } else if (isInValue(value)) { + // Handle $in + const values = value.$in.map(v => String(v).toLowerCase()); + const matchQuery = knex('search') + .select('search.entity_id') + .where({ key: normalizedKey }) + .whereIn('value', values); + this.andWhere(onEntityIdField, 'in', matchQuery); + } else if (isPrimitive(value)) { + // Handle direct value match + const matchQuery = knex('search') + .select('search.entity_id') + .where({ + key: normalizedKey, + value: String(value).toLowerCase(), + }); + this.andWhere(onEntityIdField, 'in', matchQuery); + } + } + }, + ); + } + + return targetQuery; +} + +export function applyPredicateEntityFilterToQuery(options: { + filter: EntityPredicate; + targetQuery: Knex.QueryBuilder; + onEntityIdField: string; + knex: Knex; + strategy?: 'in' | 'join'; +}): Knex.QueryBuilder { + const { filter, targetQuery, onEntityIdField, knex } = options; + + return applyPredicateInStrategy( + filter, + targetQuery, + onEntityIdField, + knex, + false, + ); +} diff --git a/plugins/catalog-node/src/api/common.ts b/plugins/catalog-node/src/api/common.ts index 7f9bd3eabf..f88857a312 100644 --- a/plugins/catalog-node/src/api/common.ts +++ b/plugins/catalog-node/src/api/common.ts @@ -85,3 +85,69 @@ export type EntitiesSearchFilter = { */ values?: string[]; }; + +/** + * Defines a predicate used to filter entities using logical and comparison operators. + * + * This type supports: + * - Primitive equality matching + * - Field existence checks + * - Inclusion checks + * - Logical composition using $all, $any, and $not + * + * @public + */ +export type EntityPredicate = + | EntityPredicateExpression + | EntityPredicatePrimitive + | { $all: EntityPredicate[] } + | { $any: EntityPredicate[] } + | { $not: EntityPredicate }; + +/** + * Represents a field-based predicate expression. + * + * Each key is treated as a JSON path into the entity object. + * Keys starting with `$` are reserved for operators and are not allowed. + * + * Example: + * ```ts + * { + * "spec.type": "service", + * "metadata.name": { $in: ["payments", "orders"] } + * } + * ``` + * + * @public + */ +export type EntityPredicateExpression = { + /** JSON path of the entity field */ + [KPath in string]: EntityPredicateValue; +} & { + /** Operator keys are not allowed as field paths */ + [KPath in `$${string}`]: never; +}; + +/** + * Represents a value condition for a predicate field. + * + * Supported forms: + * - Primitive equality match + * - Existence check using `$exists` + * - Inclusion check using `$in` + * + * @public + */ +export type EntityPredicateValue = + | EntityPredicatePrimitive + | { $exists: boolean } + | { $in: EntityPredicatePrimitive[] }; + +/** + * Represents a primitive predicate value. + * + * Used for direct equality comparison. + * + * @public + */ +export type EntityPredicatePrimitive = string | number | boolean; diff --git a/plugins/catalog-node/src/api/index.ts b/plugins/catalog-node/src/api/index.ts index 292a6cd872..5b7abbb6b9 100644 --- a/plugins/catalog-node/src/api/index.ts +++ b/plugins/catalog-node/src/api/index.ts @@ -20,6 +20,10 @@ export type { LocationSpec, EntitiesSearchFilter, EntityFilter, + EntityPredicate, + EntityPredicateExpression, + EntityPredicateValue, + EntityPredicatePrimitive, } from './common'; export type { CatalogProcessor,