diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index d182ffdc47..62c8fad605 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -179,7 +179,7 @@ export interface EntitiesCatalog { * * @param request */ - queryEntities(request?: QueryEntitiesRequest): Promise; + queryEntities(request: QueryEntitiesRequest): Promise; /** * Removes a single entity. @@ -242,7 +242,7 @@ export interface QueryEntitiesCursorRequest { authorizationToken?: string; fields?: (entity: Entity) => Entity; limit?: number; - cursor: string; + cursor: Cursor; } /** @@ -258,11 +258,11 @@ export interface QueryEntitiesResponse { /** * The cursor of the next pagination request. */ - nextCursor?: string; + nextCursor?: Cursor; /** * The cursor of the previous pagination request. */ - prevCursor?: string; + prevCursor?: Cursor; }; /** * the total number of entities matching the current filters. diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index e559f5fd49..2ee47dae19 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -20,6 +20,8 @@ import { createConditionTransformer } from '@backstage/plugin-permission-node'; import { isEntityKind } from '../permissions/rules/isEntityKind'; import { CatalogPermissionRule } from '../permissions/rules'; import { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog'; +import { Cursor, EntityFilter, QueryEntitiesResponse } from '../catalog/types'; +import { Entity } from '@backstage/catalog-model'; describe('AuthorizedEntitiesCatalog', () => { const fakeCatalog = { @@ -157,6 +159,153 @@ describe('AuthorizedEntitiesCatalog', () => { }); }); + describe('queryEntities', () => { + it('returns empty response on DENY', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.DENY }, + ]); + const catalog = createCatalog(); + + await expect( + catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }), + ).resolves.toEqual({ + items: [], + pageInfo: {}, + totalItems: 0, + }); + + expect(fakeCatalog.queryEntities).not.toHaveBeenCalled(); + }); + + it('calls underlying catalog method on ALLOW', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.ALLOW }, + ]); + const catalog = createCatalog(); + + await catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + }); + + it('calls underlying catalog method with correct filter on CONDITIONAL', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { + result: AuthorizeResult.CONDITIONAL, + conditions: { + rule: 'IS_ENTITY_KIND', + params: { kinds: ['b'] }, + }, + }, + ]); + + const requestFilter: EntityFilter = { key: 'name', values: ['name'] }; + + const entities = [ + { + kind: 'component', + namespace: 'default', + name: 'a', + } as unknown as Entity, + { + kind: 'component', + namespace: 'default', + name: 'b1', + } as unknown as Entity, + ]; + + fakeCatalog.queryEntities.mockResolvedValue({ + items: entities, + pageInfo: { + nextCursor: { + isPrevious: false, + orderFieldValues: ['xxx', null], + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + prevCursor: { + isPrevious: true, + orderFieldValues: ['a', null], + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + }, + totalItems: 4, + } as QueryEntitiesResponse); + const catalog = createCatalog(isEntityKind); + + let response = await catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'name', values: ['name'] }, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({ + authorizationToken: 'abcd', + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }); + + expect(response).toEqual({ + items: entities, + totalItems: 4, + pageInfo: { + nextCursor: { + isPrevious: false, + filter: requestFilter, + orderFieldValues: ['xxx', null], + }, + prevCursor: { + isPrevious: true, + filter: requestFilter, + orderFieldValues: ['a', null], + }, + }, + }); + + const cursor: Cursor = { + filter: requestFilter, + orderFields: [{ field: 'name', order: 'asc' }], + isPrevious: false, + orderFieldValues: ['a', null], + }; + response = await catalog.queryEntities({ + authorizationToken: 'abcd', + cursor, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenNthCalledWith(2, { + authorizationToken: 'abcd', + cursor: { + ...cursor, + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + }); + + expect(response).toEqual({ + items: entities, + totalItems: 4, + pageInfo: { + nextCursor: { + isPrevious: false, + filter: requestFilter, + orderFieldValues: ['xxx', null], + }, + prevCursor: { + isPrevious: true, + filter: requestFilter, + orderFieldValues: ['a', null], + }, + }, + }); + }); + }); + describe('removeEntityByUid', () => { it('throws error on DENY', async () => { fakeCatalog.entities.mockResolvedValue({ diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index f9ec058879..2649b45260 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -26,6 +26,7 @@ import { } from '@backstage/plugin-permission-common'; import { ConditionTransformer } from '@backstage/plugin-permission-node'; import { + Cursor, EntitiesBatchRequest, EntitiesBatchResponse, EntitiesCatalog, @@ -39,7 +40,7 @@ import { QueryEntitiesResponse, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; -import { isQueryEntitiesInitialRequest } from './util'; +import { isQueryEntitiesCursorRequest } from './util'; export class AuthorizedEntitiesCatalog implements EntitiesCatalog { constructor( @@ -110,12 +111,12 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { } async queryEntities( - request?: QueryEntitiesRequest, + request: QueryEntitiesRequest, ): Promise { const authorizeDecision = ( await this.permissionApi.authorizeConditional( [{ permission: catalogEntityReadPermission }], - { token: request?.authorizationToken }, + { token: request.authorizationToken }, ) )[0]; @@ -132,14 +133,52 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { authorizeDecision.conditions, ); - return this.entitiesCatalog.queryEntities({ - ...request, - ...(isQueryEntitiesInitialRequest(request) && { - filter: request?.filter + let permissionedRequest: QueryEntitiesRequest; + let requestFilter: EntityFilter | undefined; + + if (isQueryEntitiesCursorRequest(request)) { + requestFilter = request.cursor.filter; + + permissionedRequest = { + ...request, + cursor: { + ...request.cursor, + filter: request.cursor.filter + ? { allOf: [permissionFilter, request.cursor.filter] } + : permissionFilter, + }, + }; + } else { + permissionedRequest = { + ...request, + filter: request.filter ? { allOf: [permissionFilter, request.filter] } : permissionFilter, - }), - }); + }; + requestFilter = request.filter; + } + + const response = await this.entitiesCatalog.queryEntities( + permissionedRequest, + ); + + const prevCursor: Cursor | undefined = response.pageInfo.prevCursor && { + ...response.pageInfo.prevCursor, + filter: requestFilter, + }; + + const nextCursor: Cursor | undefined = response.pageInfo.nextCursor && { + ...response.pageInfo.nextCursor, + filter: requestFilter, + }; + + return { + ...response, + pageInfo: { + prevCursor, + nextCursor, + }, + }; } return this.entitiesCatalog.queryEntities(request); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 5bfb0e0077..740f5b2558 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1291,26 +1291,6 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.totalItems).toBe(6); }, ); - - it('should throw in case of malformed cursor', async () => { - const catalog = new DefaultEntitiesCatalog({ - database: knex, - logger: getVoidLogger(), - stitcher, - }); - - const cursor = Buffer.from( - JSON.stringify({ filter: 'notavalidfilter' }), - 'utf8', - ).toString('base64'); - const request: QueryEntitiesCursorRequest = { - cursor, - }; - - await expect(catalog.queryEntities(request)).rejects.toThrow( - Error('Malformed cursor, could not be parsed'), - ); - }); }); describe('removeEntityByUid', () => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index ba59bc55c0..a635a9a7d2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -333,10 +333,10 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } async queryEntities( - request?: QueryEntitiesRequest, + request: QueryEntitiesRequest, ): Promise { const db = this.database; - const limit = request?.limit ?? 20; + const limit = request.limit ?? 20; const cursor: Omit & { orderFieldValues?: (string | null)[]; @@ -449,32 +449,32 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { firstRow?.entity_id, ]; - const nextCursor = hasMoreResults - ? encodeCursor({ + const nextCursor: Cursor | undefined = hasMoreResults + ? { ...cursor, orderFieldValues: sortFieldsFromRow(lastRow), firstSortFieldValues, isPrevious: false, totalItems, - }) + } : undefined; - const prevCursor = + const prevCursor: Cursor | undefined = !isInitialRequest && rows.length > 0 && !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues) - ? encodeCursor({ + ? { ...cursor, orderFieldValues: sortFieldsFromRow(firstRow), firstSortFieldValues: cursor.firstSortFieldValues, isPrevious: true, totalItems, - }) + } : undefined; const items = rows .map(e => JSON.parse(e.final_entity!)) - .map(e => (request?.fields ? request.fields(e) : e)); + .map(e => (request.fields ? request.fields(e) : e)); return { items, @@ -690,16 +690,6 @@ export const cursorParser: z.ZodSchema = z.object({ totalItems: z.number().optional(), }); -function encodeCursor(cursor: Cursor) { - const json = JSON.stringify(cursor); - return Buffer.from(json, 'utf8').toString('base64'); -} - -function decodeCursor(encodedCursor: string) { - const json = Buffer.from(encodedCursor, 'base64').toString('utf8'); - return cursorParser.parse(JSON.parse(json)); -} - function parseCursorFromRequest( request?: QueryEntitiesRequest, ): Partial { @@ -712,11 +702,7 @@ function parseCursorFromRequest( return { filter, orderFields: sortFields, fullTextFilter }; } if (isQueryEntitiesCursorRequest(request)) { - try { - return decodeCursor(request.cursor); - } catch { - throw new InputError('Malformed cursor, could not be parsed'); - } + return request.cursor; } return {}; } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 5121e800da..f2f87fa800 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -25,7 +25,7 @@ import { } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; -import { EntitiesCatalog } from '../catalog/types'; +import { Cursor, EntitiesCatalog } from '../catalog/types'; import { LocationInput, LocationService, RefreshService } from './types'; import { basicEntityFilter } from './request'; import { createRouter } from './createRouter'; @@ -37,6 +37,7 @@ import { import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common'; import { CatalogProcessingOrchestrator } from '../processing/types'; import { z } from 'zod'; +import { encodeCursor } from './util'; describe('createRouter readonly disabled', () => { let entitiesCatalog: jest.Mocked; @@ -145,7 +146,7 @@ describe('createRouter readonly disabled', () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ items, totalItems: 100, - pageInfo: { nextCursor: 'something' }, + pageInfo: { nextCursor: mockCursor() }, }); const response = await request(app).get('/entities/by-query'); @@ -154,7 +155,7 @@ describe('createRouter readonly disabled', () => { items, totalItems: 100, pageInfo: { - nextCursor: 'something', + nextCursor: expect.any(String), }, }); }); @@ -202,23 +203,50 @@ describe('createRouter readonly disabled', () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ items, totalItems: 100, - pageInfo: { nextCursor: 'next' }, + pageInfo: { nextCursor: mockCursor() }, }); + const cursor = mockCursor({ totalItems: 100, isPrevious: false }); + const response = await request(app).get( - '/entities/by-query?cursor=something', + `/entities/by-query?cursor=${encodeCursor(cursor)}`, ); expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ - cursor: 'something', + cursor, }); expect(response.status).toEqual(200); expect(response.body).toEqual({ items, totalItems: 100, - pageInfo: { nextCursor: 'next' }, + pageInfo: { nextCursor: expect.any(String) }, }); }); + + it('should throw in case of malformed cursor', async () => { + const items: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items, + totalItems: 100, + pageInfo: { nextCursor: mockCursor() }, + }); + + let response = await request(app).get( + `/entities/by-query?cursor=${Buffer.from( + JSON.stringify({ bad: 'cursor' }), + 'utf8', + ).toString('base64')}`, + ); + expect(response.status).toEqual(400); + expect(response.body.error.message).toMatch(/Malformed cursor/); + + response = await request(app).get(`/entities/by-query?cursor=badcursor`); + expect(response.status).toEqual(400); + expect(response.body.error.message).toMatch(/Malformed cursor/); + }); }); describe('GET /entities/by-uid/:uid', () => { @@ -908,3 +936,12 @@ describe('NextRouter permissioning', () => { }); }); }); + +function mockCursor(partialCursor?: Partial): Cursor { + return { + orderFields: [], + orderFieldValues: [], + isPrevious: false, + ...partialCursor, + }; +} diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 7603f6955d..849096c601 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -46,6 +46,7 @@ import { parseEntityOrderParams } from './request/parseEntityOrderParams'; import { LocationService, RefreshOptions, RefreshService } from './types'; import { disallowReadonlyMode, + encodeCursor, locationInput, validateRequestBody, } from './util'; @@ -132,12 +133,24 @@ export async function createRouter( res.json(entities); }) .get('/entities/by-query', async (req, res) => { - const response = await entitiesCatalog.queryEntities({ - ...parseQueryEntitiesParams(req.query), - authorizationToken: getBearerToken(req.header('authorization')), - }); + const { items, pageInfo, totalItems } = + await entitiesCatalog.queryEntities({ + ...parseQueryEntitiesParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); - res.json(response); + res.json({ + items, + totalItems, + pageInfo: { + ...(pageInfo.nextCursor && { + nextCursor: encodeCursor(pageInfo.nextCursor), + }), + ...(pageInfo.prevCursor && { + prevCursor: encodeCursor(pageInfo.prevCursor), + }), + }, + }); }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts index da2d7e474c..43ba218eac 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -15,9 +15,11 @@ */ import { + Cursor, QueryEntitiesCursorRequest, QueryEntitiesInitialRequest, } from '../../catalog/types'; +import { encodeCursor } from '../util'; import { parseQueryEntitiesParams } from './parseQueryEntitiesParams'; describe('parseQueryEntitiesParams', () => { @@ -77,29 +79,40 @@ describe('parseQueryEntitiesParams', () => { describe('cursor request', () => { it('should parse all the defined params', () => { + const cursor: Cursor = { + totalItems: 100, + orderFields: [], + orderFieldValues: [], + isPrevious: false, + }; const validRequest = { authorizationToken: 'to_not_be_returned', fields: ['kind'], limit: '3', - cursor: 'cursor', + cursor: encodeCursor(cursor), }; const parsedObj = parseQueryEntitiesParams( validRequest, ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); - expect(parsedObj.cursor).toBe('cursor'); + expect(parsedObj.cursor).toEqual(cursor); }); it('should ignore unknown params', () => { + const cursor: Cursor = { + totalItems: 100, + orderFields: [], + orderFieldValues: [], + isPrevious: false, + }; const validRequest = { authorizationToken: 'to_not_be_returned', fields: ['kind'], limit: '3', - cursor: 'cursor', + cursor: encodeCursor(cursor), filter: ['a=1', 'b=2'], - sortField: 'sortField', - sortFieldOrder: 'desc', + orderField: 'orderField,desc', query: 'query', }; const parsedObj = parseQueryEntitiesParams( @@ -107,10 +120,9 @@ describe('parseQueryEntitiesParams', () => { ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); - expect(parsedObj.cursor).toBe('cursor'); + expect(parsedObj.cursor).toEqual(cursor); expect(parsedObj).not.toHaveProperty('filter'); - expect(parsedObj).not.toHaveProperty('sortField'); - expect(parsedObj).not.toHaveProperty('sortFieldOrder'); + expect(parsedObj).not.toHaveProperty('orderField'); expect(parsedObj).not.toHaveProperty('query'); }); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts index 67a4629c1c..ae8b649d13 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -19,6 +19,7 @@ import { QueryEntitiesInitialRequest, QueryEntitiesRequest, } from '../../catalog/types'; +import { decodeCursor } from '../util'; import { parseIntegerParam, parseStringParam } from './common'; import { parseEntityFilterParams } from './parseEntityFilterParams'; import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; @@ -32,8 +33,9 @@ export function parseQueryEntitiesParams( const limit = parseIntegerParam(params.limit, 'limit'); const cursor = parseStringParam(params.cursor, 'cursor'); if (cursor) { + const decodedCursor = decodeCursor(cursor); const response: Omit = { - cursor, + cursor: decodedCursor, fields, limit, }; diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 1ab154a86d..699bca2fc9 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -19,6 +19,8 @@ import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; import { + Cursor, + EntityFilter, QueryEntitiesCursorRequest, QueryEntitiesInitialRequest, QueryEntitiesRequest, @@ -88,3 +90,45 @@ export function isQueryEntitiesCursorRequest( } return !!(input as QueryEntitiesCursorRequest).cursor; } + +const entityFilterParser: z.ZodSchema = z.lazy(() => + z + .object({ + key: z.string(), + values: z.array(z.string()).optional(), + }) + .or(z.object({ not: entityFilterParser })) + .or(z.object({ anyOf: z.array(entityFilterParser) })) + .or(z.object({ allOf: z.array(entityFilterParser) })), +); + +export const cursorParser: z.ZodSchema = z.object({ + orderFields: z.array( + z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }), + ), + orderFieldValues: z.array(z.string().or(z.null())), + filter: entityFilterParser.optional(), + isPrevious: z.boolean(), + query: z.string().optional(), + firstSortFieldValues: z.array(z.string().or(z.null())).optional(), + totalItems: z.number().optional(), +}); + +export function encodeCursor(cursor: Cursor) { + const json = JSON.stringify(cursor); + return Buffer.from(json, 'utf8').toString('base64'); +} + +export function decodeCursor(encodedCursor: string) { + try { + const data = Buffer.from(encodedCursor, 'base64').toString('utf8'); + const result = cursorParser.safeParse(JSON.parse(data)); + + if (!result.success) { + throw new InputError(`Malformed cursor: ${result.error}`); + } + return result.data; + } catch (e) { + throw new InputError(`Malformed cursor: ${e}`); + } +}