diff --git a/.changeset/breezy-bees-care.md b/.changeset/breezy-bees-care.md new file mode 100644 index 0000000000..9a5b59fe03 --- /dev/null +++ b/.changeset/breezy-bees-care.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +Add `queryEntities` method to `CatalogApi`. diff --git a/.changeset/silly-suits-run.md b/.changeset/silly-suits-run.md new file mode 100644 index 0000000000..ec371e4077 --- /dev/null +++ b/.changeset/silly-suits-run.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Add /entities/by-query endpoint returning paginated entities. + +The endpoint supports cursor base pagination and server side sorting of the entities diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index a81314308b..22feceabec 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -58,6 +58,10 @@ export interface CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; + queryEntities( + request?: QueryEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; refreshEntity( entityRef: string, options?: CatalogRequestOptions, @@ -124,6 +128,10 @@ export class CatalogClient implements CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; + queryEntities( + request?: QueryEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; refreshEntity( entityRef: string, options?: CatalogRequestOptions, @@ -219,10 +227,7 @@ export interface GetEntityAncestorsResponse { // @public export interface GetEntityFacetsRequest { facets: string[]; - filter?: - | Record[] - | Record - | undefined; + filter?: EntityFilterQuery; } // @public @@ -244,6 +249,40 @@ type Location_2 = { }; export { Location_2 as Location }; +// @public +export type QueryEntitiesCursorRequest = { + fields?: string[]; + limit?: number; + cursor: string; +}; + +// @public +export type QueryEntitiesInitialRequest = { + fields?: string[]; + limit?: number; + filter?: EntityFilterQuery; + orderFields?: EntityOrderQuery; + fullTextFilter?: { + term: string; + fields?: string[]; + }; +}; + +// @public +export type QueryEntitiesRequest = + | QueryEntitiesInitialRequest + | QueryEntitiesCursorRequest; + +// @public +export type QueryEntitiesResponse = { + items: Entity[]; + totalItems: number; + pageInfo: { + nextCursor?: string; + prevCursor?: string; + }; +}; + // @public export type ValidateEntityResponse = | { diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 2e9d2a0679..c235ff1592 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -18,7 +18,11 @@ import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { CATALOG_FILTER_EXISTS, GetEntitiesResponse } from './types/api'; +import { + CATALOG_FILTER_EXISTS, + GetEntitiesResponse, + QueryEntitiesResponse, +} from './types/api'; import { DiscoveryApi } from './types/discovery'; const server = setupServer(); @@ -249,6 +253,185 @@ describe('CatalogClient', () => { }); }); + describe('queryEntities', () => { + const defaultResponse: QueryEntitiesResponse = { + items: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test1', + namespace: 'test1', + }, + }, + ], + pageInfo: { + nextCursor: 'next', + prevCursor: 'prev', + }, + totalItems: 10, + }; + + beforeEach(() => { + server.use( + rest.get(`${mockBaseUrl}/entities/by-query`, (_, res, ctx) => { + return res(ctx.json(defaultResponse)); + }), + ); + }); + + it('should fetch entities from correct endpoint', async () => { + const response = await client.queryEntities({}, { token }); + expect(response?.items).toEqual(defaultResponse.items); + expect(response?.totalItems).toEqual(defaultResponse.totalItems); + expect(response?.pageInfo.nextCursor).toBeDefined(); + expect(response?.pageInfo.prevCursor).toBeDefined(); + }); + + it('builds multiple entity search filters properly', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ items: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + const response = await client.queryEntities( + { + filter: [ + { + a: '1', + b: ['2', '3'], + รถ: '=', + }, + { + a: '2', + }, + { + c: CATALOG_FILTER_EXISTS, + }, + ], + }, + { token }, + ); + + expect(response).toEqual({ items: [], totalItems: 0 }); + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c', + ); + }); + + it('should send query params correctly on initial request', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ items: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + fields: ['a', 'b'], + limit: 100, + fullTextFilter: { + term: 'query', + }, + orderFields: [ + { field: 'metadata.name', order: 'asc' }, + { field: 'metadata.uid', order: 'desc' }, + ], + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?limit=100&sortField=metadata.name,asc&sortField=metadata.uid,desc&fields=a,b&fullTextFilterTerm=query', + ); + }); + + it('should ignore initial query params if cursor is passed', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ items: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + await client.queryEntities({ + fields: ['a', 'b'], + limit: 100, + fullTextFilter: { + term: 'query', + }, + orderFields: [{ field: 'metadata.name', order: 'asc' }], + cursor: 'cursor', + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?cursor=cursor&limit=100&fields=a,b', + ); + }); + + it('should return paginated functions if next and prev cursors are present', async () => { + const mockedEndpoint = jest.fn().mockImplementation((_req, res, ctx) => + res( + ctx.json({ + items: [ + { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'e1', + }, + }, + { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'e2', + }, + }, + ], + pageInfo: { + nextCursor: 'nextcursor', + prevCursor: 'prevcursor', + }, + totalItems: 100, + } as QueryEntitiesResponse), + ), + ); + + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); + + const response = await client.queryEntities({ + limit: 2, + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); + + expect(response?.pageInfo.nextCursor).toBeDefined(); + expect(response?.pageInfo.prevCursor).toBeDefined(); + + await client.queryEntities({ cursor: response!.pageInfo.nextCursor! }); + expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( + '?cursor=nextcursor', + ); + + await client.queryEntities({ cursor: response!.pageInfo.prevCursor! }); + expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( + '?cursor=prevcursor', + ); + }); + }); + describe('getEntityByRef', () => { const existingEntity: Entity = { apiVersion: 'v1', diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index af4262b842..4984d394d4 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -39,9 +39,13 @@ import { ValidateEntityResponse, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, + QueryEntitiesRequest, + QueryEntitiesResponse, + EntityFilterQuery, } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; +import { isQueryEntitiesInitialRequest } from './utils'; /** * A frontend and backend compatible client for communicating with the Backstage @@ -107,30 +111,7 @@ export class CatalogClient implements CatalogApi { limit, after, } = request ?? {}; - const params: string[] = []; - - // filter param can occur multiple times, for example - // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' - // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters - // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of [filter].flat()) { - const filterParts: string[] = []; - for (const [key, value] of Object.entries(filterItem)) { - for (const v of [value].flat()) { - if (v === CATALOG_FILTER_EXISTS) { - filterParts.push(encodeURIComponent(key)); - } else if (typeof v === 'string') { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); - } - } - } - - if (filterParts.length) { - params.push(`filter=${filterParts.join(',')}`); - } - } + const params = this.getParams(filter); if (fields.length) { params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); @@ -223,6 +204,64 @@ export class CatalogClient implements CatalogApi { return { items }; } + /** + * {@inheritdoc CatalogApi.queryEntities} + */ + async queryEntities( + request: QueryEntitiesRequest = {}, + options?: CatalogRequestOptions, + ) { + const params: string[] = []; + + if (isQueryEntitiesInitialRequest(request)) { + const { + fields = [], + filter, + limit, + orderFields, + fullTextFilter, + } = request; + params.push(...this.getParams(filter)); + + if (limit !== undefined) { + params.push(`limit=${limit}`); + } + if (orderFields !== undefined) { + (Array.isArray(orderFields) ? orderFields : [orderFields]).forEach( + ({ field, order }) => params.push(`sortField=${field},${order}`), + ); + } + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + + const normalizedFullTextFilterTerm = fullTextFilter?.term?.trim(); + if (normalizedFullTextFilterTerm) { + params.push(`fullTextFilterTerm=${normalizedFullTextFilterTerm}`); + } + if (fullTextFilter?.fields?.length) { + params.push(`fullTextFilterFields=${fullTextFilter.fields.join(',')}`); + } + } else { + const { fields = [], limit, cursor } = request; + + params.push(`cursor=${cursor}`); + if (limit !== undefined) { + params.push(`limit=${limit}`); + } + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + } + + const query = params.length ? `?${params.join('&')}` : ''; + return this.requestRequired( + 'GET', + `/entities/by-query${query}`, + options, + ); + } + /** * {@inheritdoc CatalogApi.getEntityByRef} */ @@ -290,30 +329,7 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise { const { filter = [], facets } = request; - const params: string[] = []; - - // filter param can occur multiple times, for example - // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' - // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters - // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of [filter].flat()) { - const filterParts: string[] = []; - for (const [key, value] of Object.entries(filterItem)) { - for (const v of [value].flat()) { - if (v === CATALOG_FILTER_EXISTS) { - filterParts.push(encodeURIComponent(key)); - } else if (typeof v === 'string') { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); - } - } - } - - if (filterParts.length) { - params.push(`filter=${filterParts.join(',')}`); - } - } + const params = this.getParams(filter); for (const facet of facets) { params.push(`facet=${encodeURIComponent(facet)}`); @@ -466,11 +482,11 @@ export class CatalogClient implements CatalogApi { } } - private async requestRequired( + private async requestRequired( method: string, path: string, options?: CatalogRequestOptions, - ): Promise { + ): Promise { const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; const headers: Record = options?.token ? { Authorization: `Bearer ${options.token}` } @@ -481,7 +497,7 @@ export class CatalogClient implements CatalogApi { throw await ResponseError.fromResponse(response); } - return await response.json(); + return response.json(); } private async requestOptional( @@ -504,4 +520,31 @@ export class CatalogClient implements CatalogApi { return await response.json(); } + + private getParams(filter: EntityFilterQuery = []) { + const params: string[] = []; + // filter param can occur multiple times, for example + // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' + // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters + // the "inner array" defined within a `filter` param corresponds to "allOf" filters + for (const filterItem of [filter].flat()) { + const filterParts: string[] = []; + for (const [key, value] of Object.entries(filterItem)) { + for (const v of [value].flat()) { + if (v === CATALOG_FILTER_EXISTS) { + filterParts.push(encodeURIComponent(key)); + } else if (typeof v === 'string') { + filterParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, + ); + } + } + } + + if (filterParts.length) { + params.push(`filter=${filterParts.join(',')}`); + } + } + return params; + } } diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index d662a9d106..ee6579efca 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -287,10 +287,7 @@ export interface GetEntityFacetsRequest { * (exported from this package), which means that you assert on the existence * of that key, no matter what its value is. */ - filter?: - | Record[] - | Record - | undefined; + filter?: EntityFilterQuery; /** * Dot separated paths for the facets to extract from each entity. * @@ -380,6 +377,67 @@ export type ValidateEntityResponse = | { valid: true } | { valid: false; errors: SerializedError[] }; +/** + * The request type for {@link CatalogClient.queryEntities}. + * + * @public + */ +export type QueryEntitiesRequest = + | QueryEntitiesInitialRequest + | QueryEntitiesCursorRequest; + +/** + * A request type for {@link CatalogClient.queryEntities}. + * The method takes this type in an initial pagination request, + * when requesting the first batch of entities. + * + * The properties filter, sortField, query and sortFieldOrder, are going + * to be immutable for the entire lifecycle of the following requests. + * + * @public + */ +export type QueryEntitiesInitialRequest = { + fields?: string[]; + limit?: number; + filter?: EntityFilterQuery; + orderFields?: EntityOrderQuery; + fullTextFilter?: { + term: string; + fields?: string[]; + }; +}; + +/** + * A request type for {@link CatalogClient.queryEntities}. + * The method takes this type in a pagination request, following + * the initial request. + * + * @public + */ +export type QueryEntitiesCursorRequest = { + fields?: string[]; + limit?: number; + cursor: string; +}; + +/** + * The response type for {@link CatalogClient.queryEntities}. + * + * @public + */ +export type QueryEntitiesResponse = { + /* The list of entities for the current request */ + items: Entity[]; + /* The number of entities among all the requests */ + totalItems: number; + pageInfo: { + /* The cursor for the next batch of entities */ + nextCursor?: string; + /* The cursor for the previous batch of entities */ + prevCursor?: string; + }; +}; + /** * A client for interacting with the Backstage software catalog through its API. * @@ -414,6 +472,49 @@ export interface CatalogApi { options?: CatalogRequestOptions, ): Promise; + /** + * Gets paginated entities from the catalog. + * + * @remarks + * + * @example + * + * ``` + * const response = await catalogClient.queryEntities({ + * filter: [{ kind: 'group' }], + * limit: 20, + * fullTextFilter: { + * term: 'A', + * } + * orderFields: { field: 'metadata.name' order: 'asc' }, + * }); + * ``` + * + * this will match all entities of type group having a name starting + * with 'A', ordered by name ascending. + * + * The response will contain a maximum of 20 entities. In case + * more than 20 entities exist, the response will contain a nextCursor + * property that can be used to fetch the next batch of entities. + * + * ``` + * const secondBatchResponse = await catalogClient + * .queryEntities({ cursor: response.nextCursor }); + * ``` + * + * secondBatchResponse will contain the next batch of (maximum) 20 entities, + * together with a prevCursor property, useful to fetch the previous batch. + * + * @public + * + * @param request - Request parameters + * @param options - Additional options + */ + queryEntities( + request?: QueryEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; + /** * Gets entity ancestor information, i.e. the hierarchy of parent entities * whose processing resulted in a given entity appearing in the catalog. diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 81b985eb42..9bf35886c8 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -33,5 +33,9 @@ export type { GetEntityFacetsResponse, Location, ValidateEntityResponse, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, + QueryEntitiesResponse, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts new file mode 100644 index 0000000000..de21cea0d4 --- /dev/null +++ b/packages/catalog-client/src/utils.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, +} from './types/api'; + +export function isQueryEntitiesInitialRequest( + request: QueryEntitiesInitialRequest, +): request is QueryEntitiesInitialRequest { + return !(request as QueryEntitiesCursorRequest).cursor; +} + +export function isQueryEntitiesCursorRequest( + request: QueryEntitiesRequest, +): request is QueryEntitiesCursorRequest { + return !isQueryEntitiesInitialRequest(request); +} diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index faf7451acd..62c8fad605 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -174,6 +174,13 @@ export interface EntitiesCatalog { */ entitiesBatch(request: EntitiesBatchRequest): Promise; + /** + * Fetch entities and scroll back and forth between entities. + * + * @param request + */ + queryEntities(request: QueryEntitiesRequest): Promise; + /** * Removes a single entity. * @@ -202,3 +209,103 @@ export interface EntitiesCatalog { */ facets(request: EntityFacetsRequest): Promise; } + +/** + * The request shape for {@link EntitiesCatalog.queryEntities}. + */ +export type QueryEntitiesRequest = + | QueryEntitiesInitialRequest + | QueryEntitiesCursorRequest; + +/** + * The initial request for {@link EntitiesCatalog.queryEntities}. + * The request take immutable properties that are going to be bound + * for the current and the next pagination requests. + */ +export interface QueryEntitiesInitialRequest { + authorizationToken?: string; + fields?: (entity: Entity) => Entity; + limit?: number; + filter?: EntityFilter; + orderFields?: EntityOrder[]; + fullTextFilter?: { + term: string; + fields?: string[]; + }; +} + +/** + * Request for {@link EntitiesCatalog.queryEntities} used to + * move forward or backward on the data. + */ +export interface QueryEntitiesCursorRequest { + authorizationToken?: string; + fields?: (entity: Entity) => Entity; + limit?: number; + cursor: Cursor; +} + +/** + * The response shape for {@link EntitiesCatalog.queryEntities}. + */ +export interface QueryEntitiesResponse { + /** + * The entities for the current pagination request + */ + items: Entity[]; + + pageInfo: { + /** + * The cursor of the next pagination request. + */ + nextCursor?: Cursor; + /** + * The cursor of the previous pagination request. + */ + prevCursor?: Cursor; + }; + /** + * the total number of entities matching the current filters. + */ + totalItems: number; +} + +/** + * The Cursor used internally by the catalog. + */ +export type Cursor = { + /** + * An array of fields used for sorting the data. + * For example, [ { field: 'metadata.name', order: 'asc' } ] + */ + orderFields: EntityOrder[]; + /** + * The values of the fields of a specific item used for paginating the data. + */ + orderFieldValues: Array; + /** + * A filter to be applied to the full list of entities. + */ + filter?: EntityFilter; + /** + * true if the cursor is a previous cursor. + */ + isPrevious: boolean; + /** + * Used for performing full text filtering on the given fields. + */ + fullTextFilter?: { + term: string; + fields?: string[]; + }; + /** + * The value of the fields of the first returned item used for paginating the data. + * The catalog uses this field internally to understand if the beginning + * of the list has been reached when performing cursor based pagination. + */ + firstSortFieldValues?: Array; + /** + * The number of items that match the provided filters + */ + totalItems?: number; +}; diff --git a/plugins/catalog-backend/src/integration.test.ts b/plugins/catalog-backend/src/integration.test.ts index f0f916a78d..b13e6ef9de 100644 --- a/plugins/catalog-backend/src/integration.test.ts +++ b/plugins/catalog-backend/src/integration.test.ts @@ -268,7 +268,11 @@ class TestHarness { legacySingleProcessorValidation: false, }); const stitcher = new Stitcher(db, logger); - const catalog = new DefaultEntitiesCatalog(db, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: db, + logger, + stitcher, + }); const proxyProgressTracker = new ProxyProgressTracker( new NoopProgressTracker(), ); diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 829e41c21e..2ee47dae19 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -20,6 +20,8 @@ import { createConditionTransformer } from '@backstage/plugin-permission-node'; import { isEntityKind } from '../permissions/rules/isEntityKind'; import { CatalogPermissionRule } from '../permissions/rules'; import { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog'; +import { Cursor, EntityFilter, QueryEntitiesResponse } from '../catalog/types'; +import { Entity } from '@backstage/catalog-model'; describe('AuthorizedEntitiesCatalog', () => { const fakeCatalog = { @@ -30,6 +32,7 @@ describe('AuthorizedEntitiesCatalog', () => { facets: jest.fn(), refresh: jest.fn(), listAncestors: jest.fn(), + queryEntities: jest.fn(), }; const fakePermissionApi = { authorize: jest.fn(), @@ -156,6 +159,153 @@ describe('AuthorizedEntitiesCatalog', () => { }); }); + describe('queryEntities', () => { + it('returns empty response on DENY', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.DENY }, + ]); + const catalog = createCatalog(); + + await expect( + catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }), + ).resolves.toEqual({ + items: [], + pageInfo: {}, + totalItems: 0, + }); + + expect(fakeCatalog.queryEntities).not.toHaveBeenCalled(); + }); + + it('calls underlying catalog method on ALLOW', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.ALLOW }, + ]); + const catalog = createCatalog(); + + await catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + }); + + it('calls underlying catalog method with correct filter on CONDITIONAL', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { + result: AuthorizeResult.CONDITIONAL, + conditions: { + rule: 'IS_ENTITY_KIND', + params: { kinds: ['b'] }, + }, + }, + ]); + + const requestFilter: EntityFilter = { key: 'name', values: ['name'] }; + + const entities = [ + { + kind: 'component', + namespace: 'default', + name: 'a', + } as unknown as Entity, + { + kind: 'component', + namespace: 'default', + name: 'b1', + } as unknown as Entity, + ]; + + fakeCatalog.queryEntities.mockResolvedValue({ + items: entities, + pageInfo: { + nextCursor: { + isPrevious: false, + orderFieldValues: ['xxx', null], + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + prevCursor: { + isPrevious: true, + orderFieldValues: ['a', null], + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + }, + totalItems: 4, + } as QueryEntitiesResponse); + const catalog = createCatalog(isEntityKind); + + let response = await catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'name', values: ['name'] }, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({ + authorizationToken: 'abcd', + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }); + + expect(response).toEqual({ + items: entities, + totalItems: 4, + pageInfo: { + nextCursor: { + isPrevious: false, + filter: requestFilter, + orderFieldValues: ['xxx', null], + }, + prevCursor: { + isPrevious: true, + filter: requestFilter, + orderFieldValues: ['a', null], + }, + }, + }); + + const cursor: Cursor = { + filter: requestFilter, + orderFields: [{ field: 'name', order: 'asc' }], + isPrevious: false, + orderFieldValues: ['a', null], + }; + response = await catalog.queryEntities({ + authorizationToken: 'abcd', + cursor, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenNthCalledWith(2, { + authorizationToken: 'abcd', + cursor: { + ...cursor, + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + }); + + expect(response).toEqual({ + items: entities, + totalItems: 4, + pageInfo: { + nextCursor: { + isPrevious: false, + filter: requestFilter, + orderFieldValues: ['xxx', null], + }, + prevCursor: { + isPrevious: true, + filter: requestFilter, + orderFieldValues: ['a', null], + }, + }, + }); + }); + }); + describe('removeEntityByUid', () => { it('throws error on DENY', async () => { fakeCatalog.entities.mockResolvedValue({ diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index c62e9f9a84..2649b45260 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -26,6 +26,7 @@ import { } from '@backstage/plugin-permission-common'; import { ConditionTransformer } from '@backstage/plugin-permission-node'; import { + Cursor, EntitiesBatchRequest, EntitiesBatchResponse, EntitiesCatalog, @@ -35,8 +36,11 @@ import { EntityFacetsRequest, EntityFacetsResponse, EntityFilter, + QueryEntitiesRequest, + QueryEntitiesResponse, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; +import { isQueryEntitiesCursorRequest } from './util'; export class AuthorizedEntitiesCatalog implements EntitiesCatalog { constructor( @@ -106,6 +110,80 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { return this.entitiesCatalog.entitiesBatch(request); } + async queryEntities( + request: QueryEntitiesRequest, + ): Promise { + const authorizeDecision = ( + await this.permissionApi.authorizeConditional( + [{ permission: catalogEntityReadPermission }], + { token: request.authorizationToken }, + ) + )[0]; + + if (authorizeDecision.result === AuthorizeResult.DENY) { + return { + items: [], + pageInfo: {}, + totalItems: 0, + }; + } + + if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) { + const permissionFilter: EntityFilter = this.transformConditions( + authorizeDecision.conditions, + ); + + let permissionedRequest: QueryEntitiesRequest; + let requestFilter: EntityFilter | undefined; + + if (isQueryEntitiesCursorRequest(request)) { + requestFilter = request.cursor.filter; + + permissionedRequest = { + ...request, + cursor: { + ...request.cursor, + filter: request.cursor.filter + ? { allOf: [permissionFilter, request.cursor.filter] } + : permissionFilter, + }, + }; + } else { + permissionedRequest = { + ...request, + filter: request.filter + ? { allOf: [permissionFilter, request.filter] } + : permissionFilter, + }; + requestFilter = request.filter; + } + + const response = await this.entitiesCatalog.queryEntities( + permissionedRequest, + ); + + const prevCursor: Cursor | undefined = response.pageInfo.prevCursor && { + ...response.pageInfo.prevCursor, + filter: requestFilter, + }; + + const nextCursor: Cursor | undefined = response.pageInfo.nextCursor && { + ...response.pageInfo.nextCursor, + filter: requestFilter, + }; + + return { + ...response, + pageInfo: { + prevCursor, + nextCursor, + }, + }; + } + + return this.entitiesCatalog.queryEntities(request); + } + async removeEntityByUid( uid: string, options?: { authorizationToken?: string }, diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 48bf6a08ef..3dfdec09e5 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -453,10 +453,11 @@ export class CatalogBuilder { legacySingleProcessorValidation: this.legacySingleProcessorValidation, }); const stitcher = new Stitcher(dbClient, logger); - const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog( - dbClient, + const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog({ + database: dbClient, + logger, stitcher, - ); + }); let permissionEvaluator: PermissionEvaluator; if ('authorizeConditional' in permissions) { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 6735709662..740f5b2558 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -14,10 +14,15 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; -import { v4 as uuid } from 'uuid'; +import { v4 as uuid, v4 } from 'uuid'; +import { + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, +} from '../catalog/types'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -31,6 +36,12 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { EntitiesRequest } from '../catalog/types'; describe('DefaultEntitiesCatalog', () => { + let knex: Knex; + + afterEach(async () => { + await knex.destroy(); + }); + const databases = TestDatabases.create({ ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); @@ -38,13 +49,11 @@ describe('DefaultEntitiesCatalog', () => { const stitcher: Stitcher = { stitch } as any; async function createDatabase(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); + knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); - return { knex }; } async function addEntity( - knex: Knex, entity: Entity, parents: { source?: string; entity?: Entity }[], ) { @@ -81,8 +90,8 @@ describe('DefaultEntitiesCatalog', () => { return id; } - async function addEntityToSearch(knex: Knex, entity: Entity) { - const id = uuid(); + async function addEntityToSearch(entity: Entity) { + const id = entity.metadata.uid || v4(); const entityRef = stringifyEntityRef(entity); const entityJson = JSON.stringify(entity); @@ -120,7 +129,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return the ancestry with one parent, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const grandparent: Entity = { apiVersion: 'a', @@ -141,11 +150,15 @@ describe('DefaultEntitiesCatalog', () => { spec: {}, }; - await addEntity(knex, grandparent, [{ source: 's' }]); - await addEntity(knex, parent, [{ entity: grandparent }]); - await addEntity(knex, root, [{ entity: parent }]); + await addEntity(grandparent, [{ source: 's' }]); + await addEntity(parent, [{ entity: grandparent }]); + await addEntity(root, [{ entity: parent }]); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const result = await catalog.entityAncestry('k:default/root'); expect(result.rootEntityRef).toEqual('k:default/root'); @@ -174,8 +187,12 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should throw error if the entity does not exist, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + await createDatabase(databaseId); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect(() => catalog.entityAncestry('k:default/root'), ).rejects.toThrow('No such entity k:default/root'); @@ -186,7 +203,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return the ancestry with multiple parents, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const grandparent: Entity = { apiVersion: 'a', @@ -213,12 +230,16 @@ describe('DefaultEntitiesCatalog', () => { spec: {}, }; - await addEntity(knex, grandparent, [{ source: 's' }]); - await addEntity(knex, parent1, [{ entity: grandparent }]); - await addEntity(knex, parent2, [{ entity: grandparent }]); - await addEntity(knex, root, [{ entity: parent1 }, { entity: parent2 }]); + await addEntity(grandparent, [{ source: 's' }]); + await addEntity(parent1, [{ entity: grandparent }]); + await addEntity(parent2, [{ entity: grandparent }]); + await addEntity(root, [{ entity: parent1 }, { entity: parent2 }]); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const result = await catalog.entityAncestry('k:default/root'); expect(result.rootEntityRef).toEqual('k:default/root'); @@ -257,7 +278,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entity for simple filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -272,9 +293,13 @@ describe('DefaultEntitiesCatalog', () => { test: 'test value', }, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter = { key: 'spec.test', @@ -291,7 +316,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entity for negation filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -306,9 +331,13 @@ describe('DefaultEntitiesCatalog', () => { test: 'test value', }, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter = { not: { @@ -327,7 +356,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entities for nested filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -352,11 +381,15 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'four', org: 'b', color: 'blue' }, spec: {}, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - await addEntityToSearch(knex, entity3); - await addEntityToSearch(knex, entity4); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + await addEntityToSearch(entity3); + await addEntityToSearch(entity4); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter1 = { key: 'metadata.org', @@ -397,7 +430,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entities for complex negation filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -410,9 +443,13 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'two', org: 'b', desc: 'description' }, spec: {}, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter1 = { key: 'metadata.org', @@ -440,7 +477,7 @@ describe('DefaultEntitiesCatalog', () => { 'should return no matches for an empty values array, %p', // NOTE: An empty values array is not a sensible input in a realistic scenario. async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -453,9 +490,13 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'two' }, spec: {}, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter = { key: 'kind', @@ -472,9 +513,8 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return both target and targetRef for entities', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -485,7 +525,6 @@ describe('DefaultEntitiesCatalog', () => { [], ); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -500,7 +539,11 @@ describe('DefaultEntitiesCatalog', () => { }, [], ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const { entities } = await catalog.entities(); @@ -529,7 +572,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can order and combine with filtering, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', @@ -555,12 +598,16 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'n4' }, spec: { a: 'baz', b: 'only' }, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - await addEntityToSearch(knex, entity3); - await addEntityToSearch(knex, entity4); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + await addEntityToSearch(entity3); + await addEntityToSearch(entity4); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); function f(request: EntitiesRequest): Promise { return catalog @@ -623,10 +670,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'queries for entities by ref, including duplicates, and gracefully returns null for missing entities, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -637,7 +683,6 @@ describe('DefaultEntitiesCatalog', () => { [], ); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -648,7 +693,11 @@ describe('DefaultEntitiesCatalog', () => { [], ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const { items } = await catalog.entitiesBatch({ entityRefs: [ @@ -674,11 +723,581 @@ describe('DefaultEntitiesCatalog', () => { ); }); + describe('queryEntities', () => { + it.each(databases.eachSupportedId())( + 'should return paginated entities and scroll the items accordingly, %p', + async databaseId => { + await createDatabase(databaseId); + + const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; + const entities: Entity[] = names.map(name => entityFrom(name)); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), + ); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const filter = { + key: 'spec.should_include_this', + }; + + const limit = 2; + + // initial request + const request1: QueryEntitiesInitialRequest = { + filter, + limit, + orderFields: [{ field: 'metadata.name', order: 'asc' }], + }; + const response1 = await catalog.queryEntities(request1); + expect(response1.items).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // second request (forward) + const request2: QueryEntitiesCursorRequest = { + cursor: response1.pageInfo.nextCursor!, + limit, + }; + const response2 = await catalog.queryEntities(request2); + expect(response2.items).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); + expect(response2.totalItems).toBe(names.length); + + // third request (forward) + const request3: QueryEntitiesCursorRequest = { + cursor: response2.pageInfo.nextCursor!, + limit, + }; + const response3 = await catalog.queryEntities(request3); + expect(response3.items).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response3.pageInfo.nextCursor).toBeDefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); + expect(response3.totalItems).toBe(names.length); + + // fourth request (backwards) + const request4: QueryEntitiesCursorRequest = { + cursor: response3.pageInfo.prevCursor!, + limit, + }; + const response4 = await catalog.queryEntities(request4); + expect(response4.items).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); + expect(response4.totalItems).toBe(names.length); + + // fifth request (backwards) + const request5: QueryEntitiesCursorRequest = { + cursor: response4.pageInfo.prevCursor!, + limit, + }; + const response5 = await catalog.queryEntities(request5); + expect(response5.items).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); + expect(response5.totalItems).toBe(names.length); + + // sixth request (forward) + const request6: QueryEntitiesCursorRequest = { + cursor: response5.pageInfo.nextCursor!, + limit, + }; + const response6 = await catalog.queryEntities(request6); + expect(response6.items).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response6.pageInfo.nextCursor).toBeDefined(); + expect(response6.pageInfo.prevCursor).toBeDefined(); + expect(response6.totalItems).toBe(names.length); + + // seventh request (forward) + const request7: QueryEntitiesCursorRequest = { + cursor: response6.pageInfo.nextCursor!, + limit, + }; + const response7 = await catalog.queryEntities(request7); + expect(response7.items).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response7.pageInfo.nextCursor).toBeDefined(); + expect(response7.pageInfo.prevCursor).toBeDefined(); + expect(response7.totalItems).toBe(names.length); + + // seventh.2 request (forward with a different limit) + const request7bis: QueryEntitiesCursorRequest = { + cursor: response6.pageInfo.nextCursor!, + limit: limit + 1, + }; + const response7bis = await catalog.queryEntities(request7bis); + expect(response7bis.items).toEqual([ + entityFrom('E'), + entityFrom('F'), + entityFrom('G'), + ]); + expect(response7bis.pageInfo.nextCursor).toBeUndefined(); + expect(response7bis.pageInfo.prevCursor).toBeDefined(); + expect(response7bis.totalItems).toBe(names.length); + + // last request (forward) + const request8: QueryEntitiesCursorRequest = { + cursor: response7.pageInfo.nextCursor!, + limit, + }; + const response8 = await catalog.queryEntities(request8); + expect(response8.items).toEqual([entityFrom('G')]); + expect(response8.pageInfo.nextCursor).toBeUndefined(); + expect(response8.pageInfo.prevCursor).toBeDefined(); + expect(response8.totalItems).toBe(names.length); + }, + ); + + it.each(databases.eachSupportedId())( + 'should return paginated entities ordered in descending order and scroll the items accordingly, %p', + async databaseId => { + await createDatabase(databaseId); + + const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; + const entities: Entity[] = names.map(name => entityFrom(name)); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), + ); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const filter = { + key: 'spec.should_include_this', + }; + + const limit = 2; + + // initial request + const request1: QueryEntitiesInitialRequest = { + filter, + limit, + orderFields: [{ field: 'metadata.name', order: 'desc' }], + }; + const response1 = await catalog.queryEntities(request1); + expect(response1.items).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // second request (forward) + const request2: QueryEntitiesCursorRequest = { + cursor: response1.pageInfo.nextCursor!, + limit, + }; + const response2 = await catalog.queryEntities(request2); + expect(response2.items).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); + expect(response2.totalItems).toBe(names.length); + + // third request (forward) + const request3: QueryEntitiesCursorRequest = { + cursor: response2.pageInfo.nextCursor!, + limit, + }; + const response3 = await catalog.queryEntities(request3); + expect(response3.items).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response3.pageInfo.nextCursor).toBeDefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); + expect(response3.totalItems).toBe(names.length); + + // fourth request (backwards) + const request4: QueryEntitiesCursorRequest = { + cursor: response3.pageInfo.prevCursor!, + limit, + }; + const response4 = await catalog.queryEntities(request4); + + expect(response4.items).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); + expect(response4.totalItems).toBe(names.length); + + // fifth request (backwards) + const request5: QueryEntitiesCursorRequest = { + cursor: response4.pageInfo.prevCursor!, + limit, + }; + const response5 = await catalog.queryEntities(request5); + expect(response5.items).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); + expect(response5.totalItems).toBe(names.length); + + // sixth request (forward) + const request6: QueryEntitiesCursorRequest = { + cursor: response5.pageInfo.nextCursor!, + limit, + }; + const response6 = await catalog.queryEntities(request6); + expect(response6.items).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response6.pageInfo.nextCursor).toBeDefined(); + expect(response6.pageInfo.prevCursor).toBeDefined(); + expect(response6.totalItems).toBe(names.length); + + // seventh request (forward) + const request7: QueryEntitiesCursorRequest = { + cursor: response6.pageInfo.nextCursor!, + limit, + }; + const response7 = await catalog.queryEntities(request7); + expect(response7.items).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response7.pageInfo.nextCursor).toBeDefined(); + expect(response7.pageInfo.prevCursor).toBeDefined(); + expect(response7.totalItems).toBe(names.length); + + // seventh.2 request (forward with a different limit) + const request7bis: QueryEntitiesCursorRequest = { + cursor: response6.pageInfo.nextCursor!, + limit: limit + 1, + }; + const response7bis = await catalog.queryEntities(request7bis); + expect(response7bis.items).toEqual([ + entityFrom('C'), + entityFrom('B'), + entityFrom('A'), + ]); + expect(response7bis.pageInfo.nextCursor).toBeUndefined(); + expect(response7bis.pageInfo.prevCursor).toBeDefined(); + expect(response7bis.totalItems).toBe(names.length); + + // last request (forward) + const request8: QueryEntitiesCursorRequest = { + cursor: response7.pageInfo.nextCursor!, + limit, + }; + const response8 = await catalog.queryEntities(request8); + expect(response8.items).toEqual([entityFrom('A')]); + expect(response8.pageInfo.nextCursor).toBeUndefined(); + expect(response8.pageInfo.prevCursor).toBeDefined(); + expect(response8.totalItems).toBe(names.length); + }, + ); + + it.each(databases.eachSupportedId())( + 'should filter the results when query is provided, %p', + async databaseId => { + await createDatabase(databaseId); + + const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's']; + const entities: Entity[] = names.map(name => entityFrom(name)); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), + ); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const filter = { + key: 'spec.should_include_this', + }; + + const request: QueryEntitiesInitialRequest = { + filter, + limit: 100, + + orderFields: [{ field: 'metadata.name', order: 'asc' }], + fullTextFilter: { term: 'cAt ' }, + }; + const response = await catalog.queryEntities(request); + expect(response.items).toEqual([ + entityFrom('atcatss'), + entityFrom('cat'), + entityFrom('dogcat'), + ]); + expect(response.pageInfo.nextCursor).toBeUndefined(); + expect(response.pageInfo.prevCursor).toBeUndefined(); + expect(response.totalItems).toBe(3); + }, + ); + + it.each(databases.eachSupportedId())( + 'should include totalItems and empty entities in the response in case limit is zero, %p', + async databaseId => { + await createDatabase(databaseId); + + await Promise.all( + Array(20) + .fill(0) + .map(() => + addEntityToSearch({ + apiVersion: 'a', + kind: 'k', + metadata: { name: v4() }, + }), + ), + ); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const request: QueryEntitiesInitialRequest = { + limit: 0, + }; + const response = await catalog.queryEntities(request); + expect(response).toEqual({ totalItems: 20, items: [], pageInfo: {} }); + }, + ); + + it.each(databases.eachSupportedId())( + 'should paginate results accordingly in case of clashing items, %p', + async databaseId => { + await createDatabase(databaseId); + + await Promise.all([ + addEntityToSearch(entityFrom('AA')), + addEntityToSearch(entityFrom('AA', { namespace: 'namespace2' })), + addEntityToSearch(entityFrom('AA', { namespace: 'namespace3' })), + addEntityToSearch(entityFrom('AA', { namespace: 'namespace4' })), + addEntityToSearch(entityFrom('CC')), + addEntityToSearch(entityFrom('DD')), + ]); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const limit = 2; + + // initial request + const request1: QueryEntitiesInitialRequest = { + limit, + orderFields: [{ field: 'metadata.name', order: 'asc' }], + }; + const response1 = await catalog.queryEntities(request1); + expect(response1.items).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(6); + + // second request (forward) + const request2: QueryEntitiesCursorRequest = { + cursor: response1.pageInfo.nextCursor!, + limit, + }; + const response2 = await catalog.queryEntities(request2); + expect(response2.items).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); + expect(response2.totalItems).toBe(6); + + // third request (forward) + const request3: QueryEntitiesCursorRequest = { + cursor: response2.pageInfo.nextCursor!, + limit, + }; + const response3 = await catalog.queryEntities(request3); + expect(response3.items).toEqual([entityFrom('CC'), entityFrom('DD')]); + expect(response3.pageInfo.nextCursor).toBeUndefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); + expect(response3.totalItems).toBe(6); + + // forth request (backward) + const request4: QueryEntitiesCursorRequest = { + cursor: response3.pageInfo.prevCursor!, + limit, + }; + const response4 = await catalog.queryEntities(request4); + expect(response4.items).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); + expect(response4.totalItems).toBe(6); + + // fifth request (backward) + const request5: QueryEntitiesCursorRequest = { + cursor: response4.pageInfo.prevCursor!, + limit, + }; + const response5 = await catalog.queryEntities(request5); + expect(response5.items).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); + expect(response5.totalItems).toBe(6); + }, + ); + + it.each(databases.eachSupportedId())( + 'should paginate results without sort fields, %p', + async databaseId => { + await createDatabase(databaseId); + + await Promise.all([ + addEntityToSearch(entityFrom('AA', { uid: 'id1' })), + addEntityToSearch(entityFrom('CC', { uid: 'id2' })), + addEntityToSearch( + entityFrom('AA', { namespace: 'namespace2', uid: 'id4' }), + ), + addEntityToSearch( + entityFrom('AA', { namespace: 'namespace3', uid: 'id5' }), + ), + addEntityToSearch( + entityFrom('AA', { namespace: 'namespace4', uid: 'id6' }), + ), + addEntityToSearch(entityFrom('DD', { uid: 'id3' })), + ]); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const limit = 2; + + // initial request + const request1: QueryEntitiesInitialRequest = { + limit, + }; + const response1 = await catalog.queryEntities(request1); + expect(response1.items).toMatchObject([ + entityFrom('AA'), + entityFrom('CC'), + ]); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(6); + + // second request (forward) + const request2: QueryEntitiesCursorRequest = { + cursor: response1.pageInfo.nextCursor!, + limit, + }; + const response2 = await catalog.queryEntities(request2); + expect(response2.items).toMatchObject([ + entityFrom('DD'), + entityFrom('AA', { namespace: 'namespace2' }), + ]); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); + expect(response2.totalItems).toBe(6); + + // third request (forward) + const request3: QueryEntitiesCursorRequest = { + cursor: response2.pageInfo.nextCursor!, + limit, + }; + const response3 = await catalog.queryEntities(request3); + expect(response3.items).toMatchObject([ + entityFrom('AA', { namespace: 'namespace3' }), + entityFrom('AA', { namespace: 'namespace4' }), + ]); + expect(response3.pageInfo.nextCursor).toBeUndefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); + expect(response3.totalItems).toBe(6); + + // forth request (backward) + const request4: QueryEntitiesCursorRequest = { + cursor: response3.pageInfo.prevCursor!, + limit, + }; + const response4 = await catalog.queryEntities(request4); + expect(response4.items).toMatchObject([ + entityFrom('DD'), + entityFrom('AA', { namespace: 'namespace2' }), + ]); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); + expect(response4.totalItems).toBe(6); + + // fifth request (backward) + const request5: QueryEntitiesCursorRequest = { + cursor: response4.pageInfo.prevCursor!, + limit, + }; + const response5 = await catalog.queryEntities(request5); + expect(response5.items).toMatchObject([ + entityFrom('AA'), + entityFrom('CC'), + ]); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); + expect(response5.totalItems).toBe(6); + }, + ); + }); + describe('removeEntityByUid', () => { it.each(databases.eachSupportedId())( 'also clears parent hashes, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const grandparent: Entity = { apiVersion: 'a', @@ -717,15 +1336,15 @@ describe('DefaultEntitiesCatalog', () => { spec: {}, }; - await addEntity(knex, grandparent, [{ source: 's' }]); - await addEntity(knex, parent1, [{ entity: grandparent }]); - await addEntity(knex, parent2, [{ entity: grandparent }]); - const uid = await addEntity(knex, root, [ + await addEntity(grandparent, [{ source: 's' }]); + await addEntity(parent1, [{ entity: grandparent }]); + await addEntity(parent2, [{ entity: grandparent }]); + const uid = await addEntity(root, [ { entity: parent1 }, { entity: parent2 }, ]); - await addEntity(knex, unrelated1, []); - await addEntity(knex, unrelated2, []); + await addEntity(unrelated1, []); + await addEntity(unrelated2, []); await knex('refresh_state').update({ result_hash: 'not-changed' }); await knex('relations').insert({ originating_entity_id: uid, @@ -740,7 +1359,11 @@ describe('DefaultEntitiesCatalog', () => { target_entity_ref: 'k:default/root', }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await catalog.removeEntityByUid(uid); await expect( @@ -767,27 +1390,31 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can filter and collect properly, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { name: 'one' }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { name: 'two' }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k2', metadata: { name: 'two' }, spec: {}, }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect(catalog.facets({ facets: ['kind'] })).resolves.toEqual({ facets: { @@ -804,9 +1431,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can match on annotations and labels with dots in them, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -816,7 +1443,7 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -826,7 +1453,11 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect( catalog.facets({ @@ -851,9 +1482,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can match on strings in arrays, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -862,7 +1493,7 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -871,7 +1502,11 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect( catalog.facets({ @@ -891,3 +1526,19 @@ describe('DefaultEntitiesCatalog', () => { ); }); }); + +function entityFrom( + name: string, + { uid, namespace }: { uid?: string; namespace?: string } = {}, +) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { + name, + ...(!!namespace && { namespace }), + ...(!!uid && { uid }), + }, + spec: { should_include_this: 'yes' }, + }; +} diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index d598881371..f983d7496b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -20,11 +20,14 @@ import { stringifyEntityRef, } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; -import lodash from 'lodash'; import { Knex } from 'knex'; +import { isEqual, chunk as lodashChunk } from 'lodash'; +import { Logger } from 'winston'; +import { z } from 'zod'; import { EntitiesBatchRequest, EntitiesBatchResponse, + Cursor, EntitiesCatalog, EntitiesRequest, EntitiesResponse, @@ -34,6 +37,9 @@ import { EntityFacetsResponse, EntityFilter, EntityPagination, + QueryEntitiesRequest, + QueryEntitiesResponse, + EntityOrder, } from '../catalog/types'; import { DbFinalEntitiesRow, @@ -43,8 +49,21 @@ import { DbRelationsRow, DbSearchRow, } from '../database/tables'; + import { Stitcher } from '../stitching/Stitcher'; +import { + isQueryEntitiesCursorRequest, + isQueryEntitiesInitialRequest, +} from './util'; + +const defaultSortField: EntityOrder = { + field: 'metadata.uid', + order: 'asc', +}; + +const DEFAULT_LIMIT = 20; + function parsePagination(input?: EntityPagination): { limit?: number; offset?: number; @@ -168,10 +187,15 @@ function parseFilter( } export class DefaultEntitiesCatalog implements EntitiesCatalog { - constructor( - private readonly database: Knex, - private readonly stitcher: Stitcher, - ) {} + private readonly database: Knex; + private readonly logger: Logger; + private readonly stitcher: Stitcher; + + constructor(options: { database: Knex; logger: Logger; stitcher: Stitcher }) { + this.database = options.database; + this.logger = options.logger; + this.stitcher = options.stitcher; + } async entities(request?: EntitiesRequest): Promise { const db = this.database; @@ -283,7 +307,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ): Promise { const lookup = new Map(); - for (const chunk of lodash.chunk(request.entityRefs, 200)) { + for (const chunk of lodashChunk(request.entityRefs, 200)) { let query = this.database('final_entities') .innerJoin('refresh_state', { 'refresh_state.entity_id': 'final_entities.entity_id', @@ -310,6 +334,161 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { items }; } + async queryEntities( + request: QueryEntitiesRequest, + ): Promise { + const db = this.database; + + const limit = request.limit ?? DEFAULT_LIMIT; + + const cursor: Omit & { + orderFieldValues?: (string | null)[]; + } = { + orderFields: [defaultSortField], + isPrevious: false, + ...parseCursorFromRequest(request), + }; + + const isFetchingBackwards = cursor.isPrevious; + + if (cursor.orderFields.length > 1) { + this.logger.warn(`Only one sort field is supported, ignoring the rest`); + } + + const sortField: EntityOrder = { + ...defaultSortField, + ...cursor.orderFields[0], + }; + + const [prevItemOrderFieldValue, prevItemUid] = + cursor.orderFieldValues || []; + + const dbQuery = db('search') + .join('final_entities', 'search.entity_id', 'final_entities.entity_id') + .where('search.key', sortField.field); + + if (cursor.filter) { + parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id'); + } + + const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim(); + if (normalizedFullTextFilterTerm) { + dbQuery.andWhereRaw( + 'value like ?', + `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, + ); + } + + const countQuery = dbQuery.clone(); + + const isOrderingDescending = sortField.order === 'desc'; + + if (prevItemOrderFieldValue) { + dbQuery.andWhere( + 'value', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + prevItemOrderFieldValue, + ); + dbQuery.orWhere(function nested() { + this.where('value', '=', prevItemOrderFieldValue).andWhere( + 'search.entity_id', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + prevItemUid, + ); + }); + } + + dbQuery + .orderBy([ + { + column: 'value', + order: isFetchingBackwards + ? invertOrder(sortField.order) + : sortField.order, + }, + { + column: 'search.entity_id', + order: isFetchingBackwards + ? invertOrder(sortField.order) + : sortField.order, + }, + ]) + // fetch an extra item to check if there are more items. + .limit(isFetchingBackwards ? limit : limit + 1); + + countQuery.count('search.entity_id', { as: 'count' }); + + const [rows, [{ count }]] = await Promise.all([ + dbQuery, + // for performance reasons we invoke the countQuery + // only on the first request. + // The result is then embedded into the cursor + // for subsequent requests. + typeof cursor.totalItems === 'undefined' + ? countQuery + : [{ count: cursor.totalItems }], + ]); + + const totalItems = Number(count); + + if (isFetchingBackwards) { + rows.reverse(); + } + const hasMoreResults = + limit > 0 && (isFetchingBackwards || rows.length > limit); + + // discard the extra item only when fetching forward. + if (rows.length > limit) { + rows.length -= 1; + } + + const isInitialRequest = cursor.firstSortFieldValues === undefined; + + const firstRow = rows[0]; + const lastRow = rows[rows.length - 1]; + + const firstSortFieldValues = cursor.firstSortFieldValues || [ + firstRow?.value, + firstRow?.entity_id, + ]; + + const nextCursor: Cursor | undefined = hasMoreResults + ? { + ...cursor, + orderFieldValues: sortFieldsFromRow(lastRow), + firstSortFieldValues, + isPrevious: false, + totalItems, + } + : undefined; + + const prevCursor: Cursor | undefined = + !isInitialRequest && + rows.length > 0 && + !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues) + ? { + ...cursor, + orderFieldValues: sortFieldsFromRow(firstRow), + firstSortFieldValues: cursor.firstSortFieldValues, + isPrevious: true, + totalItems, + } + : undefined; + + const items = rows + .map(e => JSON.parse(e.final_entity!)) + .map(e => (request.fields ? request.fields(e) : e)); + + return { + items, + pageInfo: { + ...(!!prevCursor && { prevCursor }), + ...(!!nextCursor && { nextCursor }), + }, + totalItems, + }; + } + async removeEntityByUid(uid: string): Promise { const dbConfig = this.database.client.config; @@ -490,3 +669,51 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { facets }; } } + +const entityFilterParser: z.ZodSchema = z.lazy(() => + z + .object({ + key: z.string(), + values: z.array(z.string()).optional(), + }) + .or(z.object({ not: entityFilterParser })) + .or(z.object({ anyOf: z.array(entityFilterParser) })) + .or(z.object({ allOf: z.array(entityFilterParser) })), +); + +export const cursorParser: z.ZodSchema = z.object({ + orderFields: z.array( + z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }), + ), + orderFieldValues: z.array(z.string().or(z.null())), + filter: entityFilterParser.optional(), + isPrevious: z.boolean(), + query: z.string().optional(), + firstSortFieldValues: z.array(z.string().or(z.null())).optional(), + totalItems: z.number().optional(), +}); + +function parseCursorFromRequest( + request?: QueryEntitiesRequest, +): Partial { + if (isQueryEntitiesInitialRequest(request)) { + const { + filter, + orderFields: sortFields = [defaultSortField], + fullTextFilter, + } = request; + return { filter, orderFields: sortFields, fullTextFilter }; + } + if (isQueryEntitiesCursorRequest(request)) { + return request.cursor; + } + return {}; +} + +function invertOrder(order: EntityOrder['order']) { + return order === 'asc' ? 'desc' : 'asc'; +} + +function sortFieldsFromRow(row: DbSearchRow) { + return [row.value, row.entity_id]; +} diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 6379d65043..f2f87fa800 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -25,7 +25,7 @@ import { } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; -import { EntitiesCatalog } from '../catalog/types'; +import { Cursor, EntitiesCatalog } from '../catalog/types'; import { LocationInput, LocationService, RefreshService } from './types'; import { basicEntityFilter } from './request'; import { createRouter } from './createRouter'; @@ -37,6 +37,7 @@ import { import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common'; import { CatalogProcessingOrchestrator } from '../processing/types'; import { z } from 'zod'; +import { encodeCursor } from './util'; describe('createRouter readonly disabled', () => { let entitiesCatalog: jest.Mocked; @@ -52,6 +53,7 @@ describe('createRouter readonly disabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + queryEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -135,6 +137,118 @@ describe('createRouter readonly disabled', () => { }); }); + describe('GET /entities/by-query', () => { + it('happy path: lists entities', async () => { + const items: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items, + totalItems: 100, + pageInfo: { nextCursor: mockCursor() }, + }); + + const response = await request(app).get('/entities/by-query'); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + items, + totalItems: 100, + pageInfo: { + nextCursor: expect.any(String), + }, + }); + }); + + it('parses initial request', async () => { + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items: [], + pageInfo: {}, + totalItems: 0, + }); + const response = await request(app).get( + '/entities/by-query?filter=a=1,a=2,b=3&filter=c=4&orderField=metadata.name,asc&orderField=metadata.uid,desc', + ); + + expect(response.status).toEqual(200); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ + filter: { + anyOf: [ + { + allOf: [ + { key: 'a', values: ['1', '2'] }, + { key: 'b', values: ['3'] }, + ], + }, + { allOf: [{ key: 'c', values: ['4'] }] }, + ], + }, + orderFields: [ + { field: 'metadata.name', order: 'asc' }, + { field: 'metadata.uid', order: 'desc' }, + ], + fullTextFilter: { + fields: undefined, + term: '', + }, + }); + }); + + it('parses cursor request', async () => { + const items: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items, + totalItems: 100, + pageInfo: { nextCursor: mockCursor() }, + }); + + const cursor = mockCursor({ totalItems: 100, isPrevious: false }); + + const response = await request(app).get( + `/entities/by-query?cursor=${encodeCursor(cursor)}`, + ); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ + cursor, + }); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + items, + totalItems: 100, + pageInfo: { nextCursor: expect.any(String) }, + }); + }); + + it('should throw in case of malformed cursor', async () => { + const items: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items, + totalItems: 100, + pageInfo: { nextCursor: mockCursor() }, + }); + + let response = await request(app).get( + `/entities/by-query?cursor=${Buffer.from( + JSON.stringify({ bad: 'cursor' }), + 'utf8', + ).toString('base64')}`, + ); + expect(response.status).toEqual(400); + expect(response.body.error.message).toMatch(/Malformed cursor/); + + response = await request(app).get(`/entities/by-query?cursor=badcursor`); + expect(response.status).toEqual(400); + expect(response.body.error.message).toMatch(/Malformed cursor/); + }); + }); + describe('GET /entities/by-uid/:uid', () => { it('can fetch entity by uid', async () => { const entity: Entity = { @@ -560,6 +674,7 @@ describe('createRouter readonly enabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + queryEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -750,6 +865,7 @@ describe('NextRouter permissioning', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + queryEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -820,3 +936,12 @@ describe('NextRouter permissioning', () => { }); }); }); + +function mockCursor(partialCursor?: Partial): Cursor { + return { + orderFields: [], + orderFieldValues: [], + isPrevious: false, + ...partialCursor, + }; +} diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 62a0f970c9..849096c601 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -39,12 +39,14 @@ import { parseEntityFilterParams, parseEntityPaginationParams, parseEntityTransformParams, + parseQueryEntitiesParams, } from './request'; import { parseEntityFacetParams } from './request/parseEntityFacetParams'; import { parseEntityOrderParams } from './request/parseEntityOrderParams'; import { LocationService, RefreshOptions, RefreshService } from './types'; import { disallowReadonlyMode, + encodeCursor, locationInput, validateRequestBody, } from './util'; @@ -130,6 +132,26 @@ export async function createRouter( // TODO(freben): encode the pageInfo in the response res.json(entities); }) + .get('/entities/by-query', async (req, res) => { + const { items, pageInfo, totalItems } = + await entitiesCatalog.queryEntities({ + ...parseQueryEntitiesParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); + + res.json({ + items, + totalItems, + pageInfo: { + ...(pageInfo.nextCursor && { + nextCursor: encodeCursor(pageInfo.nextCursor), + }), + ...(pageInfo.prevCursor && { + prevCursor: encodeCursor(pageInfo.prevCursor), + }), + }, + }); + }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; const { entities } = await entitiesCatalog.entities({ diff --git a/plugins/catalog-backend/src/service/request/index.ts b/plugins/catalog-backend/src/service/request/index.ts index 0b09355e72..d9297fc122 100644 --- a/plugins/catalog-backend/src/service/request/index.ts +++ b/plugins/catalog-backend/src/service/request/index.ts @@ -19,3 +19,4 @@ export { basicEntityFilter } from './basicEntityFilter'; export { parseEntityFilterParams } from './parseEntityFilterParams'; export { parseEntityPaginationParams } from './parseEntityPaginationParams'; export { parseEntityTransformParams } from './parseEntityTransformParams'; +export { parseQueryEntitiesParams } from './parseQueryEntitiesParams'; diff --git a/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts new file mode 100644 index 0000000000..b732163394 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; + +describe('parseEntityOrderFieldParams', () => { + it('supports no order fields', () => { + const result = parseEntityOrderFieldParams({}); + expect(result).toEqual(undefined); + }); + + it('supports single orderField', () => { + const result = parseEntityOrderFieldParams({ + orderField: ['metadata.name,desc'], + })!; + expect(result).toEqual([{ field: 'metadata.name', order: 'desc' }]); + }); + + it('supports single orderField without order', () => { + const result = parseEntityOrderFieldParams({ + orderField: ['metadata.name'], + })!; + expect(result).toEqual([{ field: 'metadata.name' }]); + }); + + it('supports multiple order fields', () => { + const result = parseEntityOrderFieldParams({ + orderField: ['metadata.name,desc', 'metadata.uid,asc'], + }); + + expect(result).toEqual([ + { field: 'metadata.name', order: 'desc' }, + { field: 'metadata.uid', order: 'asc' }, + ]); + }); + + it('throws if orderField order is not valid', () => { + expect(() => + parseEntityOrderFieldParams({ + orderField: ['metadata.name,desc', 'metadata.uid,invalid'], + }), + ).toThrow(/Invalid order field order/); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts new file mode 100644 index 0000000000..5ab8cbfcf9 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { EntityOrder } from '../../catalog/types'; +import { parseStringsParam } from './common'; + +export function parseEntityOrderFieldParams( + params: Record, +): EntityOrder[] | undefined { + const orderFieldStrings = parseStringsParam(params.orderField, 'orderField'); + if (!orderFieldStrings) { + return undefined; + } + + return orderFieldStrings.map(orderFieldString => { + const [field, order] = orderFieldString.split(','); + + if (order !== undefined && !isOrder(order)) { + throw new InputError('Invalid order field order, must be asc or desc'); + } + return { field, order }; + }); +} + +export function isOrder(order: string): order is 'asc' | 'desc' { + return ['asc', 'desc'].includes(order); +} diff --git a/plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts b/plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts new file mode 100644 index 0000000000..02ba5864fc --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts @@ -0,0 +1,28 @@ +import { parseStringParam } from './common'; + +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function parseFullTextFilterFields(params: Record) { + const fullTextFilterFields = parseStringParam( + params.fullTextFilterFields, + 'fullTextFilterFields', + ); + if (!fullTextFilterFields) { + return undefined; + } + + return fullTextFilterFields.split(','); +} diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts new file mode 100644 index 0000000000..43ba218eac --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Cursor, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, +} from '../../catalog/types'; +import { encodeCursor } from '../util'; +import { parseQueryEntitiesParams } from './parseQueryEntitiesParams'; + +describe('parseQueryEntitiesParams', () => { + describe('initial request', () => { + it('should parse all the defined params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + filter: ['a=1', 'b=2'], + orderField: ['metadata.name,desc'], + fullTextFilterTerm: 'query', + fullTextFilterFields: 'metadata.name,metadata.namespace', + }; + const parsedObj = parseQueryEntitiesParams( + validRequest, + ) as QueryEntitiesInitialRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.orderFields).toEqual([ + { field: 'metadata.name', order: 'desc' }, + ]); + expect(parsedObj.filter).toBeDefined(); + expect(parsedObj.fullTextFilter).toEqual({ + term: 'query', + fields: ['metadata.name', 'metadata.namespace'], + }); + expect(parsedObj).not.toHaveProperty('authorizationToken'); + expect(parsedObj).not.toHaveProperty('cursor'); + }); + it('should ignore optional params', () => { + const parsedObj = parseQueryEntitiesParams( + {}, + ) as QueryEntitiesInitialRequest; + expect(parsedObj.limit).toBeUndefined(); + expect(parsedObj.fields).toBeUndefined(); + expect(parsedObj.orderFields).toBeUndefined(); + expect(parsedObj.filter).toBeUndefined(); + expect(parsedObj.fullTextFilter).toEqual({ term: '', fields: undefined }); + expect(parsedObj).not.toHaveProperty('authorizationToken'); + expect(parsedObj).not.toHaveProperty('cursor'); + }); + + it.each([ + { + limit: 'asd', + }, + { filter: 3 }, + { orderField: ['metadata.uid,diagonal'] }, + { fields: [4] }, + { fullTextFilterTerm: [] }, + { fullTextFilterFields: 3 }, + ])('should throw if some parameter is not valid %p', params => { + expect(() => parseQueryEntitiesParams(params)).toThrow(); + }); + }); + + describe('cursor request', () => { + it('should parse all the defined params', () => { + const cursor: Cursor = { + totalItems: 100, + orderFields: [], + orderFieldValues: [], + isPrevious: false, + }; + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + cursor: encodeCursor(cursor), + }; + const parsedObj = parseQueryEntitiesParams( + validRequest, + ) as QueryEntitiesCursorRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.cursor).toEqual(cursor); + }); + + it('should ignore unknown params', () => { + const cursor: Cursor = { + totalItems: 100, + orderFields: [], + orderFieldValues: [], + isPrevious: false, + }; + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + cursor: encodeCursor(cursor), + filter: ['a=1', 'b=2'], + orderField: 'orderField,desc', + query: 'query', + }; + const parsedObj = parseQueryEntitiesParams( + validRequest, + ) as QueryEntitiesCursorRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.cursor).toEqual(cursor); + expect(parsedObj).not.toHaveProperty('filter'); + expect(parsedObj).not.toHaveProperty('orderField'); + expect(parsedObj).not.toHaveProperty('query'); + }); + + it('should ignore optional params', () => { + const parsedObj = parseQueryEntitiesParams( + {}, + ) as QueryEntitiesCursorRequest; + expect(parsedObj.limit).toBeUndefined(); + expect(parsedObj.fields).toBeUndefined(); + }); + + it.each([ + { + limit: 'asd', + }, + { cursor: [] }, + { fields: [4] }, + ])('should throw if some parameter is not valid %p', params => { + expect(() => parseQueryEntitiesParams(params)).toThrow(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts new file mode 100644 index 0000000000..ae8b649d13 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, +} from '../../catalog/types'; +import { decodeCursor } from '../util'; +import { parseIntegerParam, parseStringParam } from './common'; +import { parseEntityFilterParams } from './parseEntityFilterParams'; +import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; +import { parseEntityTransformParams } from './parseEntityTransformParams'; +import { parseFullTextFilterFields } from './parseFullTextFilterFields'; + +export function parseQueryEntitiesParams( + params: Record, +): Omit { + const fields = parseEntityTransformParams(params); + const limit = parseIntegerParam(params.limit, 'limit'); + const cursor = parseStringParam(params.cursor, 'cursor'); + if (cursor) { + const decodedCursor = decodeCursor(cursor); + const response: Omit = { + cursor: decodedCursor, + fields, + limit, + }; + return response; + } + + const filter = parseEntityFilterParams(params); + const fullTextFilterTerm = parseStringParam( + params.fullTextFilterTerm, + 'fullTextFilterTerm', + ); + const fullTextFilterFields = parseFullTextFilterFields(params); + + const orderFields = parseEntityOrderFieldParams(params); + + const response: Omit = { + fields, + filter, + limit, + orderFields, + fullTextFilter: { + term: fullTextFilterTerm || '', + fields: fullTextFilterFields, + }, + }; + + return response; +} diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 9a914aadcb..699bca2fc9 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -18,6 +18,13 @@ import { InputError, NotAllowedError } from '@backstage/errors'; import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; +import { + Cursor, + EntityFilter, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, +} from '../catalog/types'; export async function requireRequestBody(req: Request): Promise { const contentType = req.header('content-type'); @@ -65,3 +72,63 @@ export function disallowReadonlyMode(readonly: boolean) { throw new NotAllowedError('This operation not allowed in readonly mode'); } } + +export function isQueryEntitiesInitialRequest( + input: QueryEntitiesRequest | undefined, +): input is QueryEntitiesInitialRequest { + if (!input) { + return false; + } + return !isQueryEntitiesCursorRequest(input); +} + +export function isQueryEntitiesCursorRequest( + input: QueryEntitiesRequest | undefined, +): input is QueryEntitiesCursorRequest { + if (!input) { + return false; + } + return !!(input as QueryEntitiesCursorRequest).cursor; +} + +const entityFilterParser: z.ZodSchema = z.lazy(() => + z + .object({ + key: z.string(), + values: z.array(z.string()).optional(), + }) + .or(z.object({ not: entityFilterParser })) + .or(z.object({ anyOf: z.array(entityFilterParser) })) + .or(z.object({ allOf: z.array(entityFilterParser) })), +); + +export const cursorParser: z.ZodSchema = z.object({ + orderFields: z.array( + z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }), + ), + orderFieldValues: z.array(z.string().or(z.null())), + filter: entityFilterParser.optional(), + isPrevious: z.boolean(), + query: z.string().optional(), + firstSortFieldValues: z.array(z.string().or(z.null())).optional(), + totalItems: z.number().optional(), +}); + +export function encodeCursor(cursor: Cursor) { + const json = JSON.stringify(cursor); + return Buffer.from(json, 'utf8').toString('base64'); +} + +export function decodeCursor(encodedCursor: string) { + try { + const data = Buffer.from(encodedCursor, 'base64').toString('utf8'); + const result = cursorParser.safeParse(JSON.parse(data)); + + if (!result.success) { + throw new InputError(`Malformed cursor: ${result.error}`); + } + return result.data; + } catch (e) { + throw new InputError(`Malformed cursor: ${e}`); + } +}