From 8b5585a88477e9e22660ca9d297229cab1d5058a Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 16:48:36 +0200 Subject: [PATCH 01/50] Add paginated entities endpoint Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 115 +++++ .../src/service/AuthorizedEntitiesCatalog.ts | 38 ++ .../service/DefaultEntitiesCatalog.test.ts | 458 ++++++++++++++++++ .../src/service/DefaultEntitiesCatalog.ts | 183 ++++++- .../src/service/createRouter.test.ts | 75 +++ .../src/service/createRouter.ts | 9 + .../src/service/request/index.ts | 1 + .../parsePaginatedEntitiesParams.test.ts | 132 +++++ .../request/parsePaginatedEntitiesParams.ts | 74 +++ plugins/catalog-backend/src/service/util.ts | 25 + 10 files changed, 1109 insertions(+), 1 deletion(-) create mode 100644 plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts create mode 100644 plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index faf7451acd..9f5ad642f2 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -174,6 +174,16 @@ export interface EntitiesCatalog { */ entitiesBatch(request: EntitiesBatchRequest): Promise; + /** + * Fetch entities and scroll back and forth between entities. + * + * @alpha + * @param request + */ + paginatedEntities( + request?: PaginatedEntitiesRequest, + ): Promise; + /** * Removes a single entity. * @@ -202,3 +212,108 @@ export interface EntitiesCatalog { */ facets(request: EntityFacetsRequest): Promise; } + +/** + * The request shape for {@link EntitiesCatalog.paginatedEntities}. + * + * @alpha + */ +export type PaginatedEntitiesRequest = + | PaginatedEntitiesInitialRequest + | PaginatedEntitiesCursorRequest; + +/** + * The initial request for {@link EntitiesCatalog.paginatedEntities}. + * The request take immutable properties that are going to be bound + * for the current and the next pagination requests. + * + * @alpha + */ +export interface PaginatedEntitiesInitialRequest { + authorizationToken?: string; + fields?: (entity: Entity) => Entity; + limit?: number; + filter?: EntityFilter; + sortField?: string; + query?: string; + sortFieldOrder?: 'asc' | 'desc' | undefined; +} + +/** + * Request for {@link EntitiesCatalog.paginatedEntities} used to + * move forward or backward on the data. + * + * @alpha + */ +export interface PaginatedEntitiesCursorRequest { + authorizationToken?: string; + fields?: (entity: Entity) => Entity; + limit?: number; + cursor: string; +} + +/** + * The response shape for {@link EntitiesCatalog.paginatedEntities}. + * + * @alpha + */ +export interface PaginatedEntitiesResponse { + /** + * The entities for the current pagination request + */ + entities: Entity[]; + /** + * The cursor of the next pagination request. + */ + nextCursor?: string; + /** + * The cursor of the previous pagination request. + */ + prevCursor?: string; + /** + * the total number of entities matching the current filters. + */ + totalItems: number; +} + +/** + * The Cursor used internally by the catalog. + * + * @alpha + */ +export type Cursor = { + /** + * the id of the field used for sorting the data. + * For example, metadata.name + */ + sortField: string; + /** + * The value of the last item returned + * by the request. This is used for performing + * cursor based pagination. + */ + sortFieldId: string; + /** + * In which order the data should be paginated. + */ + sortFieldOrder: 'asc' | 'desc'; + filter?: EntityFilter; + /** + * true if the cursor is a previous cursor. + */ + isPrevious: boolean; + /** + * Filter the data by name. + */ + query?: string; + /** + * Sort field id of the first item. + * The catalog uses this field internally for understanding when the beginning + * of the list has been reached when performing cursor based pagination. + */ + firstFieldId: string; + /** + * The number of items that match the provided filters + */ + totalItems?: number; +}; diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index c62e9f9a84..5ae31158f8 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -35,8 +35,11 @@ import { EntityFacetsRequest, EntityFacetsResponse, EntityFilter, + PaginatedEntitiesRequest, + PaginatedEntitiesResponse, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; +import { isPaginatedEntitiesInitialRequest } from './util'; export class AuthorizedEntitiesCatalog implements EntitiesCatalog { constructor( @@ -106,6 +109,41 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { return this.entitiesCatalog.entitiesBatch(request); } + async paginatedEntities( + request?: PaginatedEntitiesRequest, + ): Promise { + const authorizeDecision = ( + await this.permissionApi.authorizeConditional( + [{ permission: catalogEntityReadPermission }], + { token: request?.authorizationToken }, + ) + )[0]; + + if (authorizeDecision.result === AuthorizeResult.DENY) { + return { + entities: [], + totalItems: 0, + }; + } + + if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) { + const permissionFilter: EntityFilter = this.transformConditions( + authorizeDecision.conditions, + ); + + return this.entitiesCatalog.paginatedEntities({ + ...request, + ...(isPaginatedEntitiesInitialRequest(request) && { + filter: request?.filter + ? { allOf: [permissionFilter, request.filter] } + : permissionFilter, + }), + }); + } + + return this.entitiesCatalog.paginatedEntities(request); + } + async removeEntityByUid( uid: string, options?: { authorizationToken?: string }, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 6735709662..a9592ddf86 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -18,6 +18,10 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; +import { + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, +} from '../catalog/types'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -674,6 +678,376 @@ describe('DefaultEntitiesCatalog', () => { ); }); + describe('paginatedEntities', () => { + it.each(databases.eachSupportedId())( + 'should return paginated entities and scroll the items accordingly, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name }, + spec: { should_include_this: 'yes' }, + }; + } + + const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; + const entities: Entity[] = names.map(entityFrom); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities + .concat(notFoundEntities) + .map(e => addEntityToSearch(knex, e)), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const filter = { + key: 'spec.should_include_this', + }; + + const limit = 2; + + // initial request + const request1: PaginatedEntitiesInitialRequest = { + filter, + limit, + sortField: 'metadata.name', + }; + const response1 = await catalog.paginatedEntities(request1); + expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response1.nextCursor).toBeDefined(); + expect(response1.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // second request (forward) + const request2: PaginatedEntitiesCursorRequest = { + cursor: response1.nextCursor!, + limit, + }; + const response2 = await catalog.paginatedEntities(request2); + expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response2.nextCursor).toBeDefined(); + expect(response2.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // third request (forward) + const request3: PaginatedEntitiesCursorRequest = { + cursor: response2.nextCursor!, + limit, + }; + const response3 = await catalog.paginatedEntities(request3); + expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response3.nextCursor).toBeDefined(); + expect(response3.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fourth request (backwards) + const request4: PaginatedEntitiesCursorRequest = { + cursor: response3.prevCursor!, + limit, + }; + const response4 = await catalog.paginatedEntities(request4); + expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response4.nextCursor).toBeDefined(); + expect(response4.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fifth request (backwards) + const request5: PaginatedEntitiesCursorRequest = { + cursor: response4.prevCursor!, + limit, + }; + const response5 = await catalog.paginatedEntities(request5); + expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response5.nextCursor).toBeDefined(); + expect(response5.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // sixth request (forward) + const request6: PaginatedEntitiesCursorRequest = { + cursor: response5.nextCursor!, + limit, + }; + const response6 = await catalog.paginatedEntities(request6); + expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response6.nextCursor).toBeDefined(); + expect(response6.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh request (forward) + const request7: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit, + }; + const response7 = await catalog.paginatedEntities(request7); + expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response7.nextCursor).toBeDefined(); + expect(response7.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh.2 request (forward with a different limit) + const request7bis: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit: limit + 1, + }; + const response7bis = await catalog.paginatedEntities(request7bis); + expect(response7bis.entities).toEqual([ + entityFrom('E'), + entityFrom('F'), + entityFrom('G'), + ]); + expect(response7bis.nextCursor).toBeUndefined(); + expect(response7bis.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // last request (forward) + const request8: PaginatedEntitiesCursorRequest = { + cursor: response7.nextCursor!, + limit, + }; + const response8 = await catalog.paginatedEntities(request8); + expect(response8.entities).toEqual([entityFrom('G')]); + expect(response8.nextCursor).toBeUndefined(); + expect(response8.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + }, + ); + + it.each(databases.eachSupportedId())( + 'should return paginated entities ordered in descending order and scroll the items accordingly, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name }, + spec: { should_include_this: 'yes' }, + }; + } + + const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; + const entities: Entity[] = names.map(entityFrom); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities + .concat(notFoundEntities) + .map(e => addEntityToSearch(knex, e)), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const filter = { + key: 'spec.should_include_this', + }; + + const limit = 2; + + // initial request + const request1: PaginatedEntitiesInitialRequest = { + filter, + limit, + sortField: 'metadata.name', + sortFieldOrder: 'desc', + }; + const response1 = await catalog.paginatedEntities(request1); + expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response1.nextCursor).toBeDefined(); + expect(response1.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // second request (forward) + const request2: PaginatedEntitiesCursorRequest = { + cursor: response1.nextCursor!, + limit, + }; + const response2 = await catalog.paginatedEntities(request2); + expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response2.nextCursor).toBeDefined(); + expect(response2.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // third request (forward) + const request3: PaginatedEntitiesCursorRequest = { + cursor: response2.nextCursor!, + limit, + }; + const response3 = await catalog.paginatedEntities(request3); + expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response3.nextCursor).toBeDefined(); + expect(response3.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fourth request (backwards) + const request4: PaginatedEntitiesCursorRequest = { + cursor: response3.prevCursor!, + limit, + }; + const response4 = await catalog.paginatedEntities(request4); + + expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response4.nextCursor).toBeDefined(); + expect(response4.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fifth request (backwards) + const request5: PaginatedEntitiesCursorRequest = { + cursor: response4.prevCursor!, + limit, + }; + const response5 = await catalog.paginatedEntities(request5); + expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response5.nextCursor).toBeDefined(); + expect(response5.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // sixth request (forward) + const request6: PaginatedEntitiesCursorRequest = { + cursor: response5.nextCursor!, + limit, + }; + const response6 = await catalog.paginatedEntities(request6); + expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response6.nextCursor).toBeDefined(); + expect(response6.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh request (forward) + const request7: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit, + }; + const response7 = await catalog.paginatedEntities(request7); + expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response7.nextCursor).toBeDefined(); + expect(response7.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh.2 request (forward with a different limit) + const request7bis: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit: limit + 1, + }; + const response7bis = await catalog.paginatedEntities(request7bis); + expect(response7bis.entities).toEqual([ + entityFrom('C'), + entityFrom('B'), + entityFrom('A'), + ]); + expect(response7bis.nextCursor).toBeUndefined(); + expect(response7bis.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // last request (forward) + const request8: PaginatedEntitiesCursorRequest = { + cursor: response7.nextCursor!, + limit, + }; + const response8 = await catalog.paginatedEntities(request8); + expect(response8.entities).toEqual([entityFrom('A')]); + expect(response8.nextCursor).toBeUndefined(); + expect(response8.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + }, + ); + + it.each(databases.eachSupportedId())( + 'should filter the results when query is provided, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name }, + spec: { should_include_this: 'yes' }, + }; + } + + const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's']; + const entities: Entity[] = names.map(entityFrom); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities + .concat(notFoundEntities) + .map(e => addEntityToSearch(knex, e)), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const filter = { + key: 'spec.should_include_this', + }; + + const request: PaginatedEntitiesInitialRequest = { + filter, + limit: 100, + sortField: 'metadata.name', + query: 'cAt ', + }; + const response = await catalog.paginatedEntities(request); + expect(response.entities).toEqual([ + entityFrom('atcatss'), + entityFrom('cat'), + entityFrom('dogcat'), + ]); + expect(response.nextCursor).toBeUndefined(); + expect(response.prevCursor).toBeUndefined(); + expect(response.totalItems).toBe(3); + }, + ); + }); + describe('removeEntityByUid', () => { it.each(databases.eachSupportedId())( 'also clears parent hashes, %p', @@ -889,5 +1263,89 @@ describe('DefaultEntitiesCatalog', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'can match relations', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'one', + tags: ['java', 'rust'], + }, + spec: {}, + relations: [ + { + targetRef: 'targetRef', + type: 'ownedBy', + target: { kind: 'k', namespace: 'default', name: 'targetRef' }, + }, + ], + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'two', + tags: ['java', 'node'], + }, + spec: {}, + relations: [ + { + targetRef: 'anotherTargetRef', + type: 'ownedBy', + target: { + kind: 'k', + namespace: 'default', + name: 'anotherTargetRef', + }, + }, + { + targetRef: 'tt', + type: 'somethingelse', + target: { + kind: 'k', + namespace: 'default', + name: 'tt', + }, + }, + ], + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'three', + tags: ['go'], + }, + spec: {}, + relations: [ + { + targetRef: 'targetRef', + type: 'ownedBy', + target: { kind: 'k', namespace: 'default', name: 'targetRef' }, + }, + ], + }); + + const catalog = new DefaultEntitiesCatalog(knex); + + await expect( + catalog.facets({ + facets: ['relations.ownedBy'], + }), + ).resolves.toEqual({ + facets: { + 'relations.ownedBy': [ + { count: 1, value: 'anotherTargetRef' }, + { count: 2, value: 'targetRef' }, + ], + }, + }); + }, + ); }); }); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index d598881371..b860c16c86 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -25,6 +25,7 @@ import { Knex } from 'knex'; import { EntitiesBatchRequest, EntitiesBatchResponse, + Cursor, EntitiesCatalog, EntitiesRequest, EntitiesResponse, @@ -34,6 +35,8 @@ import { EntityFacetsResponse, EntityFilter, EntityPagination, + PaginatedEntitiesRequest, + PaginatedEntitiesResponse, } from '../catalog/types'; import { DbFinalEntitiesRow, @@ -43,8 +46,17 @@ import { DbRelationsRow, DbSearchRow, } from '../database/tables'; + import { Stitcher } from '../stitching/Stitcher'; +import { + isPaginatedEntitiesCursorRequest, + isPaginatedEntitiesInitialRequest, +} from './util'; + +const defaultSortField = 'metadata.name'; +const defaultSortFieldOrder = 'asc'; + function parsePagination(input?: EntityPagination): { limit?: number; offset?: number; @@ -97,7 +109,7 @@ function addCondition( // make a lot of sense. However, it had abysmal performance on sqlite // when datasets grew large, so we're using IN instead. const matchQuery = db('search') - .select('search.entity_id') + .select(entityIdField) .where({ key: filter.key.toLowerCase() }) .andWhere(function keyFilter() { if (filter.values) { @@ -310,6 +322,141 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { items }; } + async paginatedEntities( + request?: PaginatedEntitiesRequest, + ): Promise { + const db = this.database; + const limit = request?.limit ?? 20; + + const cursor: Omit & { sortFieldId?: string } = { + firstFieldId: '', + sortField: defaultSortField, + sortFieldOrder: defaultSortFieldOrder, + isPrevious: false, + ...parseCursorFromRequest(request), + }; + + const allowedSortFields = ['metadata.name', 'metadata.uid']; + if (!allowedSortFields.includes(cursor.sortField)) { + throw new InputError( + `Invalid sortField. Allowed values are ${allowedSortFields.join( + ', ', + )}.`, + ); + } + + const isFetchingBackwards = cursor.isPrevious; + + /** + * page number is used for quickly + * detecting the initial batch of items + * when navigating backwards without performing + * extra operations on the database. + */ + /* + const currentPage = isFetchingBackwards + ? cursor.previousPage - 1 + : cursor.previousPage + 1; + */ + + const dbQuery = db('search') + .join('final_entities', 'search.entity_id', 'final_entities.entity_id') + .where('key', cursor.sortField); + + if (cursor.filter) { + parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id'); + } + + const normalizedQueryByName = cursor.query?.trim(); + if (normalizedQueryByName) { + dbQuery.andWhereLike( + 'value', + `%${normalizedQueryByName.toLocaleLowerCase('en-US')}%`, + ); + } + + const countQuery = dbQuery.clone(); + + const isOrderingDescending = cursor.sortFieldOrder === 'desc'; + if (cursor.sortFieldId) { + dbQuery.andWhere( + 'value', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + cursor.sortFieldId, + ); + } + + dbQuery + .orderBy( + 'value', + isFetchingBackwards + ? invertOrder(cursor.sortFieldOrder) + : cursor.sortFieldOrder, + ) + // fetch an extra item for + // checking if there are more items + .limit(isFetchingBackwards ? limit : limit + 1); + + countQuery.count('search.entity_id', { as: 'count' }); + + const [rows, [{ count }]] = await Promise.all([ + dbQuery, + // for performance reasons we invoke the countQuery + // only on the first request. + // The result is then embedded into the cursor + // for subsequent requests. + typeof cursor.totalItems === 'undefined' + ? countQuery + : [{ count: cursor.totalItems }], + ]); + + const totalItems = Number(count); + + if (isFetchingBackwards) { + rows.reverse(); + } + const hasMoreResults = + limit > 0 && (isFetchingBackwards || rows.length > limit); + + // discard the extra item only when fetching forward. + if (rows.length > limit) { + rows.length -= 1; + } + + const isInitialRequest = cursor.firstFieldId === ''; + + const firstFieldId = cursor.firstFieldId || rows[0].value; + + const nextCursor = hasMoreResults + ? encodeCursor({ + ...cursor, + sortFieldId: rows[rows.length - 1].value, + firstFieldId, + isPrevious: false, + totalItems, + }) + : undefined; + + const prevCursor = + !isInitialRequest && + rows.length > 0 && + rows[0].value !== cursor.firstFieldId + ? encodeCursor({ + ...cursor, + sortFieldId: rows[0].value, + firstFieldId: cursor.firstFieldId, + isPrevious: true, + totalItems, + }) + : undefined; + + const entities = rows + .map(e => JSON.parse(e.final_entity!)) + .map(e => (request?.fields ? request.fields(e) : e)); + + return { entities, prevCursor, nextCursor, totalItems }; + } + async removeEntityByUid(uid: string): Promise { const dbConfig = this.database.client.config; @@ -490,3 +637,37 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { facets }; } } + +function encodeCursor(cursor: Cursor) { + const json = JSON.stringify(cursor); + return Buffer.from(json, 'utf8').toString('base64'); +} + +function parseCursorFromRequest( + request?: PaginatedEntitiesRequest, +): Partial { + if (isPaginatedEntitiesInitialRequest(request)) { + const { + filter, + sortField = defaultSortField, + sortFieldOrder = defaultSortFieldOrder, + query, + } = request; + return { filter, sortField, sortFieldOrder, query }; + } + if (isPaginatedEntitiesCursorRequest(request)) { + try { + const json = Buffer.from(request.cursor, 'base64').toString('utf8'); + const cursor = JSON.parse(json); + // TODO(vinzscam): validate the shit + return cursor as unknown as Cursor; + } catch { + throw new InputError('Malformed cursor, could not be parsed'); + } + } + return {}; +} + +function invertOrder(order: Cursor['sortFieldOrder']) { + return order === 'asc' ? 'desc' : 'asc'; +} diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 6379d65043..85fede25ca 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -52,6 +52,7 @@ describe('createRouter readonly disabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + paginatedEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -135,6 +136,80 @@ describe('createRouter readonly disabled', () => { }); }); + describe('GET /v2beta1/entities', () => { + it('happy path: lists entities', async () => { + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entities, + totalItems: 100, + nextCursor: 'something', + }); + + const response = await request(app).get('/v2beta1/entities'); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + entities, + totalItems: 100, + nextCursor: 'something', + }); + }); + + it('parses initial request', async () => { + entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entities: [], + totalItems: 0, + }); + const response = await request(app).get( + '/v2beta1/entities?filter=a=1,a=2,b=3&filter=c=4', + ); + + expect(response.status).toEqual(200); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ + filter: { + anyOf: [ + { + allOf: [ + { key: 'a', values: ['1', '2'] }, + { key: 'b', values: ['3'] }, + ], + }, + { allOf: [{ key: 'c', values: ['4'] }] }, + ], + }, + }); + }); + + it('parses cursor request', async () => { + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entities, + totalItems: 100, + nextCursor: 'next', + }); + + const response = await request(app).get( + '/v2beta1/entities?cursor=something', + ); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ + cursor: 'something', + }); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + entities, + totalItems: 100, + nextCursor: 'next', + }); + }); + }); + describe('GET /entities/by-uid/:uid', () => { it('can fetch entity by uid', async () => { const entity: Entity = { diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 62a0f970c9..a7f5373a03 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -39,6 +39,7 @@ import { parseEntityFilterParams, parseEntityPaginationParams, parseEntityTransformParams, + parsePaginatedEntitiesParams, } from './request'; import { parseEntityFacetParams } from './request/parseEntityFacetParams'; import { parseEntityOrderParams } from './request/parseEntityOrderParams'; @@ -130,6 +131,14 @@ export async function createRouter( // TODO(freben): encode the pageInfo in the response res.json(entities); }) + .get('/v2beta1/entities', async (req, res) => { + const response = await entitiesCatalog.paginatedEntities({ + ...parsePaginatedEntitiesParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); + + res.json(response); + }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; const { entities } = await entitiesCatalog.entities({ diff --git a/plugins/catalog-backend/src/service/request/index.ts b/plugins/catalog-backend/src/service/request/index.ts index 0b09355e72..5a55c11ca0 100644 --- a/plugins/catalog-backend/src/service/request/index.ts +++ b/plugins/catalog-backend/src/service/request/index.ts @@ -19,3 +19,4 @@ export { basicEntityFilter } from './basicEntityFilter'; export { parseEntityFilterParams } from './parseEntityFilterParams'; export { parseEntityPaginationParams } from './parseEntityPaginationParams'; export { parseEntityTransformParams } from './parseEntityTransformParams'; +export { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams'; diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts new file mode 100644 index 0000000000..838b931888 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts @@ -0,0 +1,132 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, +} from '../../catalog/types'; +import { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams'; + +describe('parsePaginatedEntitiesParams', () => { + describe('initial request', () => { + it('should parse all the defined params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + filter: ['a=1', 'b=2'], + sortField: 'sortField', + sortFieldOrder: 'desc', + query: 'query', + }; + const parsedObj = parsePaginatedEntitiesParams( + validRequest, + ) as PaginatedEntitiesInitialRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.sortField).toBe('sortField'); + expect(parsedObj.sortFieldOrder).toBe('desc'); + expect(parsedObj.filter).toBeDefined(); + expect(parsedObj.query).toBe('query'); + expect(parsedObj).not.toHaveProperty('authorizationToken'); + expect(parsedObj).not.toHaveProperty('cursor'); + }); + it('should ignore optional params', () => { + const parsedObj = parsePaginatedEntitiesParams( + {}, + ) as PaginatedEntitiesInitialRequest; + expect(parsedObj.limit).toBeUndefined(); + expect(parsedObj.fields).toBeUndefined(); + expect(parsedObj.sortField).toBeUndefined(); + expect(parsedObj.sortFieldOrder).toBeUndefined(); + expect(parsedObj.filter).toBeUndefined(); + expect(parsedObj.query).toBeUndefined(); + expect(parsedObj).not.toHaveProperty('authorizationToken'); + expect(parsedObj).not.toHaveProperty('cursor'); + }); + + it.each([ + { + limit: 'asd', + }, + { filter: 3 }, + { sortField: [] }, + { sortFieldOrder: 'something' }, + { fields: [4] }, + { query: [] }, + ])('should throw if some parameter is not valid %p', params => { + expect(() => parsePaginatedEntitiesParams(params)).toThrow(); + }); + }); + + describe('cursor request', () => { + it('should parse all the defined params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + cursor: 'cursor', + }; + const parsedObj = parsePaginatedEntitiesParams( + validRequest, + ) as PaginatedEntitiesCursorRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.cursor).toBe('cursor'); + }); + + it('should ignore unknown params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + cursor: 'cursor', + filter: ['a=1', 'b=2'], + sortField: 'sortField', + sortFieldOrder: 'desc', + query: 'query', + }; + const parsedObj = parsePaginatedEntitiesParams( + validRequest, + ) as PaginatedEntitiesCursorRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.cursor).toBe('cursor'); + expect(parsedObj).not.toHaveProperty('filter'); + expect(parsedObj).not.toHaveProperty('sortField'); + expect(parsedObj).not.toHaveProperty('sortFieldOrder'); + expect(parsedObj).not.toHaveProperty('query'); + }); + + it('should ignore optional params', () => { + const parsedObj = parsePaginatedEntitiesParams( + {}, + ) as PaginatedEntitiesCursorRequest; + expect(parsedObj.limit).toBeUndefined(); + expect(parsedObj.fields).toBeUndefined(); + }); + + it.each([ + { + limit: 'asd', + }, + { cursor: [] }, + { fields: [4] }, + ])('should throw if some parameter is not valid %p', params => { + expect(() => parsePaginatedEntitiesParams(params)).toThrow(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts new file mode 100644 index 0000000000..84d519ec0b --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, + PaginatedEntitiesRequest, +} from '../../catalog/types'; +import { parseIntegerParam, parseStringParam } from './common'; +import { parseEntityFilterParams } from './parseEntityFilterParams'; +import { parseEntityTransformParams } from './parseEntityTransformParams'; + +export function parsePaginatedEntitiesParams( + params: Record, +): Omit { + const fields = parseEntityTransformParams(params); + const limit = parseIntegerParam(params.limit, 'limit'); + const cursor = parseStringParam(params.cursor, 'cursor'); + if (cursor) { + const response: Omit = + { + cursor, + fields, + limit, + }; + return response; + } + + const filter = parseEntityFilterParams(params); + const query = parseStringParam(params.query, 'query'); + const sortField = parseStringParam(params.sortField, 'sortField'); + const sortFieldOrder = parseSortFieldOrder( + params.sortFieldOrder, + 'sortFieldOrder', + ); + + const response: Omit = + { + fields, + filter, + limit, + sortField, + sortFieldOrder, + query, + }; + + return response; +} + +function parseSortFieldOrder(sortFieldOrder: unknown, ctx: string) { + const isSortFieldOrder = + sortFieldOrder === undefined || + sortFieldOrder === 'asc' || + sortFieldOrder === 'desc'; + + if (isSortFieldOrder) { + return sortFieldOrder; + } + throw new InputError(`Invalid ${ctx}, not asc or desc`); +} diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 9a914aadcb..bb86654565 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -18,6 +18,11 @@ import { InputError, NotAllowedError } from '@backstage/errors'; import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; +import { + EntitiesRequest, + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, +} from '../catalog/types'; export async function requireRequestBody(req: Request): Promise { const contentType = req.header('content-type'); @@ -65,3 +70,23 @@ export function disallowReadonlyMode(readonly: boolean) { throw new NotAllowedError('This operation not allowed in readonly mode'); } } + +export function isPaginatedEntitiesInitialRequest( + input: EntitiesRequest | undefined, +): input is PaginatedEntitiesInitialRequest { + if (!input) { + return false; + } + // TODO(vinzscam) expand this + return !isPaginatedEntitiesCursorRequest(input); +} + +export function isPaginatedEntitiesCursorRequest( + input: EntitiesRequest | undefined, +): input is PaginatedEntitiesCursorRequest { + if (!input) { + return false; + } + // TODO(vinzscam) expand this + return input?.hasOwnProperty('cursor') ?? false; +} From 6a1e7fbe52b7e66b31dcee04b83dcb0ad550b198 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 18:46:25 +0200 Subject: [PATCH 02/50] CatalogClient: add getPaginatedEntities method Signed-off-by: Vincenzo Scamporlino --- .../catalog-client/src/CatalogClient.test.ts | 194 ++++++++++++++++++ packages/catalog-client/src/CatalogClient.ts | 163 ++++++++++----- packages/catalog-client/src/index.ts | 1 + packages/catalog-client/src/types/api.ts | 110 +++++++++- packages/catalog-client/src/types/index.ts | 3 + packages/catalog-client/src/utils.ts | 36 ++++ 6 files changed, 452 insertions(+), 55 deletions(-) create mode 100644 packages/catalog-client/src/utils.ts diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 2e9d2a0679..8a449c730e 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -249,6 +249,200 @@ describe('CatalogClient', () => { }); }); + describe('getPaginatedEntities', () => { + const defaultServiceResponse = { + entities: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test1', + namespace: 'test1', + }, + }, + ], + totalItems: 10, + nextCursor: 'next', + prevCursor: 'prev', + }; + + const defaultClientResponse = { + entities: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test1', + namespace: 'test1', + }, + }, + ], + totalItems: 10, + next: jest.fn(), + prev: jest.fn(), + }; + + beforeEach(() => { + server.use( + rest.get(`${mockBaseUrl}/v2beta1/entities`, (_, res, ctx) => { + return res(ctx.json(defaultServiceResponse)); + }), + ); + }); + + it('should fetch entities from correct endpoint', async () => { + const response = await client.getPaginatedEntities?.({}, { token }); + expect(response?.entities).toEqual(defaultClientResponse.entities); + expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); + expect(response?.next).toBeDefined(); + expect(response?.prev).toBeDefined(); + }); + + it('builds multiple entity search filters properly', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ entities: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + const response = await client.getPaginatedEntities?.( + { + filter: [ + { + a: '1', + b: ['2', '3'], + รถ: '=', + }, + { + a: '2', + }, + { + c: CATALOG_FILTER_EXISTS, + }, + ], + }, + { token }, + ); + + expect(response).toEqual({ entities: [], totalItems: 0 }); + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c', + ); + }); + + it('should send query params correctly on initial request', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ entities: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + await client.getPaginatedEntities?.({ + fields: ['a', 'b'], + limit: 100, + query: 'query', + sortField: 'metadata.name', + sortFieldOrder: 'asc', + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?limit=100&sortField=metadata.name&sortFieldOrder=asc&fields=a,b&query=query', + ); + }); + + it('should ignore initial query params if cursor is passed', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ entities: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + await client.getPaginatedEntities?.({ + fields: ['a', 'b'], + limit: 100, + query: 'query', + sortField: 'metadata.name', + sortFieldOrder: 'asc', + cursor: 'cursor', + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?cursor=cursor&limit=100&fields=a,b', + ); + }); + + it('should return paginated functions if next and prev cursors are present', async () => { + const mockedEndpoint = jest.fn().mockImplementation((_req, res, ctx) => + res( + ctx.json({ + entities: [ + { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'e1', + }, + }, + { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'e2', + }, + }, + ], + nextCursor: 'nextcursor', + prevCursor: 'prevcursor', + totalItems: 100, + }), + ), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + const response = await client.getPaginatedEntities?.({ + limit: 2, + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); + + expect(response?.next).toBeDefined(); + expect(response?.prev).toBeDefined(); + + await response?.next?.(); + expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( + '?cursor=nextcursor', + ); + + await response?.prev?.(); + expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( + '?cursor=prevcursor', + ); + }); + }); + describe('getEntityByRef', () => { const existingEntity: Entity = { apiVersion: 'v1', diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index af4262b842..03fdd600e9 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -39,9 +39,14 @@ import { ValidateEntityResponse, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, + GetPaginatedEntitiesRequest, + GetPaginatedEntitiesResponse, + EntitiesFilter, + GetPaginatedEntitiesCursorRequest, } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; +import { isPaginatedEntitiesInitialRequest } from './utils'; /** * A frontend and backend compatible client for communicating with the Backstage @@ -107,30 +112,7 @@ export class CatalogClient implements CatalogApi { limit, after, } = request ?? {}; - const params: string[] = []; - - // filter param can occur multiple times, for example - // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' - // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters - // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of [filter].flat()) { - const filterParts: string[] = []; - for (const [key, value] of Object.entries(filterItem)) { - for (const v of [value].flat()) { - if (v === CATALOG_FILTER_EXISTS) { - filterParts.push(encodeURIComponent(key)); - } else if (typeof v === 'string') { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); - } - } - } - - if (filterParts.length) { - params.push(`filter=${filterParts.join(',')}`); - } - } + const params = this.getParams(filter); if (fields.length) { params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); @@ -223,6 +205,70 @@ export class CatalogClient implements CatalogApi { return { items }; } + /** + * {@inheritdoc CatalogApi.getPaginatedEntities} + */ + async getPaginatedEntities?( + request?: GetPaginatedEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise { + const params: string[] = []; + + if (isPaginatedEntitiesInitialRequest(request)) { + const { + fields = [], + filter, + limit, + sortField, + sortFieldOrder, + query, + } = request ?? {}; + params.push(...this.getParams(filter)); + + if (limit !== undefined) { + params.push(`limit=${limit}`); + } + if (sortField !== undefined) { + params.push(`sortField=${sortField}`); + } + if (sortFieldOrder !== undefined) { + params.push(`sortFieldOrder=${sortFieldOrder}`); + } + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + + const normalizedQuery = query?.trim(); + if (normalizedQuery) { + params.push(`query=${normalizedQuery}`); + } + } else { + const { fields = [], limit, cursor } = request; + + params.push(`cursor=${cursor}`); + if (limit !== undefined) { + params.push(`limit=${limit}`); + } + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + } + + const query = params.length ? `?${params.join('&')}` : ''; + const { entities, totalItems, nextCursor, prevCursor } = + await this.requestRequired<{ + entities: Entity[]; + totalItems: number; + nextCursor?: string; + prevCursor?: string; + }>('GET', `/v2beta1/entities${query}`, options); + + const next = this.getEntitiesFromCursor(nextCursor, options); + const prev = this.getEntitiesFromCursor(prevCursor, options); + + return { entities, totalItems, next, prev }; + } + /** * {@inheritdoc CatalogApi.getEntityByRef} */ @@ -290,30 +336,7 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise { const { filter = [], facets } = request; - const params: string[] = []; - - // filter param can occur multiple times, for example - // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' - // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters - // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of [filter].flat()) { - const filterParts: string[] = []; - for (const [key, value] of Object.entries(filterItem)) { - for (const v of [value].flat()) { - if (v === CATALOG_FILTER_EXISTS) { - filterParts.push(encodeURIComponent(key)); - } else if (typeof v === 'string') { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); - } - } - } - - if (filterParts.length) { - params.push(`filter=${filterParts.join(',')}`); - } - } + const params = this.getParams(filter); for (const facet of facets) { params.push(`facet=${encodeURIComponent(facet)}`); @@ -466,11 +489,11 @@ export class CatalogClient implements CatalogApi { } } - private async requestRequired( + private async requestRequired( method: string, path: string, options?: CatalogRequestOptions, - ): Promise { + ): Promise { const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; const headers: Record = options?.token ? { Authorization: `Bearer ${options.token}` } @@ -481,7 +504,7 @@ export class CatalogClient implements CatalogApi { throw await ResponseError.fromResponse(response); } - return await response.json(); + return response.json(); } private async requestOptional( @@ -504,4 +527,42 @@ export class CatalogClient implements CatalogApi { return await response.json(); } + + private getParams(filter: EntitiesFilter = []) { + const params: string[] = []; + // filter param can occur multiple times, for example + // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' + // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters + // the "inner array" defined within a `filter` param corresponds to "allOf" filters + for (const filterItem of [filter].flat()) { + const filterParts: string[] = []; + for (const [key, value] of Object.entries(filterItem)) { + for (const v of [value].flat()) { + if (v === CATALOG_FILTER_EXISTS) { + filterParts.push(encodeURIComponent(key)); + } else if (typeof v === 'string') { + filterParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, + ); + } + } + } + + if (filterParts.length) { + params.push(`filter=${filterParts.join(',')}`); + } + } + return params; + } + + private getEntitiesFromCursor( + cursor: string | undefined, + options?: CatalogRequestOptions, + ) { + if (!cursor) { + return undefined; + } + return (request: Omit = {}) => + this.getPaginatedEntities!({ ...request, cursor }, options); + } } diff --git a/packages/catalog-client/src/index.ts b/packages/catalog-client/src/index.ts index 2e81942c2f..f0b10a3eef 100644 --- a/packages/catalog-client/src/index.ts +++ b/packages/catalog-client/src/index.ts @@ -22,3 +22,4 @@ export { CatalogClient } from './CatalogClient'; export * from './types'; +export * from './utils'; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index d662a9d106..1390a0576e 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -287,10 +287,7 @@ export interface GetEntityFacetsRequest { * (exported from this package), which means that you assert on the existence * of that key, no matter what its value is. */ - filter?: - | Record[] - | Record - | undefined; + filter?: EntitiesFilter; /** * Dot separated paths for the facets to extract from each entity. * @@ -359,6 +356,11 @@ export type AddLocationRequest = { dryRun?: boolean; }; +export type EntitiesFilter = + | Record[] + | Record + | undefined; + /** * The response type for {@link CatalogClient.addLocation}. * @@ -380,6 +382,69 @@ export type ValidateEntityResponse = | { valid: true } | { valid: false; errors: SerializedError[] }; +/** + * The request type for {@link CatalogClient.getPaginatedEntities}. + * + * @alpha + */ +export type GetPaginatedEntitiesRequest = + | GetPaginatedEntitiesInitialRequest + | GetPaginatedEntitiesCursorRequest; + +/** + * A request type for {@link CatalogClient.getPaginatedEntities}. + * The method takes this type in an initial pagination request, + * when requesting the first batch of entities. + * + * The properties filter, sortField, query and sortFieldOrder, are going + * to be immutable for the entire lifecycle of the following requests. + * + * @alpha + */ +export type GetPaginatedEntitiesInitialRequest = + | { + fields?: string[]; + limit?: number; + filter?: EntitiesFilter; + sortField?: string; + query?: string; + sortFieldOrder?: 'asc' | 'desc'; + } + | undefined; + +/** + * A request type for {@link CatalogClient.getPaginatedEntities}. + * The method takes this type in a pagination request, following + * the initial request. + * + * @alpha + */ +export type GetPaginatedEntitiesCursorRequest = { + fields?: string[]; + limit?: number; + cursor: string; +}; + +/** + * The response type for {@link CatalogClient.getPaginatedEntities}. + * + * @alpha + */ +export type GetPaginatedEntitiesResponse = { + /* The list of entities for the current request */ + entities: Entity[]; + /* The number of entities among all the requests */ + totalItems: number; + /* A method returning a promise containing the next batch of entities. */ + next?( + request?: Omit, + ): Promise; + /* A method returning a promise containing the previous batch of entities. */ + prev?( + request?: Omit, + ): Promise; +}; + /** * A client for interacting with the Backstage software catalog through its API. * @@ -414,6 +479,43 @@ export interface CatalogApi { options?: CatalogRequestOptions, ): Promise; + /** + * Gets paginated entities from the catalog. + * The method takes + * @alpha + * @remarks + * + * Example: + * + * const response = await catalogClient.getPaginatedEntities({ + * filter: [{ kind: 'group' }], + * limit: 20, + * query: 'A', + * sortField: 'metadata.name', + * sortFieldOrder: 'asc' + * }); + * + * this will match all entities of type group having a name starting + * with 'A', ordered by name ascending. + * + * The response will contain a maximum of 20 entities. In case + * more than 20 entities exist, the response will contain a next function, invocable + * using: + * + * const secondBatchResponse = await response.next({ limit: 20 }); + * + * secondBatchResponse will contain the next batch of maximum 20 entities, together + * with a prev function useful for navigating backwards and a next function + * in case more entities matching the filters of the first request are present in the catalog. + * + * @param request - Request parameters + * @param options - Additional options + */ + getPaginatedEntities?( + request?: GetPaginatedEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; + /** * Gets entity ancestor information, i.e. the hierarchy of parent entities * whose processing resulted in a given entity appearing in the catalog. diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 81b985eb42..7da07979fc 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -33,5 +33,8 @@ export type { GetEntityFacetsResponse, Location, ValidateEntityResponse, + GetPaginatedEntitiesCursorRequest, + GetPaginatedEntitiesInitialRequest, + GetPaginatedEntitiesResponse, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts new file mode 100644 index 0000000000..5a26053fb9 --- /dev/null +++ b/packages/catalog-client/src/utils.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + GetPaginatedEntitiesCursorRequest, + GetPaginatedEntitiesInitialRequest, + GetPaginatedEntitiesRequest, +} from './types/api'; + +export function isPaginatedEntitiesInitialRequest( + request: GetPaginatedEntitiesInitialRequest, +): request is GetPaginatedEntitiesInitialRequest { + if (!request) { + return true; + } + return !(request as GetPaginatedEntitiesCursorRequest).cursor; +} + +export function isPaginatedEntitiesCursorRequest( + request: GetPaginatedEntitiesRequest, +): request is GetPaginatedEntitiesCursorRequest { + return !isPaginatedEntitiesInitialRequest(request); +} From 8b39d640b1e836e5c8840de1f33f5da4284d192a Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 20:30:42 +0200 Subject: [PATCH 03/50] Fix typings Signed-off-by: Vincenzo Scamporlino --- .../src/service/AuthorizedEntitiesCatalog.test.ts | 1 + .../src/service/DefaultEntitiesCatalog.test.ts | 12 ------------ .../catalog-backend/src/service/createRouter.test.ts | 2 ++ 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 829e41c21e..8ee10fdbcb 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -30,6 +30,7 @@ describe('AuthorizedEntitiesCatalog', () => { facets: jest.fn(), refresh: jest.fn(), listAncestors: jest.fn(), + paginatedEntities: jest.fn(), }; const fakePermissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index a9592ddf86..9b315f42b7 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1281,7 +1281,6 @@ describe('DefaultEntitiesCatalog', () => { { targetRef: 'targetRef', type: 'ownedBy', - target: { kind: 'k', namespace: 'default', name: 'targetRef' }, }, ], }); @@ -1297,20 +1296,10 @@ describe('DefaultEntitiesCatalog', () => { { targetRef: 'anotherTargetRef', type: 'ownedBy', - target: { - kind: 'k', - namespace: 'default', - name: 'anotherTargetRef', - }, }, { targetRef: 'tt', type: 'somethingelse', - target: { - kind: 'k', - namespace: 'default', - name: 'tt', - }, }, ], }); @@ -1326,7 +1315,6 @@ describe('DefaultEntitiesCatalog', () => { { targetRef: 'targetRef', type: 'ownedBy', - target: { kind: 'k', namespace: 'default', name: 'targetRef' }, }, ], }); diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 85fede25ca..6e2d52c63e 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -635,6 +635,7 @@ describe('createRouter readonly enabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + paginatedEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -825,6 +826,7 @@ describe('NextRouter permissioning', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + paginatedEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), From 7f4ea3d3602faa71760f8fac64134412db7794c2 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 20:37:19 +0200 Subject: [PATCH 04/50] Add changeset Signed-off-by: Vincenzo Scamporlino --- .changeset/silly-suits-run.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/silly-suits-run.md diff --git a/.changeset/silly-suits-run.md b/.changeset/silly-suits-run.md new file mode 100644 index 0000000000..ed0961a584 --- /dev/null +++ b/.changeset/silly-suits-run.md @@ -0,0 +1,8 @@ +--- +'@backstage/catalog-client': minor +'@backstage/plugin-catalog-backend': minor +--- + +Add /v2beta1/entities endpoint returning paginated entities. + +The endpoint supports cursor base pagination and server side sorting of the entities by name or uid. From 29e2c48496e6cbe903fb6a001426ed510923af46 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 20:51:50 +0200 Subject: [PATCH 05/50] Add tests for limit zero Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 29 ++++++++++++++++++- .../src/service/DefaultEntitiesCatalog.ts | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 9b315f42b7..35c1f7153b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -17,7 +17,7 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; -import { v4 as uuid } from 'uuid'; +import { v4 as uuid, v4 } from 'uuid'; import { PaginatedEntitiesCursorRequest, PaginatedEntitiesInitialRequest, @@ -1046,6 +1046,33 @@ describe('DefaultEntitiesCatalog', () => { expect(response.totalItems).toBe(3); }, ); + + it.each(databases.eachSupportedId())( + 'should include totalItems and empty entities in the response in case limit is zero, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await Promise.all( + Array(20) + .fill(0) + .map(() => + addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { name: v4() }, + }), + ), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const request: PaginatedEntitiesInitialRequest = { + limit: 0, + }; + const response = await catalog.paginatedEntities(request); + expect(response).toEqual({ totalItems: 20, entities: [] }); + }, + ); }); describe('removeEntityByUid', () => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index b860c16c86..bdd36af483 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -425,7 +425,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isInitialRequest = cursor.firstFieldId === ''; - const firstFieldId = cursor.firstFieldId || rows[0].value; + const firstFieldId = cursor.firstFieldId || rows[0]?.value; const nextCursor = hasMoreResults ? encodeCursor({ From 296d4f6179c81433c4a763365dfc2e7194774b6e Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 21:21:45 +0200 Subject: [PATCH 06/50] Improve api reports Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 57 +++++++++++++++++-- packages/catalog-client/src/CatalogClient.ts | 2 + packages/catalog-client/src/index.ts | 1 - packages/catalog-client/src/types/api.ts | 14 ++++- packages/catalog-client/src/types/index.ts | 2 + .../service/DefaultEntitiesCatalog.test.ts | 10 ++-- 6 files changed, 74 insertions(+), 12 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index a81314308b..f612407b1d 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -58,6 +58,11 @@ export interface CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; + // @alpha + getPaginatedEntities?( + request?: GetPaginatedEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; refreshEntity( entityRef: string, options?: CatalogRequestOptions, @@ -124,6 +129,11 @@ export class CatalogClient implements CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; + // @alpha + getPaginatedEntities?( + request?: GetPaginatedEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; refreshEntity( entityRef: string, options?: CatalogRequestOptions, @@ -149,6 +159,12 @@ export interface CatalogRequestOptions { token?: string; } +// @public +export type EntitiesFilter = + | Record[] + | Record + | undefined; + // @public export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = 'backstage.io/catalog-processing'; @@ -219,10 +235,7 @@ export interface GetEntityAncestorsResponse { // @public export interface GetEntityFacetsRequest { facets: string[]; - filter?: - | Record[] - | Record - | undefined; + filter?: EntitiesFilter; } // @public @@ -236,6 +249,42 @@ export interface GetEntityFacetsResponse { >; } +// @alpha +export type GetPaginatedEntitiesCursorRequest = { + fields?: string[]; + limit?: number; + cursor: string; +}; + +// @alpha +export type GetPaginatedEntitiesInitialRequest = + | { + fields?: string[]; + limit?: number; + filter?: EntitiesFilter; + sortField?: string; + query?: string; + sortFieldOrder?: 'asc' | 'desc'; + } + | undefined; + +// @alpha +export type GetPaginatedEntitiesRequest = + | GetPaginatedEntitiesInitialRequest + | GetPaginatedEntitiesCursorRequest; + +// @alpha +export type GetPaginatedEntitiesResponse = { + entities: Entity[]; + totalItems: number; + next?( + request?: Omit, + ): Promise; + prev?( + request?: Omit, + ): Promise; +}; + // @public type Location_2 = { id: string; diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 03fdd600e9..5846430a78 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -207,6 +207,8 @@ export class CatalogClient implements CatalogApi { /** * {@inheritdoc CatalogApi.getPaginatedEntities} + * + * @alpha */ async getPaginatedEntities?( request?: GetPaginatedEntitiesRequest, diff --git a/packages/catalog-client/src/index.ts b/packages/catalog-client/src/index.ts index f0b10a3eef..2e81942c2f 100644 --- a/packages/catalog-client/src/index.ts +++ b/packages/catalog-client/src/index.ts @@ -22,4 +22,3 @@ export { CatalogClient } from './CatalogClient'; export * from './types'; -export * from './utils'; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 1390a0576e..b273eb9e13 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -356,6 +356,11 @@ export type AddLocationRequest = { dryRun?: boolean; }; +/** + * Used for filtering entities that match the given patterns. + * + * @public + */ export type EntitiesFilter = | Record[] | Record @@ -481,12 +486,12 @@ export interface CatalogApi { /** * Gets paginated entities from the catalog. - * The method takes - * @alpha + * * @remarks * * Example: * + * ``` * const response = await catalogClient.getPaginatedEntities({ * filter: [{ kind: 'group' }], * limit: 20, @@ -494,6 +499,7 @@ export interface CatalogApi { * sortField: 'metadata.name', * sortFieldOrder: 'asc' * }); + * ``` * * this will match all entities of type group having a name starting * with 'A', ordered by name ascending. @@ -502,12 +508,16 @@ export interface CatalogApi { * more than 20 entities exist, the response will contain a next function, invocable * using: * + * ``` * const secondBatchResponse = await response.next({ limit: 20 }); + * ``` * * secondBatchResponse will contain the next batch of maximum 20 entities, together * with a prev function useful for navigating backwards and a next function * in case more entities matching the filters of the first request are present in the catalog. * + * @alpha + * * @param request - Request parameters * @param options - Additional options */ diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 7da07979fc..e85b86f4af 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -25,6 +25,7 @@ export type { EntityOrderQuery, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, + EntitiesFilter, GetEntitiesRequest, GetEntitiesResponse, GetEntityAncestorsRequest, @@ -35,6 +36,7 @@ export type { ValidateEntityResponse, GetPaginatedEntitiesCursorRequest, GetPaginatedEntitiesInitialRequest, + GetPaginatedEntitiesRequest, GetPaginatedEntitiesResponse, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 35c1f7153b..8f043a594b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -717,7 +717,7 @@ describe('DefaultEntitiesCatalog', () => { .map(e => addEntityToSearch(knex, e)), ); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog(knex, stitcher); const filter = { key: 'spec.should_include_this', @@ -869,7 +869,7 @@ describe('DefaultEntitiesCatalog', () => { .map(e => addEntityToSearch(knex, e)), ); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog(knex, stitcher); const filter = { key: 'spec.should_include_this', @@ -1023,7 +1023,7 @@ describe('DefaultEntitiesCatalog', () => { .map(e => addEntityToSearch(knex, e)), ); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog(knex, stitcher); const filter = { key: 'spec.should_include_this', @@ -1064,7 +1064,7 @@ describe('DefaultEntitiesCatalog', () => { ), ); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog(knex, stitcher); const request: PaginatedEntitiesInitialRequest = { limit: 0, @@ -1346,7 +1346,7 @@ describe('DefaultEntitiesCatalog', () => { ], }); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog(knex, stitcher); await expect( catalog.facets({ From 9ca120a52195ba22f0c937d25c43284cc2c619bd Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 21:46:57 +0200 Subject: [PATCH 07/50] Remove unused comment Signed-off-by: Vincenzo Scamporlino --- .../src/service/DefaultEntitiesCatalog.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index bdd36af483..c471e1379a 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -347,18 +347,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isFetchingBackwards = cursor.isPrevious; - /** - * page number is used for quickly - * detecting the initial batch of items - * when navigating backwards without performing - * extra operations on the database. - */ - /* - const currentPage = isFetchingBackwards - ? cursor.previousPage - 1 - : cursor.previousPage + 1; - */ - const dbQuery = db('search') .join('final_entities', 'search.entity_id', 'final_entities.entity_id') .where('key', cursor.sortField); @@ -393,8 +381,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ? invertOrder(cursor.sortFieldOrder) : cursor.sortFieldOrder, ) - // fetch an extra item for - // checking if there are more items + // fetch an extra item to check if there are more items. .limit(isFetchingBackwards ? limit : limit + 1); countQuery.count('search.entity_id', { as: 'count' }); From b8c262e4fbd2ee73acb9b1d082b817c6895d2bf2 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 23:01:37 +0200 Subject: [PATCH 08/50] Remove sortField guard Signed-off-by: Vincenzo Scamporlino --- .../src/service/DefaultEntitiesCatalog.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index c471e1379a..9190c76dcc 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -336,15 +336,6 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ...parseCursorFromRequest(request), }; - const allowedSortFields = ['metadata.name', 'metadata.uid']; - if (!allowedSortFields.includes(cursor.sortField)) { - throw new InputError( - `Invalid sortField. Allowed values are ${allowedSortFields.join( - ', ', - )}.`, - ); - } - const isFetchingBackwards = cursor.isPrevious; const dbQuery = db('search') From 5b35637307e072872bd51af8415f375fa21508d0 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 9 Sep 2022 11:17:21 +0200 Subject: [PATCH 09/50] catalog: add support for multiple sortfields Signed-off-by: Vincenzo Scamporlino --- .../catalog-client/src/CatalogClient.test.ts | 11 ++-- packages/catalog-client/src/CatalogClient.ts | 18 ++---- packages/catalog-client/src/types/api.ts | 8 ++- plugins/catalog-backend/src/catalog/types.ts | 26 +++++---- .../service/DefaultEntitiesCatalog.test.ts | 8 +-- .../src/service/DefaultEntitiesCatalog.ts | 47 ++++++++------- .../src/service/createRouter.test.ts | 6 +- .../parseEntitySortFieldParams.test.ts | 57 +++++++++++++++++++ .../request/parseEntitySortFieldParams.ts | 41 +++++++++++++ .../parsePaginatedEntitiesParams.test.ts | 14 ++--- .../request/parsePaginatedEntitiesParams.ts | 23 +------- 11 files changed, 173 insertions(+), 86 deletions(-) create mode 100644 plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts create mode 100644 plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 8a449c730e..4e5065e089 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -362,11 +362,13 @@ describe('CatalogClient', () => { fields: ['a', 'b'], limit: 100, query: 'query', - sortField: 'metadata.name', - sortFieldOrder: 'asc', + sortFields: [ + { field: 'metadata.name', order: 'asc' }, + { field: 'metadata.uid', order: 'desc' }, + ], }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( - '?limit=100&sortField=metadata.name&sortFieldOrder=asc&fields=a,b&query=query', + '?limit=100&sortField=metadata.name,asc&sortField=metadata.uid,desc&fields=a,b&query=query', ); }); @@ -383,8 +385,7 @@ describe('CatalogClient', () => { fields: ['a', 'b'], limit: 100, query: 'query', - sortField: 'metadata.name', - sortFieldOrder: 'asc', + sortFields: [{ field: 'metadata.name', order: 'asc' }], cursor: 'cursor', }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 5846430a78..18b614fd2e 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -217,24 +217,16 @@ export class CatalogClient implements CatalogApi { const params: string[] = []; if (isPaginatedEntitiesInitialRequest(request)) { - const { - fields = [], - filter, - limit, - sortField, - sortFieldOrder, - query, - } = request ?? {}; + const { fields = [], filter, limit, sortFields, query } = request ?? {}; params.push(...this.getParams(filter)); if (limit !== undefined) { params.push(`limit=${limit}`); } - if (sortField !== undefined) { - params.push(`sortField=${sortField}`); - } - if (sortFieldOrder !== undefined) { - params.push(`sortFieldOrder=${sortFieldOrder}`); + if (sortFields !== undefined) { + sortFields.forEach(({ field, order }) => + params.push(`sortField=${field},${order}`), + ); } if (fields.length) { params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index b273eb9e13..d006a6877c 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -366,6 +366,11 @@ export type EntitiesFilter = | Record | undefined; +/** + * Used for sorting the entities. + */ +export type EntitySortField = { field: string; order: 'asc' | 'desc' }; + /** * The response type for {@link CatalogClient.addLocation}. * @@ -411,9 +416,8 @@ export type GetPaginatedEntitiesInitialRequest = fields?: string[]; limit?: number; filter?: EntitiesFilter; - sortField?: string; + sortFields?: EntitySortField[]; query?: string; - sortFieldOrder?: 'asc' | 'desc'; } | undefined; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 9f5ad642f2..966e09b821 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -46,6 +46,12 @@ export type EntityOrder = { order: 'asc' | 'desc'; }; +// TODO(vinzscam): remove this type in favor of EntityOrder +export type EntitySortField = { + field: string; + order?: 'asc' | 'desc' | undefined; +}; + /** * Matches rows in the search table. * @public @@ -234,9 +240,8 @@ export interface PaginatedEntitiesInitialRequest { fields?: (entity: Entity) => Entity; limit?: number; filter?: EntityFilter; - sortField?: string; + sortFields?: EntitySortField[]; query?: string; - sortFieldOrder?: 'asc' | 'desc' | undefined; } /** @@ -283,20 +288,19 @@ export interface PaginatedEntitiesResponse { */ export type Cursor = { /** - * the id of the field used for sorting the data. - * For example, metadata.name + * An array of fields used for sorting the data. + * For example, [ { field: 'metadata.name', order: 'asc' } ] */ - sortField: string; + sortFields: EntitySortField[]; /** - * The value of the last item returned - * by the request. This is used for performing - * cursor based pagination. + * The value of the cursor of the last item returned. + * This is used for performing pagination. */ - sortFieldId: string; + sortFieldIds: string[]; + /** - * In which order the data should be paginated. + * A filter to apply on the full list of entities. */ - sortFieldOrder: 'asc' | 'desc'; filter?: EntityFilter; /** * true if the cursor is a previous cursor. diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 8f043a594b..af4bacadaa 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -729,7 +729,7 @@ describe('DefaultEntitiesCatalog', () => { const request1: PaginatedEntitiesInitialRequest = { filter, limit, - sortField: 'metadata.name', + sortFields: [{ field: 'metadata.name' }], }; const response1 = await catalog.paginatedEntities(request1); expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); @@ -881,8 +881,7 @@ describe('DefaultEntitiesCatalog', () => { const request1: PaginatedEntitiesInitialRequest = { filter, limit, - sortField: 'metadata.name', - sortFieldOrder: 'desc', + sortFields: [{ field: 'metadata.name', order: 'desc' }], }; const response1 = await catalog.paginatedEntities(request1); expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]); @@ -1032,7 +1031,8 @@ describe('DefaultEntitiesCatalog', () => { const request: PaginatedEntitiesInitialRequest = { filter, limit: 100, - sortField: 'metadata.name', + + sortFields: [{ field: 'metadata.name' }], query: 'cAt ', }; const response = await catalog.paginatedEntities(request); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 9190c76dcc..022313db7c 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -35,6 +35,7 @@ import { EntityFacetsResponse, EntityFilter, EntityPagination, + EntitySortField, PaginatedEntitiesRequest, PaginatedEntitiesResponse, } from '../catalog/types'; @@ -54,8 +55,10 @@ import { isPaginatedEntitiesInitialRequest, } from './util'; -const defaultSortField = 'metadata.name'; -const defaultSortFieldOrder = 'asc'; +const defaultSortField: EntitySortField = { + field: 'metadata.name', + order: 'asc', +}; function parsePagination(input?: EntityPagination): { limit?: number; @@ -328,19 +331,26 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const db = this.database; const limit = request?.limit ?? 20; - const cursor: Omit & { sortFieldId?: string } = { + const cursor: Omit & { sortFieldIds?: string[] } = { firstFieldId: '', - sortField: defaultSortField, - sortFieldOrder: defaultSortFieldOrder, + sortFields: [defaultSortField], isPrevious: false, ...parseCursorFromRequest(request), }; const isFetchingBackwards = cursor.isPrevious; + // TODO(vinzscam): at the moment only a single sortField is supported + const sortField: EntitySortField = { + ...defaultSortField, + ...cursor.sortFields[0], + }; + + const sortFieldId = cursor.sortFieldIds?.[0]; + const dbQuery = db('search') .join('final_entities', 'search.entity_id', 'final_entities.entity_id') - .where('key', cursor.sortField); + .where('key', sortField.field); if (cursor.filter) { parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id'); @@ -356,21 +366,19 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const countQuery = dbQuery.clone(); - const isOrderingDescending = cursor.sortFieldOrder === 'desc'; - if (cursor.sortFieldId) { + const isOrderingDescending = sortField.order === 'desc'; + if (sortFieldId) { dbQuery.andWhere( 'value', isFetchingBackwards !== isOrderingDescending ? '<' : '>', - cursor.sortFieldId, + sortFieldId, ); } dbQuery .orderBy( 'value', - isFetchingBackwards - ? invertOrder(cursor.sortFieldOrder) - : cursor.sortFieldOrder, + isFetchingBackwards ? invertOrder(sortField.order) : sortField.order, ) // fetch an extra item to check if there are more items. .limit(isFetchingBackwards ? limit : limit + 1); @@ -408,7 +416,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const nextCursor = hasMoreResults ? encodeCursor({ ...cursor, - sortFieldId: rows[rows.length - 1].value, + sortFieldIds: [rows[rows.length - 1].value], firstFieldId, isPrevious: false, totalItems, @@ -421,7 +429,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { rows[0].value !== cursor.firstFieldId ? encodeCursor({ ...cursor, - sortFieldId: rows[0].value, + sortFieldIds: [rows[0].value], firstFieldId: cursor.firstFieldId, isPrevious: true, totalItems, @@ -625,13 +633,8 @@ function parseCursorFromRequest( request?: PaginatedEntitiesRequest, ): Partial { if (isPaginatedEntitiesInitialRequest(request)) { - const { - filter, - sortField = defaultSortField, - sortFieldOrder = defaultSortFieldOrder, - query, - } = request; - return { filter, sortField, sortFieldOrder, query }; + const { filter, sortFields = [defaultSortField], query } = request; + return { filter, sortFields, query }; } if (isPaginatedEntitiesCursorRequest(request)) { try { @@ -646,6 +649,6 @@ function parseCursorFromRequest( return {}; } -function invertOrder(order: Cursor['sortFieldOrder']) { +function invertOrder(order: EntitySortField['order']) { return order === 'asc' ? 'desc' : 'asc'; } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 6e2d52c63e..c3b96fbfd8 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -163,7 +163,7 @@ describe('createRouter readonly disabled', () => { totalItems: 0, }); const response = await request(app).get( - '/v2beta1/entities?filter=a=1,a=2,b=3&filter=c=4', + '/v2beta1/entities?filter=a=1,a=2,b=3&filter=c=4&sortField=metadata.name,asc&sortField=metadata.uid,desc', ); expect(response.status).toEqual(200); @@ -180,6 +180,10 @@ describe('createRouter readonly disabled', () => { { allOf: [{ key: 'c', values: ['4'] }] }, ], }, + sortFields: [ + { field: 'metadata.name', order: 'asc' }, + { field: 'metadata.uid', order: 'desc' }, + ], }); }); diff --git a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts b/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts new file mode 100644 index 0000000000..35f7c2cac0 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { parseEntitySortFieldParams } from './parseEntitySortFieldParams'; + +describe('parseEntitySortFieldParams', () => { + it('supports no sort fields', () => { + const result = parseEntitySortFieldParams({}); + expect(result).toEqual(undefined); + }); + + it('supports single sortField', () => { + const result = parseEntitySortFieldParams({ + sortField: ['metadata.name,desc'], + })!; + expect(result).toEqual([{ field: 'metadata.name', order: 'desc' }]); + }); + + it('supports single sortField without order', () => { + const result = parseEntitySortFieldParams({ + sortField: ['metadata.name'], + })!; + expect(result).toEqual([{ field: 'metadata.name' }]); + }); + + it('supports multiple sort fields', () => { + const result = parseEntitySortFieldParams({ + sortField: ['metadata.name,desc', 'metadata.uid,asc'], + }); + + expect(result).toEqual([ + { field: 'metadata.name', order: 'desc' }, + { field: 'metadata.uid', order: 'asc' }, + ]); + }); + + it('throws if sortField order is not valid', () => { + expect(() => + parseEntitySortFieldParams({ + sortField: ['metadata.name,desc', 'metadata.uid,invalid'], + }), + ).toThrow(/Invalid sort field order/); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts b/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts new file mode 100644 index 0000000000..d16295aa30 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { EntitySortField } from '../../catalog/types'; +import { parseStringsParam } from './common'; + +export function parseEntitySortFieldParams( + params: Record, +): EntitySortField[] | undefined { + const sortFieldStrings = parseStringsParam(params.sortField, 'sortField'); + if (!sortFieldStrings) { + return undefined; + } + + return sortFieldStrings.map(sortFieldString => { + const [field, order] = sortFieldString.split(','); + + if (order !== undefined && !isOrder(order)) { + throw new InputError('Invalid sort field order, must be asc or desc'); + } + return { field, order }; + }); +} + +export function isOrder(order: string): order is 'asc' | 'desc' { + return ['asc', 'desc'].includes(order); +} diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts index 838b931888..53d44c8ca5 100644 --- a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts @@ -28,8 +28,7 @@ describe('parsePaginatedEntitiesParams', () => { fields: ['kind'], limit: '3', filter: ['a=1', 'b=2'], - sortField: 'sortField', - sortFieldOrder: 'desc', + sortField: ['metadata.name,desc'], query: 'query', }; const parsedObj = parsePaginatedEntitiesParams( @@ -37,8 +36,9 @@ describe('parsePaginatedEntitiesParams', () => { ) as PaginatedEntitiesInitialRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); - expect(parsedObj.sortField).toBe('sortField'); - expect(parsedObj.sortFieldOrder).toBe('desc'); + expect(parsedObj.sortFields).toEqual([ + { field: 'metadata.name', order: 'desc' }, + ]); expect(parsedObj.filter).toBeDefined(); expect(parsedObj.query).toBe('query'); expect(parsedObj).not.toHaveProperty('authorizationToken'); @@ -50,8 +50,7 @@ describe('parsePaginatedEntitiesParams', () => { ) as PaginatedEntitiesInitialRequest; expect(parsedObj.limit).toBeUndefined(); expect(parsedObj.fields).toBeUndefined(); - expect(parsedObj.sortField).toBeUndefined(); - expect(parsedObj.sortFieldOrder).toBeUndefined(); + expect(parsedObj.sortFields).toBeUndefined(); expect(parsedObj.filter).toBeUndefined(); expect(parsedObj.query).toBeUndefined(); expect(parsedObj).not.toHaveProperty('authorizationToken'); @@ -63,8 +62,7 @@ describe('parsePaginatedEntitiesParams', () => { limit: 'asd', }, { filter: 3 }, - { sortField: [] }, - { sortFieldOrder: 'something' }, + { sortField: ['metadata.uid,diagonal'] }, { fields: [4] }, { query: [] }, ])('should throw if some parameter is not valid %p', params => { diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts index 84d519ec0b..f5730d3297 100644 --- a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { InputError } from '@backstage/errors'; import { PaginatedEntitiesCursorRequest, PaginatedEntitiesInitialRequest, @@ -22,6 +21,7 @@ import { } from '../../catalog/types'; import { parseIntegerParam, parseStringParam } from './common'; import { parseEntityFilterParams } from './parseEntityFilterParams'; +import { parseEntitySortFieldParams } from './parseEntitySortFieldParams'; import { parseEntityTransformParams } from './parseEntityTransformParams'; export function parsePaginatedEntitiesParams( @@ -42,33 +42,16 @@ export function parsePaginatedEntitiesParams( const filter = parseEntityFilterParams(params); const query = parseStringParam(params.query, 'query'); - const sortField = parseStringParam(params.sortField, 'sortField'); - const sortFieldOrder = parseSortFieldOrder( - params.sortFieldOrder, - 'sortFieldOrder', - ); + const sortFields = parseEntitySortFieldParams(params); const response: Omit = { fields, filter, limit, - sortField, - sortFieldOrder, + sortFields, query, }; return response; } - -function parseSortFieldOrder(sortFieldOrder: unknown, ctx: string) { - const isSortFieldOrder = - sortFieldOrder === undefined || - sortFieldOrder === 'asc' || - sortFieldOrder === 'desc'; - - if (isSortFieldOrder) { - return sortFieldOrder; - } - throw new InputError(`Invalid ${ctx}, not asc or desc`); -} From 34b15da9a7bf4bc236cfe9d83773e383e44f8750 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Tue, 13 Sep 2022 11:00:38 +0200 Subject: [PATCH 10/50] catalog-client: export EntitySortField Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 9 +++++++-- packages/catalog-client/src/types/api.ts | 2 ++ packages/catalog-client/src/types/index.ts | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index f612407b1d..22afa0457e 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -188,6 +188,12 @@ export type EntityOrderQuery = order: 'asc' | 'desc'; }>; +// @public +export type EntitySortField = { + field: string; + order: 'asc' | 'desc'; +}; + // @public export interface GetEntitiesByRefsRequest { entityRefs: string[]; @@ -262,9 +268,8 @@ export type GetPaginatedEntitiesInitialRequest = fields?: string[]; limit?: number; filter?: EntitiesFilter; - sortField?: string; + sortFields?: EntitySortField[]; query?: string; - sortFieldOrder?: 'asc' | 'desc'; } | undefined; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index d006a6877c..26c16af64d 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -368,6 +368,8 @@ export type EntitiesFilter = /** * Used for sorting the entities. + * + * @public */ export type EntitySortField = { field: string; order: 'asc' | 'desc' }; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index e85b86f4af..61a993eb19 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -26,6 +26,7 @@ export type { GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, EntitiesFilter, + EntitySortField, GetEntitiesRequest, GetEntitiesResponse, GetEntityAncestorsRequest, From f2776735343d29988a90fdeb639b0f6d880ae3b3 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 29 Dec 2022 22:54:46 -0600 Subject: [PATCH 11/50] catalog-backend: improve cursor properties and docs Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 966e09b821..30472782a8 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -293,13 +293,11 @@ export type Cursor = { */ sortFields: EntitySortField[]; /** - * The value of the cursor of the last item returned. - * This is used for performing pagination. + * The values of the fields of a specific item used for paginating the data. */ - sortFieldIds: string[]; - + sortFieldValues: string[]; /** - * A filter to apply on the full list of entities. + * A filter to be applied to the full list of entities. */ filter?: EntityFilter; /** @@ -307,15 +305,15 @@ export type Cursor = { */ isPrevious: boolean; /** - * Filter the data by name. + * Used for filtering the data by name. */ query?: string; /** - * Sort field id of the first item. - * The catalog uses this field internally for understanding when the beginning + * The value of the fields of the first returned item used for paginating the data. + * The catalog uses this field internally to understand if the beginning * of the list has been reached when performing cursor based pagination. */ - firstFieldId: string; + firstSortFieldValues?: string[]; /** * The number of items that match the provided filters */ From 77a08f4ffce0fab40cbc52ea5d31a5eb8b37ab54 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 29 Dec 2022 22:56:07 -0600 Subject: [PATCH 12/50] catalog-backend: handle clashing items on pagination Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 134 +++++++++++++++--- .../src/service/DefaultEntitiesCatalog.ts | 73 +++++++--- 2 files changed, 168 insertions(+), 39 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index af4bacadaa..d5619ec540 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -85,8 +85,7 @@ describe('DefaultEntitiesCatalog', () => { return id; } - async function addEntityToSearch(knex: Knex, entity: Entity) { - const id = uuid(); + async function addEntityToSearch(knex: Knex, entity: Entity, id = uuid()) { const entityRef = stringifyEntityRef(entity); const entityJson = JSON.stringify(entity); @@ -746,7 +745,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]); expect(response2.nextCursor).toBeDefined(); expect(response2.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response2.totalItems).toBe(names.length); // third request (forward) const request3: PaginatedEntitiesCursorRequest = { @@ -757,7 +756,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]); expect(response3.nextCursor).toBeDefined(); expect(response3.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response3.totalItems).toBe(names.length); // fourth request (backwards) const request4: PaginatedEntitiesCursorRequest = { @@ -768,7 +767,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]); expect(response4.nextCursor).toBeDefined(); expect(response4.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response4.totalItems).toBe(names.length); // fifth request (backwards) const request5: PaginatedEntitiesCursorRequest = { @@ -779,7 +778,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]); expect(response5.nextCursor).toBeDefined(); expect(response5.prevCursor).toBeUndefined(); - expect(response1.totalItems).toBe(names.length); + expect(response5.totalItems).toBe(names.length); // sixth request (forward) const request6: PaginatedEntitiesCursorRequest = { @@ -790,7 +789,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]); expect(response6.nextCursor).toBeDefined(); expect(response6.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response6.totalItems).toBe(names.length); // seventh request (forward) const request7: PaginatedEntitiesCursorRequest = { @@ -801,7 +800,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]); expect(response7.nextCursor).toBeDefined(); expect(response7.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response7.totalItems).toBe(names.length); // seventh.2 request (forward with a different limit) const request7bis: PaginatedEntitiesCursorRequest = { @@ -816,7 +815,7 @@ describe('DefaultEntitiesCatalog', () => { ]); expect(response7bis.nextCursor).toBeUndefined(); expect(response7bis.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response7bis.totalItems).toBe(names.length); // last request (forward) const request8: PaginatedEntitiesCursorRequest = { @@ -827,7 +826,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response8.entities).toEqual([entityFrom('G')]); expect(response8.nextCursor).toBeUndefined(); expect(response8.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response8.totalItems).toBe(names.length); }, ); @@ -898,7 +897,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]); expect(response2.nextCursor).toBeDefined(); expect(response2.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response2.totalItems).toBe(names.length); // third request (forward) const request3: PaginatedEntitiesCursorRequest = { @@ -909,7 +908,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]); expect(response3.nextCursor).toBeDefined(); expect(response3.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response3.totalItems).toBe(names.length); // fourth request (backwards) const request4: PaginatedEntitiesCursorRequest = { @@ -921,7 +920,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]); expect(response4.nextCursor).toBeDefined(); expect(response4.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response4.totalItems).toBe(names.length); // fifth request (backwards) const request5: PaginatedEntitiesCursorRequest = { @@ -932,7 +931,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]); expect(response5.nextCursor).toBeDefined(); expect(response5.prevCursor).toBeUndefined(); - expect(response1.totalItems).toBe(names.length); + expect(response5.totalItems).toBe(names.length); // sixth request (forward) const request6: PaginatedEntitiesCursorRequest = { @@ -943,7 +942,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]); expect(response6.nextCursor).toBeDefined(); expect(response6.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response6.totalItems).toBe(names.length); // seventh request (forward) const request7: PaginatedEntitiesCursorRequest = { @@ -954,7 +953,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]); expect(response7.nextCursor).toBeDefined(); expect(response7.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response7.totalItems).toBe(names.length); // seventh.2 request (forward with a different limit) const request7bis: PaginatedEntitiesCursorRequest = { @@ -969,7 +968,7 @@ describe('DefaultEntitiesCatalog', () => { ]); expect(response7bis.nextCursor).toBeUndefined(); expect(response7bis.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response7bis.totalItems).toBe(names.length); // last request (forward) const request8: PaginatedEntitiesCursorRequest = { @@ -980,7 +979,7 @@ describe('DefaultEntitiesCatalog', () => { expect(response8.entities).toEqual([entityFrom('A')]); expect(response8.nextCursor).toBeUndefined(); expect(response8.prevCursor).toBeDefined(); - expect(response1.totalItems).toBe(names.length); + expect(response8.totalItems).toBe(names.length); }, ); @@ -1073,6 +1072,105 @@ describe('DefaultEntitiesCatalog', () => { expect(response).toEqual({ totalItems: 20, entities: [] }); }, ); + + it.each(databases.eachSupportedId())( + 'should paginate results accordingly in case of clashing items, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string, namespace?: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name, ...(!!namespace && { namespace }) }, + spec: { should_include_this: 'yes' }, + }; + } + + await Promise.all([ + addEntityToSearch(knex, entityFrom('AA')), + addEntityToSearch(knex, entityFrom('AA', 'namespace2')), + addEntityToSearch(knex, entityFrom('AA', 'namespace3')), + addEntityToSearch(knex, entityFrom('AA', 'namespace4')), + addEntityToSearch(knex, entityFrom('CC')), + addEntityToSearch(knex, entityFrom('DD')), + ]); + + const catalog = new DefaultEntitiesCatalog(knex); + + const limit = 2; + + // initial request + const request1: PaginatedEntitiesInitialRequest = { + limit, + sortFields: [{ field: 'metadata.name' }], + }; + const response1 = await catalog.paginatedEntities(request1); + expect(response1.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response1.nextCursor).toBeDefined(); + expect(response1.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(6); + + // second request (forward) + const request2: PaginatedEntitiesCursorRequest = { + cursor: response1.nextCursor!, + limit, + }; + const response2 = await catalog.paginatedEntities(request2); + expect(response2.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response2.nextCursor).toBeDefined(); + expect(response2.prevCursor).toBeDefined(); + expect(response2.totalItems).toBe(6); + + // third request (forward) + const request3: PaginatedEntitiesCursorRequest = { + cursor: response2.nextCursor!, + limit, + }; + const response3 = await catalog.paginatedEntities(request3); + expect(response3.entities).toEqual([ + entityFrom('CC'), + entityFrom('DD'), + ]); + expect(response3.nextCursor).toBeUndefined(); + expect(response3.prevCursor).toBeDefined(); + expect(response3.totalItems).toBe(6); + + // forth request (backward) + const request4: PaginatedEntitiesCursorRequest = { + cursor: response3.prevCursor!, + limit, + }; + const response4 = await catalog.paginatedEntities(request4); + expect(response4.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response4.nextCursor).toBeDefined(); + expect(response4.prevCursor).toBeDefined(); + expect(response4.totalItems).toBe(6); + + // fifth request (backward) + const request5: PaginatedEntitiesCursorRequest = { + cursor: response4.prevCursor!, + limit, + }; + const response5 = await catalog.paginatedEntities(request5); + expect(response5.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response5.nextCursor).toBeDefined(); + expect(response5.prevCursor).toBeUndefined(); + expect(response5.totalItems).toBe(6); + }, + ); }); describe('removeEntityByUid', () => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 022313db7c..ce330eb9c1 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -20,8 +20,8 @@ import { stringifyEntityRef, } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; -import lodash from 'lodash'; import { Knex } from 'knex'; +import { isEqual } from 'lodash'; import { EntitiesBatchRequest, EntitiesBatchResponse, @@ -331,8 +331,9 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const db = this.database; const limit = request?.limit ?? 20; - const cursor: Omit & { sortFieldIds?: string[] } = { - firstFieldId: '', + const cursor: Omit & { + sortFieldValues?: string[]; + } = { sortFields: [defaultSortField], isPrevious: false, ...parseCursorFromRequest(request), @@ -346,11 +347,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ...cursor.sortFields[0], }; - const sortFieldId = cursor.sortFieldIds?.[0]; + const [sortFieldId, metadataSortFieldId] = cursor.sortFieldValues || []; const dbQuery = db('search') .join('final_entities', 'search.entity_id', 'final_entities.entity_id') - .where('key', sortField.field); + .where('search.key', sortField.field); if (cursor.filter) { parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id'); @@ -368,18 +369,36 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isOrderingDescending = sortField.order === 'desc'; if (sortFieldId) { - dbQuery.andWhere( - 'value', - isFetchingBackwards !== isOrderingDescending ? '<' : '>', - sortFieldId, - ); + dbQuery + .andWhere( + 'value', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + sortFieldId, + ) + .orWhere(function nested() { + this.where('value', '=', sortFieldId).andWhere( + 'search.entity_id', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + metadataSortFieldId, + ); + }); } dbQuery - .orderBy( - 'value', - isFetchingBackwards ? invertOrder(sortField.order) : sortField.order, - ) + .orderBy([ + { + column: 'value', + order: isFetchingBackwards + ? invertOrder(sortField.order) + : sortField.order, + }, + { + column: 'search.entity_id', + order: isFetchingBackwards + ? invertOrder(sortField.order) + : sortField.order, + }, + ]) // fetch an extra item to check if there are more items. .limit(isFetchingBackwards ? limit : limit + 1); @@ -409,15 +428,22 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { rows.length -= 1; } - const isInitialRequest = cursor.firstFieldId === ''; + const isInitialRequest = cursor.firstSortFieldValues === undefined; - const firstFieldId = cursor.firstFieldId || rows[0]?.value; + const firstSortFieldValues = cursor.firstSortFieldValues || [ + rows[0]?.value, + rows[0]?.entity_id, + ]; const nextCursor = hasMoreResults ? encodeCursor({ ...cursor, - sortFieldIds: [rows[rows.length - 1].value], - firstFieldId, + sortFieldValues: [ + // TODO generalize + rows[rows.length - 1].value, + rows[rows.length - 1].entity_id, + ], + firstSortFieldValues, isPrevious: false, totalItems, }) @@ -426,11 +452,12 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const prevCursor = !isInitialRequest && rows.length > 0 && - rows[0].value !== cursor.firstFieldId + !isEqual(sortFieldsFromRow(rows[0]), cursor.firstSortFieldValues) ? encodeCursor({ ...cursor, - sortFieldIds: [rows[0].value], - firstFieldId: cursor.firstFieldId, + // TODO generalize + sortFieldValues: [rows[0].value, rows[0].entity_id], + firstSortFieldValues: cursor.firstSortFieldValues, isPrevious: true, totalItems, }) @@ -652,3 +679,7 @@ function parseCursorFromRequest( function invertOrder(order: EntitySortField['order']) { return order === 'asc' ? 'desc' : 'asc'; } + +function sortFieldsFromRow(row: DbSearchRow) { + return [row.value, row.entity_id]; +} From f0135c34ab9581c22e10d9c44fb247b38fcd1efb Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Mon, 2 Jan 2023 11:36:24 -0600 Subject: [PATCH 13/50] catalog: destroy knex connections once done Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 155 +++++++++--------- 1 file changed, 75 insertions(+), 80 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index d5619ec540..9d03cf5c61 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -35,6 +35,12 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { EntitiesRequest } from '../catalog/types'; describe('DefaultEntitiesCatalog', () => { + let knex: Knex; + + afterEach(async () => { + await knex.destroy(); + }); + const databases = TestDatabases.create({ ids: ['MYSQL_8', 'POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); @@ -42,13 +48,11 @@ describe('DefaultEntitiesCatalog', () => { const stitcher: Stitcher = { stitch } as any; async function createDatabase(databaseId: TestDatabaseId) { - const knex = await databases.init(databaseId); + knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); - return { knex }; } async function addEntity( - knex: Knex, entity: Entity, parents: { source?: string; entity?: Entity }[], ) { @@ -85,7 +89,7 @@ describe('DefaultEntitiesCatalog', () => { return id; } - async function addEntityToSearch(knex: Knex, entity: Entity, id = uuid()) { + async function addEntityToSearch(entity: Entity, id = uuid()) { const entityRef = stringifyEntityRef(entity); const entityJson = JSON.stringify(entity); @@ -123,7 +127,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return the ancestry with one parent, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const grandparent: Entity = { apiVersion: 'a', @@ -144,9 +148,9 @@ describe('DefaultEntitiesCatalog', () => { spec: {}, }; - await addEntity(knex, grandparent, [{ source: 's' }]); - await addEntity(knex, parent, [{ entity: grandparent }]); - await addEntity(knex, root, [{ entity: parent }]); + await addEntity(grandparent, [{ source: 's' }]); + await addEntity(parent, [{ entity: grandparent }]); + await addEntity(root, [{ entity: parent }]); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const result = await catalog.entityAncestry('k:default/root'); @@ -177,7 +181,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should throw error if the entity does not exist, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const catalog = new DefaultEntitiesCatalog(knex, stitcher); await expect(() => catalog.entityAncestry('k:default/root'), @@ -189,7 +193,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return the ancestry with multiple parents, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const grandparent: Entity = { apiVersion: 'a', @@ -216,10 +220,10 @@ describe('DefaultEntitiesCatalog', () => { spec: {}, }; - await addEntity(knex, grandparent, [{ source: 's' }]); - await addEntity(knex, parent1, [{ entity: grandparent }]); - await addEntity(knex, parent2, [{ entity: grandparent }]); - await addEntity(knex, root, [{ entity: parent1 }, { entity: parent2 }]); + await addEntity(grandparent, [{ source: 's' }]); + await addEntity(parent1, [{ entity: grandparent }]); + await addEntity(parent2, [{ entity: grandparent }]); + await addEntity(root, [{ entity: parent1 }, { entity: parent2 }]); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const result = await catalog.entityAncestry('k:default/root'); @@ -260,7 +264,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entity for simple filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -275,8 +279,8 @@ describe('DefaultEntitiesCatalog', () => { test: 'test value', }, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const testFilter = { @@ -294,7 +298,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entity for negation filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -309,8 +313,8 @@ describe('DefaultEntitiesCatalog', () => { test: 'test value', }, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const testFilter = { @@ -330,7 +334,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entities for nested filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -355,10 +359,10 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'four', org: 'b', color: 'blue' }, spec: {}, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - await addEntityToSearch(knex, entity3); - await addEntityToSearch(knex, entity4); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + await addEntityToSearch(entity3); + await addEntityToSearch(entity4); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const testFilter1 = { @@ -400,7 +404,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return correct entities for complex negation filter, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -413,8 +417,8 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'two', org: 'b', desc: 'description' }, spec: {}, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const testFilter1 = { @@ -443,7 +447,7 @@ describe('DefaultEntitiesCatalog', () => { 'should return no matches for an empty values array, %p', // NOTE: An empty values array is not a sensible input in a realistic scenario. async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', kind: 'k', @@ -456,8 +460,8 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'two' }, spec: {}, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); const catalog = new DefaultEntitiesCatalog(knex, stitcher); const testFilter = { @@ -475,9 +479,8 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return both target and targetRef for entities', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -488,7 +491,6 @@ describe('DefaultEntitiesCatalog', () => { [], ); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -681,7 +683,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return paginated entities and scroll the items accordingly, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); function entityFrom(name: string) { return { @@ -711,9 +713,7 @@ describe('DefaultEntitiesCatalog', () => { ]; await Promise.all( - entities - .concat(notFoundEntities) - .map(e => addEntityToSearch(knex, e)), + entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), ); const catalog = new DefaultEntitiesCatalog(knex, stitcher); @@ -833,7 +833,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should return paginated entities ordered in descending order and scroll the items accordingly, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); function entityFrom(name: string) { return { @@ -863,9 +863,7 @@ describe('DefaultEntitiesCatalog', () => { ]; await Promise.all( - entities - .concat(notFoundEntities) - .map(e => addEntityToSearch(knex, e)), + entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), ); const catalog = new DefaultEntitiesCatalog(knex, stitcher); @@ -986,7 +984,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should filter the results when query is provided, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); function entityFrom(name: string) { return { @@ -1016,9 +1014,7 @@ describe('DefaultEntitiesCatalog', () => { ]; await Promise.all( - entities - .concat(notFoundEntities) - .map(e => addEntityToSearch(knex, e)), + entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), ); const catalog = new DefaultEntitiesCatalog(knex, stitcher); @@ -1049,13 +1045,13 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'should include totalItems and empty entities in the response in case limit is zero, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); await Promise.all( Array(20) .fill(0) .map(() => - addEntityToSearch(knex, { + addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { name: v4() }, @@ -1074,9 +1070,9 @@ describe('DefaultEntitiesCatalog', () => { ); it.each(databases.eachSupportedId())( - 'should paginate results accordingly in case of clashing items, %p', + 'should paginate results without sort fields, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); function entityFrom(name: string, namespace?: string) { return { @@ -1088,12 +1084,12 @@ describe('DefaultEntitiesCatalog', () => { } await Promise.all([ - addEntityToSearch(knex, entityFrom('AA')), - addEntityToSearch(knex, entityFrom('AA', 'namespace2')), - addEntityToSearch(knex, entityFrom('AA', 'namespace3')), - addEntityToSearch(knex, entityFrom('AA', 'namespace4')), - addEntityToSearch(knex, entityFrom('CC')), - addEntityToSearch(knex, entityFrom('DD')), + addEntityToSearch(entityFrom('AA'), 'id1'), + addEntityToSearch(entityFrom('CC'), 'id2'), + addEntityToSearch(entityFrom('AA', 'namespace2'), 'id4'), + addEntityToSearch(entityFrom('AA', 'namespace3'), 'id5'), + addEntityToSearch(entityFrom('AA', 'namespace4'), 'id6'), + addEntityToSearch(entityFrom('DD'), 'id3'), ]); const catalog = new DefaultEntitiesCatalog(knex); @@ -1103,12 +1099,11 @@ describe('DefaultEntitiesCatalog', () => { // initial request const request1: PaginatedEntitiesInitialRequest = { limit, - sortFields: [{ field: 'metadata.name' }], }; const response1 = await catalog.paginatedEntities(request1); expect(response1.entities).toMatchObject([ entityFrom('AA'), - entityFrom('AA'), + entityFrom('CC'), ]); expect(response1.nextCursor).toBeDefined(); expect(response1.prevCursor).toBeUndefined(); @@ -1121,7 +1116,7 @@ describe('DefaultEntitiesCatalog', () => { }; const response2 = await catalog.paginatedEntities(request2); expect(response2.entities).toMatchObject([ - entityFrom('AA'), + entityFrom('DD'), entityFrom('AA'), ]); expect(response2.nextCursor).toBeDefined(); @@ -1135,8 +1130,8 @@ describe('DefaultEntitiesCatalog', () => { }; const response3 = await catalog.paginatedEntities(request3); expect(response3.entities).toEqual([ - entityFrom('CC'), - entityFrom('DD'), + entityFrom('AA'), + entityFrom('AA'), ]); expect(response3.nextCursor).toBeUndefined(); expect(response3.prevCursor).toBeDefined(); @@ -1149,7 +1144,7 @@ describe('DefaultEntitiesCatalog', () => { }; const response4 = await catalog.paginatedEntities(request4); expect(response4.entities).toMatchObject([ - entityFrom('AA'), + entityFrom('DD'), entityFrom('AA'), ]); expect(response4.nextCursor).toBeDefined(); @@ -1164,7 +1159,7 @@ describe('DefaultEntitiesCatalog', () => { const response5 = await catalog.paginatedEntities(request5); expect(response5.entities).toMatchObject([ entityFrom('AA'), - entityFrom('AA'), + entityFrom('CC'), ]); expect(response5.nextCursor).toBeDefined(); expect(response5.prevCursor).toBeUndefined(); @@ -1177,7 +1172,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'also clears parent hashes, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const grandparent: Entity = { apiVersion: 'a', @@ -1216,15 +1211,15 @@ describe('DefaultEntitiesCatalog', () => { spec: {}, }; - await addEntity(knex, grandparent, [{ source: 's' }]); - await addEntity(knex, parent1, [{ entity: grandparent }]); - await addEntity(knex, parent2, [{ entity: grandparent }]); - const uid = await addEntity(knex, root, [ + await addEntity(grandparent, [{ source: 's' }]); + await addEntity(parent1, [{ entity: grandparent }]); + await addEntity(parent2, [{ entity: grandparent }]); + const uid = await addEntity(root, [ { entity: parent1 }, { entity: parent2 }, ]); - await addEntity(knex, unrelated1, []); - await addEntity(knex, unrelated2, []); + await addEntity(unrelated1, []); + await addEntity(unrelated2, []); await knex('refresh_state').update({ result_hash: 'not-changed' }); await knex('relations').insert({ originating_entity_id: uid, @@ -1266,21 +1261,21 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can filter and collect properly, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { name: 'one' }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { name: 'two' }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k2', metadata: { name: 'two' }, @@ -1303,9 +1298,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can match on annotations and labels with dots in them, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -1315,7 +1310,7 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -1350,9 +1345,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can match on strings in arrays, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -1361,7 +1356,7 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { From 99dafad573e66297dd4349c24df93c3ba4dbe8d0 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Mon, 2 Jan 2023 13:49:53 -0600 Subject: [PATCH 14/50] catalog-backend: add tests for clashing items during pagination Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 107 +++++++++++++++++- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 9d03cf5c61..bedf856ff7 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1069,6 +1069,105 @@ describe('DefaultEntitiesCatalog', () => { }, ); + it.each(databases.eachSupportedId())( + 'should paginate results accordingly in case of clashing items, %p', + async databaseId => { + await createDatabase(databaseId); + + function entityFrom(name: string, namespace?: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name, ...(!!namespace && { namespace }) }, + spec: { should_include_this: 'yes' }, + }; + } + + await Promise.all([ + addEntityToSearch(entityFrom('AA')), + addEntityToSearch(entityFrom('AA', 'namespace2')), + addEntityToSearch(entityFrom('AA', 'namespace3')), + addEntityToSearch(entityFrom('AA', 'namespace4')), + addEntityToSearch(entityFrom('CC')), + addEntityToSearch(entityFrom('DD')), + ]); + + const catalog = new DefaultEntitiesCatalog(knex); + + const limit = 2; + + // initial request + const request1: PaginatedEntitiesInitialRequest = { + limit, + sortFields: [{ field: 'metadata.name' }], + }; + const response1 = await catalog.paginatedEntities(request1); + expect(response1.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response1.nextCursor).toBeDefined(); + expect(response1.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(6); + + // second request (forward) + const request2: PaginatedEntitiesCursorRequest = { + cursor: response1.nextCursor!, + limit, + }; + const response2 = await catalog.paginatedEntities(request2); + expect(response2.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response2.nextCursor).toBeDefined(); + expect(response2.prevCursor).toBeDefined(); + expect(response2.totalItems).toBe(6); + + // third request (forward) + const request3: PaginatedEntitiesCursorRequest = { + cursor: response2.nextCursor!, + limit, + }; + const response3 = await catalog.paginatedEntities(request3); + expect(response3.entities).toEqual([ + entityFrom('CC'), + entityFrom('DD'), + ]); + expect(response3.nextCursor).toBeUndefined(); + expect(response3.prevCursor).toBeDefined(); + expect(response3.totalItems).toBe(6); + + // forth request (backward) + const request4: PaginatedEntitiesCursorRequest = { + cursor: response3.prevCursor!, + limit, + }; + const response4 = await catalog.paginatedEntities(request4); + expect(response4.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response4.nextCursor).toBeDefined(); + expect(response4.prevCursor).toBeDefined(); + expect(response4.totalItems).toBe(6); + + // fifth request (backward) + const request5: PaginatedEntitiesCursorRequest = { + cursor: response4.prevCursor!, + limit, + }; + const response5 = await catalog.paginatedEntities(request5); + expect(response5.entities).toMatchObject([ + entityFrom('AA'), + entityFrom('AA'), + ]); + expect(response5.nextCursor).toBeDefined(); + expect(response5.prevCursor).toBeUndefined(); + expect(response5.totalItems).toBe(6); + }, + ); + it.each(databases.eachSupportedId())( 'should paginate results without sort fields, %p', async databaseId => { @@ -1117,7 +1216,7 @@ describe('DefaultEntitiesCatalog', () => { const response2 = await catalog.paginatedEntities(request2); expect(response2.entities).toMatchObject([ entityFrom('DD'), - entityFrom('AA'), + entityFrom('AA', 'namespace2'), ]); expect(response2.nextCursor).toBeDefined(); expect(response2.prevCursor).toBeDefined(); @@ -1130,8 +1229,8 @@ describe('DefaultEntitiesCatalog', () => { }; const response3 = await catalog.paginatedEntities(request3); expect(response3.entities).toEqual([ - entityFrom('AA'), - entityFrom('AA'), + entityFrom('AA', 'namespace3'), + entityFrom('AA', 'namespace4'), ]); expect(response3.nextCursor).toBeUndefined(); expect(response3.prevCursor).toBeDefined(); @@ -1145,7 +1244,7 @@ describe('DefaultEntitiesCatalog', () => { const response4 = await catalog.paginatedEntities(request4); expect(response4.entities).toMatchObject([ entityFrom('DD'), - entityFrom('AA'), + entityFrom('AA', 'namespace2'), ]); expect(response4.nextCursor).toBeDefined(); expect(response4.prevCursor).toBeDefined(); From c14c909126a17bff5ccfc165dd4e33f7983e3981 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Mon, 2 Jan 2023 13:58:10 -0600 Subject: [PATCH 15/50] catalog-backend: improve namings on paginateEntities Signed-off-by: Vincenzo Scamporlino --- .../src/service/DefaultEntitiesCatalog.ts | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index ce330eb9c1..ba68fb9a0e 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -56,7 +56,7 @@ import { } from './util'; const defaultSortField: EntitySortField = { - field: 'metadata.name', + field: 'metadata.uid', order: 'asc', }; @@ -341,13 +341,15 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isFetchingBackwards = cursor.isPrevious; - // TODO(vinzscam): at the moment only a single sortField is supported + // TODO(vinzscam): log if multiple sort fields are provided + const sortField: EntitySortField = { ...defaultSortField, ...cursor.sortFields[0], }; - const [sortFieldId, metadataSortFieldId] = cursor.sortFieldValues || []; + const [sortFieldValue, metadataSortFieldValue] = + cursor.sortFieldValues || []; const dbQuery = db('search') .join('final_entities', 'search.entity_id', 'final_entities.entity_id') @@ -368,20 +370,20 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const countQuery = dbQuery.clone(); const isOrderingDescending = sortField.order === 'desc'; - if (sortFieldId) { - dbQuery - .andWhere( - 'value', + + if (sortFieldValue) { + dbQuery.andWhere( + 'value', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + sortFieldValue, + ); + dbQuery.orWhere(function nested() { + this.where('value', '=', sortFieldValue).andWhere( + 'search.entity_id', isFetchingBackwards !== isOrderingDescending ? '<' : '>', - sortFieldId, - ) - .orWhere(function nested() { - this.where('value', '=', sortFieldId).andWhere( - 'search.entity_id', - isFetchingBackwards !== isOrderingDescending ? '<' : '>', - metadataSortFieldId, - ); - }); + metadataSortFieldValue, + ); + }); } dbQuery @@ -430,19 +432,18 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isInitialRequest = cursor.firstSortFieldValues === undefined; + const firstRow = rows[0]; + const lastRow = rows[rows.length - 1]; + const firstSortFieldValues = cursor.firstSortFieldValues || [ - rows[0]?.value, - rows[0]?.entity_id, + firstRow?.value, + firstRow?.entity_id, ]; const nextCursor = hasMoreResults ? encodeCursor({ ...cursor, - sortFieldValues: [ - // TODO generalize - rows[rows.length - 1].value, - rows[rows.length - 1].entity_id, - ], + sortFieldValues: sortFieldsFromRow(lastRow), firstSortFieldValues, isPrevious: false, totalItems, @@ -452,11 +453,10 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const prevCursor = !isInitialRequest && rows.length > 0 && - !isEqual(sortFieldsFromRow(rows[0]), cursor.firstSortFieldValues) + !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues) ? encodeCursor({ ...cursor, - // TODO generalize - sortFieldValues: [rows[0].value, rows[0].entity_id], + sortFieldValues: sortFieldsFromRow(firstRow), firstSortFieldValues: cursor.firstSortFieldValues, isPrevious: true, totalItems, @@ -681,5 +681,5 @@ function invertOrder(order: EntitySortField['order']) { } function sortFieldsFromRow(row: DbSearchRow) { - return [row.value, row.entity_id]; + return [row.value!, row.entity_id]; } From 2977e34ed5bc88cf5ae42a1f0e8fca1129d4b854 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Mon, 2 Jan 2023 14:15:08 -0600 Subject: [PATCH 16/50] catalog-backend: log in case of multiple sortfields are passed Signed-off-by: Vincenzo Scamporlino --- .../src/service/CatalogBuilder.ts | 7 +- .../service/DefaultEntitiesCatalog.test.ts | 139 ++++++++++++++---- .../src/service/DefaultEntitiesCatalog.ts | 22 ++- 3 files changed, 127 insertions(+), 41 deletions(-) diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 48bf6a08ef..3dfdec09e5 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -453,10 +453,11 @@ export class CatalogBuilder { legacySingleProcessorValidation: this.legacySingleProcessorValidation, }); const stitcher = new Stitcher(dbClient, logger); - const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog( - dbClient, + const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog({ + database: dbClient, + logger, stitcher, - ); + }); let permissionEvaluator: PermissionEvaluator; if ('authorizeConditional' in permissions) { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index bedf856ff7..48ac1c4b0c 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; @@ -152,7 +153,11 @@ describe('DefaultEntitiesCatalog', () => { await addEntity(parent, [{ entity: grandparent }]); await addEntity(root, [{ entity: parent }]); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const result = await catalog.entityAncestry('k:default/root'); expect(result.rootEntityRef).toEqual('k:default/root'); @@ -182,7 +187,11 @@ describe('DefaultEntitiesCatalog', () => { 'should throw error if the entity does not exist, %p', async databaseId => { await createDatabase(databaseId); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect(() => catalog.entityAncestry('k:default/root'), ).rejects.toThrow('No such entity k:default/root'); @@ -225,7 +234,11 @@ describe('DefaultEntitiesCatalog', () => { await addEntity(parent2, [{ entity: grandparent }]); await addEntity(root, [{ entity: parent1 }, { entity: parent2 }]); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const result = await catalog.entityAncestry('k:default/root'); expect(result.rootEntityRef).toEqual('k:default/root'); @@ -281,7 +294,11 @@ describe('DefaultEntitiesCatalog', () => { }; await addEntityToSearch(entity1); await addEntityToSearch(entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter = { key: 'spec.test', @@ -315,7 +332,11 @@ describe('DefaultEntitiesCatalog', () => { }; await addEntityToSearch(entity1); await addEntityToSearch(entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter = { not: { @@ -363,7 +384,11 @@ describe('DefaultEntitiesCatalog', () => { await addEntityToSearch(entity2); await addEntityToSearch(entity3); await addEntityToSearch(entity4); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter1 = { key: 'metadata.org', @@ -419,7 +444,11 @@ describe('DefaultEntitiesCatalog', () => { }; await addEntityToSearch(entity1); await addEntityToSearch(entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter1 = { key: 'metadata.org', @@ -462,7 +491,11 @@ describe('DefaultEntitiesCatalog', () => { }; await addEntityToSearch(entity1); await addEntityToSearch(entity2); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const testFilter = { key: 'kind', @@ -505,7 +538,11 @@ describe('DefaultEntitiesCatalog', () => { }, [], ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const { entities } = await catalog.entities(); @@ -716,7 +753,11 @@ describe('DefaultEntitiesCatalog', () => { entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const filter = { key: 'spec.should_include_this', @@ -866,7 +907,11 @@ describe('DefaultEntitiesCatalog', () => { entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const filter = { key: 'spec.should_include_this', @@ -1017,7 +1062,11 @@ describe('DefaultEntitiesCatalog', () => { entities.concat(notFoundEntities).map(e => addEntityToSearch(e)), ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const filter = { key: 'spec.should_include_this', @@ -1059,7 +1108,11 @@ describe('DefaultEntitiesCatalog', () => { ), ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const request: PaginatedEntitiesInitialRequest = { limit: 0, @@ -1092,7 +1145,10 @@ describe('DefaultEntitiesCatalog', () => { addEntityToSearch(entityFrom('DD')), ]); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + }); const limit = 2; @@ -1191,7 +1247,10 @@ describe('DefaultEntitiesCatalog', () => { addEntityToSearch(entityFrom('DD'), 'id3'), ]); - const catalog = new DefaultEntitiesCatalog(knex); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + }); const limit = 2; @@ -1320,20 +1379,26 @@ describe('DefaultEntitiesCatalog', () => { await addEntity(unrelated1, []); await addEntity(unrelated2, []); await knex('refresh_state').update({ result_hash: 'not-changed' }); - await knex('relations').insert({ - originating_entity_id: uid, - type: 't', - source_entity_ref: 'k:default/root', - target_entity_ref: 'k:default/unrelated1', - }); - await knex('relations').insert({ - originating_entity_id: uid, - type: 't', - source_entity_ref: 'k:default/unrelated2', - target_entity_ref: 'k:default/root', - }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + // TODO(vinzscam): check whether this is needed + // await knex('relations').insert({ + // originating_entity_id: uid, + // type: 't', + // source_entity_ref: 'k:default/root', + // target_entity_ref: 'k:default/unrelated1', + // }); + // await knex('relations').insert({ + // originating_entity_id: uid, + // type: 't', + // source_entity_ref: 'k:default/unrelated2', + // target_entity_ref: 'k:default/root', + // }); + + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await catalog.removeEntityByUid(uid); await expect( @@ -1380,7 +1445,11 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'two' }, spec: {}, }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect(catalog.facets({ facets: ['kind'] })).resolves.toEqual({ facets: { @@ -1419,7 +1488,11 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect( catalog.facets({ @@ -1464,7 +1537,11 @@ describe('DefaultEntitiesCatalog', () => { }, spec: {}, }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect( catalog.facets({ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index ba68fb9a0e..25d232958b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -21,7 +21,8 @@ import { } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; -import { isEqual } from 'lodash'; +import { isEqual, chunk as lodashChunk } from 'lodash'; +import { Logger } from 'winston'; import { EntitiesBatchRequest, EntitiesBatchResponse, @@ -183,10 +184,15 @@ function parseFilter( } export class DefaultEntitiesCatalog implements EntitiesCatalog { - constructor( - private readonly database: Knex, - private readonly stitcher: Stitcher, - ) {} + private readonly database: Knex; + private readonly logger: Logger; + private readonly stitcher: Stitcher; + + constructor(options: { database: Knex; logger: Logger; stitcher: Stitcher }) { + this.database = options.database; + this.logger = options.logger; + this.stitcher = options.stitcher; + } async entities(request?: EntitiesRequest): Promise { const db = this.database; @@ -298,7 +304,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ): Promise { const lookup = new Map(); - for (const chunk of lodash.chunk(request.entityRefs, 200)) { + for (const chunk of lodashChunk(request.entityRefs, 200)) { let query = this.database('final_entities') .innerJoin('refresh_state', { 'refresh_state.entity_id': 'final_entities.entity_id', @@ -341,7 +347,9 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isFetchingBackwards = cursor.isPrevious; - // TODO(vinzscam): log if multiple sort fields are provided + if (cursor.sortFields.length > 1) { + this.logger.warn(`Only one sort field is supported, ignoring the rest`); + } const sortField: EntitySortField = { ...defaultSortField, From 58c78d37480a1e5d8364921ff9c75ccc99e5e37b Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Tue, 24 Jan 2023 11:12:35 +0100 Subject: [PATCH 17/50] catalog: accept null as field value Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 4 ++-- plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 30472782a8..dc85e44e7a 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -295,7 +295,7 @@ export type Cursor = { /** * The values of the fields of a specific item used for paginating the data. */ - sortFieldValues: string[]; + sortFieldValues: Array; /** * A filter to be applied to the full list of entities. */ @@ -313,7 +313,7 @@ export type Cursor = { * The catalog uses this field internally to understand if the beginning * of the list has been reached when performing cursor based pagination. */ - firstSortFieldValues?: string[]; + firstSortFieldValues?: Array; /** * The number of items that match the provided filters */ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 25d232958b..bf5c7f8242 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -338,7 +338,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const limit = request?.limit ?? 20; const cursor: Omit & { - sortFieldValues?: string[]; + sortFieldValues?: (string | null)[]; } = { sortFields: [defaultSortField], isPrevious: false, @@ -689,5 +689,5 @@ function invertOrder(order: EntitySortField['order']) { } function sortFieldsFromRow(row: DbSearchRow) { - return [row.value!, row.entity_id]; + return [row.value, row.entity_id]; } From 7c47a38cc1680cc9dc755eaabcd3d0633151ea65 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Tue, 24 Jan 2023 11:25:57 +0100 Subject: [PATCH 18/50] catalog: improve sort fields namings Signed-off-by: Vincenzo Scamporlino --- .../src/service/DefaultEntitiesCatalog.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index bf5c7f8242..53e30e663e 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -356,8 +356,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { ...cursor.sortFields[0], }; - const [sortFieldValue, metadataSortFieldValue] = - cursor.sortFieldValues || []; + const [prevItemSortFieldValue, prevItemUid] = cursor.sortFieldValues || []; const dbQuery = db('search') .join('final_entities', 'search.entity_id', 'final_entities.entity_id') @@ -379,17 +378,17 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isOrderingDescending = sortField.order === 'desc'; - if (sortFieldValue) { + if (prevItemSortFieldValue) { dbQuery.andWhere( 'value', isFetchingBackwards !== isOrderingDescending ? '<' : '>', - sortFieldValue, + prevItemSortFieldValue, ); dbQuery.orWhere(function nested() { - this.where('value', '=', sortFieldValue).andWhere( + this.where('value', '=', prevItemSortFieldValue).andWhere( 'search.entity_id', isFetchingBackwards !== isOrderingDescending ? '<' : '>', - metadataSortFieldValue, + prevItemUid, ); }); } From 959a728e71e7c165e68d267678d82d60fa7caf1a Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 27 Jan 2023 14:44:22 +0100 Subject: [PATCH 19/50] catalog-backend: dependencies fix Signed-off-by: Vincenzo Scamporlino --- .../catalog-backend/src/integration.test.ts | 6 ++- .../service/DefaultEntitiesCatalog.test.ts | 42 ++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend/src/integration.test.ts b/plugins/catalog-backend/src/integration.test.ts index 4ba046834b..05fedd13de 100644 --- a/plugins/catalog-backend/src/integration.test.ts +++ b/plugins/catalog-backend/src/integration.test.ts @@ -258,7 +258,11 @@ class TestHarness { legacySingleProcessorValidation: false, }); const stitcher = new Stitcher(db, logger); - const catalog = new DefaultEntitiesCatalog(db, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: db, + logger, + stitcher, + }); const proxyProgressTracker = new ProxyProgressTracker( new NoopProgressTracker(), ); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 48ac1c4b0c..1a24747bf3 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -571,7 +571,7 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can order and combine with filtering, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); const entity1: Entity = { apiVersion: 'a', @@ -597,12 +597,16 @@ describe('DefaultEntitiesCatalog', () => { metadata: { name: 'n4' }, spec: { a: 'baz', b: 'only' }, }; - await addEntityToSearch(knex, entity1); - await addEntityToSearch(knex, entity2); - await addEntityToSearch(knex, entity3); - await addEntityToSearch(knex, entity4); + await addEntityToSearch(entity1); + await addEntityToSearch(entity2); + await addEntityToSearch(entity3); + await addEntityToSearch(entity4); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); function f(request: EntitiesRequest): Promise { return catalog @@ -665,10 +669,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'queries for entities by ref, including duplicates, and gracefully returns null for missing entities, %p', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -679,7 +682,6 @@ describe('DefaultEntitiesCatalog', () => { [], ); await addEntity( - knex, { apiVersion: 'a', kind: 'k', @@ -690,7 +692,11 @@ describe('DefaultEntitiesCatalog', () => { [], ); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); const { items } = await catalog.entitiesBatch({ entityRefs: [ @@ -1148,6 +1154,7 @@ describe('DefaultEntitiesCatalog', () => { const catalog = new DefaultEntitiesCatalog({ database: knex, logger: getVoidLogger(), + stitcher, }); const limit = 2; @@ -1250,6 +1257,7 @@ describe('DefaultEntitiesCatalog', () => { const catalog = new DefaultEntitiesCatalog({ database: knex, logger: getVoidLogger(), + stitcher, }); const limit = 2; @@ -1563,9 +1571,9 @@ describe('DefaultEntitiesCatalog', () => { it.each(databases.eachSupportedId())( 'can match relations', async databaseId => { - const { knex } = await createDatabase(databaseId); + await createDatabase(databaseId); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -1580,7 +1588,7 @@ describe('DefaultEntitiesCatalog', () => { }, ], }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -1599,7 +1607,7 @@ describe('DefaultEntitiesCatalog', () => { }, ], }); - await addEntityToSearch(knex, { + await addEntityToSearch({ apiVersion: 'a', kind: 'k', metadata: { @@ -1615,7 +1623,11 @@ describe('DefaultEntitiesCatalog', () => { ], }); - const catalog = new DefaultEntitiesCatalog(knex, stitcher); + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); await expect( catalog.facets({ From 9c214ee62941220339eb93939571fd406b9d95ba Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 27 Jan 2023 22:38:28 +0100 Subject: [PATCH 20/50] catalog-backend: fix missing uid from search Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 130 +++++++----------- .../src/service/DefaultEntitiesCatalog.ts | 2 +- 2 files changed, 54 insertions(+), 78 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 1a24747bf3..75d120e421 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -90,7 +90,8 @@ describe('DefaultEntitiesCatalog', () => { return id; } - async function addEntityToSearch(entity: Entity, id = uuid()) { + async function addEntityToSearch(entity: Entity) { + const id = entity.metadata.uid || v4(); const entityRef = stringifyEntityRef(entity); const entityJson = JSON.stringify(entity); @@ -728,17 +729,8 @@ describe('DefaultEntitiesCatalog', () => { async databaseId => { await createDatabase(databaseId); - function entityFrom(name: string) { - return { - apiVersion: 'a', - kind: 'k', - metadata: { name }, - spec: { should_include_this: 'yes' }, - }; - } - const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; - const entities: Entity[] = names.map(entityFrom); + const entities: Entity[] = names.map(name => entityFrom(name)); const notFoundEntities: Entity[] = [ { @@ -882,17 +874,8 @@ describe('DefaultEntitiesCatalog', () => { async databaseId => { await createDatabase(databaseId); - function entityFrom(name: string) { - return { - apiVersion: 'a', - kind: 'k', - metadata: { name }, - spec: { should_include_this: 'yes' }, - }; - } - const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; - const entities: Entity[] = names.map(entityFrom); + const entities: Entity[] = names.map(name => entityFrom(name)); const notFoundEntities: Entity[] = [ { @@ -1037,17 +1020,8 @@ describe('DefaultEntitiesCatalog', () => { async databaseId => { await createDatabase(databaseId); - function entityFrom(name: string) { - return { - apiVersion: 'a', - kind: 'k', - metadata: { name }, - spec: { should_include_this: 'yes' }, - }; - } - const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's']; - const entities: Entity[] = names.map(entityFrom); + const entities: Entity[] = names.map(name => entityFrom(name)); const notFoundEntities: Entity[] = [ { @@ -1133,20 +1107,11 @@ describe('DefaultEntitiesCatalog', () => { async databaseId => { await createDatabase(databaseId); - function entityFrom(name: string, namespace?: string) { - return { - apiVersion: 'a', - kind: 'k', - metadata: { name, ...(!!namespace && { namespace }) }, - spec: { should_include_this: 'yes' }, - }; - } - await Promise.all([ addEntityToSearch(entityFrom('AA')), - addEntityToSearch(entityFrom('AA', 'namespace2')), - addEntityToSearch(entityFrom('AA', 'namespace3')), - addEntityToSearch(entityFrom('AA', 'namespace4')), + addEntityToSearch(entityFrom('AA', { namespace: 'namespace2' })), + addEntityToSearch(entityFrom('AA', { namespace: 'namespace3' })), + addEntityToSearch(entityFrom('AA', { namespace: 'namespace4' })), addEntityToSearch(entityFrom('CC')), addEntityToSearch(entityFrom('DD')), ]); @@ -1236,22 +1201,19 @@ describe('DefaultEntitiesCatalog', () => { async databaseId => { await createDatabase(databaseId); - function entityFrom(name: string, namespace?: string) { - return { - apiVersion: 'a', - kind: 'k', - metadata: { name, ...(!!namespace && { namespace }) }, - spec: { should_include_this: 'yes' }, - }; - } - await Promise.all([ - addEntityToSearch(entityFrom('AA'), 'id1'), - addEntityToSearch(entityFrom('CC'), 'id2'), - addEntityToSearch(entityFrom('AA', 'namespace2'), 'id4'), - addEntityToSearch(entityFrom('AA', 'namespace3'), 'id5'), - addEntityToSearch(entityFrom('AA', 'namespace4'), 'id6'), - addEntityToSearch(entityFrom('DD'), 'id3'), + addEntityToSearch(entityFrom('AA', { uid: 'id1' })), + addEntityToSearch(entityFrom('CC', { uid: 'id2' })), + addEntityToSearch( + entityFrom('AA', { namespace: 'namespace2', uid: 'id4' }), + ), + addEntityToSearch( + entityFrom('AA', { namespace: 'namespace3', uid: 'id5' }), + ), + addEntityToSearch( + entityFrom('AA', { namespace: 'namespace4', uid: 'id6' }), + ), + addEntityToSearch(entityFrom('DD', { uid: 'id3' })), ]); const catalog = new DefaultEntitiesCatalog({ @@ -1283,7 +1245,7 @@ describe('DefaultEntitiesCatalog', () => { const response2 = await catalog.paginatedEntities(request2); expect(response2.entities).toMatchObject([ entityFrom('DD'), - entityFrom('AA', 'namespace2'), + entityFrom('AA', { namespace: 'namespace2' }), ]); expect(response2.nextCursor).toBeDefined(); expect(response2.prevCursor).toBeDefined(); @@ -1295,9 +1257,9 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response3 = await catalog.paginatedEntities(request3); - expect(response3.entities).toEqual([ - entityFrom('AA', 'namespace3'), - entityFrom('AA', 'namespace4'), + expect(response3.entities).toMatchObject([ + entityFrom('AA', { namespace: 'namespace3' }), + entityFrom('AA', { namespace: 'namespace4' }), ]); expect(response3.nextCursor).toBeUndefined(); expect(response3.prevCursor).toBeDefined(); @@ -1311,7 +1273,7 @@ describe('DefaultEntitiesCatalog', () => { const response4 = await catalog.paginatedEntities(request4); expect(response4.entities).toMatchObject([ entityFrom('DD'), - entityFrom('AA', 'namespace2'), + entityFrom('AA', { namespace: 'namespace2' }), ]); expect(response4.nextCursor).toBeDefined(); expect(response4.prevCursor).toBeDefined(); @@ -1387,20 +1349,18 @@ describe('DefaultEntitiesCatalog', () => { await addEntity(unrelated1, []); await addEntity(unrelated2, []); await knex('refresh_state').update({ result_hash: 'not-changed' }); - - // TODO(vinzscam): check whether this is needed - // await knex('relations').insert({ - // originating_entity_id: uid, - // type: 't', - // source_entity_ref: 'k:default/root', - // target_entity_ref: 'k:default/unrelated1', - // }); - // await knex('relations').insert({ - // originating_entity_id: uid, - // type: 't', - // source_entity_ref: 'k:default/unrelated2', - // target_entity_ref: 'k:default/root', - // }); + await knex('relations').insert({ + originating_entity_id: uid, + type: 't', + source_entity_ref: 'k:default/root', + target_entity_ref: 'k:default/unrelated1', + }); + await knex('relations').insert({ + originating_entity_id: uid, + type: 't', + source_entity_ref: 'k:default/unrelated2', + target_entity_ref: 'k:default/root', + }); const catalog = new DefaultEntitiesCatalog({ database: knex, @@ -1645,3 +1605,19 @@ describe('DefaultEntitiesCatalog', () => { ); }); }); + +function entityFrom( + name: string, + { uid, namespace }: { uid?: string; namespace?: string } = {}, +) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { + name, + ...(!!namespace && { namespace }), + ...(!!uid && { uid }), + }, + spec: { should_include_this: 'yes' }, + }; +} diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 53e30e663e..37245876aa 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -113,7 +113,7 @@ function addCondition( // make a lot of sense. However, it had abysmal performance on sqlite // when datasets grew large, so we're using IN instead. const matchQuery = db('search') - .select(entityIdField) + .select('search.entity_id') .where({ key: filter.key.toLowerCase() }) .andWhere(function keyFilter() { if (filter.values) { From d3b3065a76b0749e65212f8028041fe6db143601 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 27 Jan 2023 23:31:26 +0100 Subject: [PATCH 21/50] catalog-backend: reuse EntityOrderQuery Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 10 ++-------- .../src/service/DefaultEntitiesCatalog.test.ts | 6 +++--- .../src/service/DefaultEntitiesCatalog.ts | 8 ++++---- .../src/service/request/parseEntitySortFieldParams.ts | 4 ++-- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index dc85e44e7a..c5da9726c2 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -46,12 +46,6 @@ export type EntityOrder = { order: 'asc' | 'desc'; }; -// TODO(vinzscam): remove this type in favor of EntityOrder -export type EntitySortField = { - field: string; - order?: 'asc' | 'desc' | undefined; -}; - /** * Matches rows in the search table. * @public @@ -240,7 +234,7 @@ export interface PaginatedEntitiesInitialRequest { fields?: (entity: Entity) => Entity; limit?: number; filter?: EntityFilter; - sortFields?: EntitySortField[]; + sortFields?: EntityOrder[]; query?: string; } @@ -291,7 +285,7 @@ export type Cursor = { * An array of fields used for sorting the data. * For example, [ { field: 'metadata.name', order: 'asc' } ] */ - sortFields: EntitySortField[]; + sortFields: EntityOrder[]; /** * The values of the fields of a specific item used for paginating the data. */ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 75d120e421..c2f1843845 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -767,7 +767,7 @@ describe('DefaultEntitiesCatalog', () => { const request1: PaginatedEntitiesInitialRequest = { filter, limit, - sortFields: [{ field: 'metadata.name' }], + sortFields: [{ field: 'metadata.name', order: 'asc' }], }; const response1 = await catalog.paginatedEntities(request1); expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); @@ -1056,7 +1056,7 @@ describe('DefaultEntitiesCatalog', () => { filter, limit: 100, - sortFields: [{ field: 'metadata.name' }], + sortFields: [{ field: 'metadata.name', order: 'asc' }], query: 'cAt ', }; const response = await catalog.paginatedEntities(request); @@ -1127,7 +1127,7 @@ describe('DefaultEntitiesCatalog', () => { // initial request const request1: PaginatedEntitiesInitialRequest = { limit, - sortFields: [{ field: 'metadata.name' }], + sortFields: [{ field: 'metadata.name', order: 'asc' }], }; const response1 = await catalog.paginatedEntities(request1); expect(response1.entities).toMatchObject([ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 37245876aa..43e0de026d 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -36,9 +36,9 @@ import { EntityFacetsResponse, EntityFilter, EntityPagination, - EntitySortField, PaginatedEntitiesRequest, PaginatedEntitiesResponse, + EntityOrder, } from '../catalog/types'; import { DbFinalEntitiesRow, @@ -56,7 +56,7 @@ import { isPaginatedEntitiesInitialRequest, } from './util'; -const defaultSortField: EntitySortField = { +const defaultSortField: EntityOrder = { field: 'metadata.uid', order: 'asc', }; @@ -351,7 +351,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { this.logger.warn(`Only one sort field is supported, ignoring the rest`); } - const sortField: EntitySortField = { + const sortField: EntityOrder = { ...defaultSortField, ...cursor.sortFields[0], }; @@ -683,7 +683,7 @@ function parseCursorFromRequest( return {}; } -function invertOrder(order: EntitySortField['order']) { +function invertOrder(order: EntityOrder['order']) { return order === 'asc' ? 'desc' : 'asc'; } diff --git a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts b/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts index d16295aa30..5efa2de05e 100644 --- a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts +++ b/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts @@ -15,12 +15,12 @@ */ import { InputError } from '@backstage/errors'; -import { EntitySortField } from '../../catalog/types'; +import { EntityOrder } from '../../catalog/types'; import { parseStringsParam } from './common'; export function parseEntitySortFieldParams( params: Record, -): EntitySortField[] | undefined { +): EntityOrder[] | undefined { const sortFieldStrings = parseStringsParam(params.sortField, 'sortField'); if (!sortFieldStrings) { return undefined; From d1ed81af930cf59cdde2df86bc8d24eb28cc9aec Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 27 Jan 2023 23:34:57 +0100 Subject: [PATCH 22/50] catalog-client: reuse EntityOrderQuery Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 8 +------- packages/catalog-client/src/CatalogClient.ts | 4 ++-- packages/catalog-client/src/types/api.ts | 9 +-------- packages/catalog-client/src/types/index.ts | 1 - 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 22afa0457e..b9c9543801 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -188,12 +188,6 @@ export type EntityOrderQuery = order: 'asc' | 'desc'; }>; -// @public -export type EntitySortField = { - field: string; - order: 'asc' | 'desc'; -}; - // @public export interface GetEntitiesByRefsRequest { entityRefs: string[]; @@ -268,7 +262,7 @@ export type GetPaginatedEntitiesInitialRequest = fields?: string[]; limit?: number; filter?: EntitiesFilter; - sortFields?: EntitySortField[]; + sortFields?: EntityOrderQuery; query?: string; } | undefined; diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 18b614fd2e..a76a66577c 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -224,8 +224,8 @@ export class CatalogClient implements CatalogApi { params.push(`limit=${limit}`); } if (sortFields !== undefined) { - sortFields.forEach(({ field, order }) => - params.push(`sortField=${field},${order}`), + (Array.isArray(sortFields) ? sortFields : [sortFields]).forEach( + ({ field, order }) => params.push(`sortField=${field},${order}`), ); } if (fields.length) { diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 26c16af64d..103c990783 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -366,13 +366,6 @@ export type EntitiesFilter = | Record | undefined; -/** - * Used for sorting the entities. - * - * @public - */ -export type EntitySortField = { field: string; order: 'asc' | 'desc' }; - /** * The response type for {@link CatalogClient.addLocation}. * @@ -418,7 +411,7 @@ export type GetPaginatedEntitiesInitialRequest = fields?: string[]; limit?: number; filter?: EntitiesFilter; - sortFields?: EntitySortField[]; + sortFields?: EntityOrderQuery; query?: string; } | undefined; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 61a993eb19..e85b86f4af 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -26,7 +26,6 @@ export type { GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, EntitiesFilter, - EntitySortField, GetEntitiesRequest, GetEntitiesResponse, GetEntityAncestorsRequest, From c2bd94e82a3b1c68a2c58681ec015b2a95526fa5 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 27 Jan 2023 23:46:31 +0100 Subject: [PATCH 23/50] catalog-client: reuse EntityFilterQuery Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 10 ++-------- packages/catalog-client/src/CatalogClient.ts | 4 ++-- packages/catalog-client/src/types/api.ts | 14 ++------------ packages/catalog-client/src/types/index.ts | 1 - 4 files changed, 6 insertions(+), 23 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index b9c9543801..416f1bc4fd 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -159,12 +159,6 @@ export interface CatalogRequestOptions { token?: string; } -// @public -export type EntitiesFilter = - | Record[] - | Record - | undefined; - // @public export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = 'backstage.io/catalog-processing'; @@ -235,7 +229,7 @@ export interface GetEntityAncestorsResponse { // @public export interface GetEntityFacetsRequest { facets: string[]; - filter?: EntitiesFilter; + filter?: EntityFilterQuery; } // @public @@ -261,7 +255,7 @@ export type GetPaginatedEntitiesInitialRequest = | { fields?: string[]; limit?: number; - filter?: EntitiesFilter; + filter?: EntityFilterQuery; sortFields?: EntityOrderQuery; query?: string; } diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index a76a66577c..ba410beb92 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -41,7 +41,7 @@ import { GetEntitiesByRefsResponse, GetPaginatedEntitiesRequest, GetPaginatedEntitiesResponse, - EntitiesFilter, + EntityFilterQuery, GetPaginatedEntitiesCursorRequest, } from './types/api'; import { DiscoveryApi } from './types/discovery'; @@ -522,7 +522,7 @@ export class CatalogClient implements CatalogApi { return await response.json(); } - private getParams(filter: EntitiesFilter = []) { + private getParams(filter: EntityFilterQuery = []) { const params: string[] = []; // filter param can occur multiple times, for example // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 103c990783..d6aeba5d1d 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -287,7 +287,7 @@ export interface GetEntityFacetsRequest { * (exported from this package), which means that you assert on the existence * of that key, no matter what its value is. */ - filter?: EntitiesFilter; + filter?: EntityFilterQuery; /** * Dot separated paths for the facets to extract from each entity. * @@ -356,16 +356,6 @@ export type AddLocationRequest = { dryRun?: boolean; }; -/** - * Used for filtering entities that match the given patterns. - * - * @public - */ -export type EntitiesFilter = - | Record[] - | Record - | undefined; - /** * The response type for {@link CatalogClient.addLocation}. * @@ -410,7 +400,7 @@ export type GetPaginatedEntitiesInitialRequest = | { fields?: string[]; limit?: number; - filter?: EntitiesFilter; + filter?: EntityFilterQuery; sortFields?: EntityOrderQuery; query?: string; } diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index e85b86f4af..715639e1c3 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -25,7 +25,6 @@ export type { EntityOrderQuery, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, - EntitiesFilter, GetEntitiesRequest, GetEntitiesResponse, GetEntityAncestorsRequest, From 2f2baf4b6513d671425296994c99bc5a9d790171 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Sat, 28 Jan 2023 00:04:46 +0100 Subject: [PATCH 24/50] catalog-client: remove undefined from GetPaginatedEntitiesInitialRequest Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 16 +++++++--------- packages/catalog-client/src/CatalogClient.ts | 4 ++-- packages/catalog-client/src/types/api.ts | 16 +++++++--------- packages/catalog-client/src/utils.ts | 3 --- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 416f1bc4fd..47c6ff7f36 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -251,15 +251,13 @@ export type GetPaginatedEntitiesCursorRequest = { }; // @alpha -export type GetPaginatedEntitiesInitialRequest = - | { - fields?: string[]; - limit?: number; - filter?: EntityFilterQuery; - sortFields?: EntityOrderQuery; - query?: string; - } - | undefined; +export type GetPaginatedEntitiesInitialRequest = { + fields?: string[]; + limit?: number; + filter?: EntityFilterQuery; + sortFields?: EntityOrderQuery; + query?: string; +}; // @alpha export type GetPaginatedEntitiesRequest = diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index ba410beb92..7347535ca6 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -211,13 +211,13 @@ export class CatalogClient implements CatalogApi { * @alpha */ async getPaginatedEntities?( - request?: GetPaginatedEntitiesRequest, + request: GetPaginatedEntitiesRequest = {}, options?: CatalogRequestOptions, ): Promise { const params: string[] = []; if (isPaginatedEntitiesInitialRequest(request)) { - const { fields = [], filter, limit, sortFields, query } = request ?? {}; + const { fields = [], filter, limit, sortFields, query } = request; params.push(...this.getParams(filter)); if (limit !== undefined) { diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index d6aeba5d1d..4e7f679072 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -396,15 +396,13 @@ export type GetPaginatedEntitiesRequest = * * @alpha */ -export type GetPaginatedEntitiesInitialRequest = - | { - fields?: string[]; - limit?: number; - filter?: EntityFilterQuery; - sortFields?: EntityOrderQuery; - query?: string; - } - | undefined; +export type GetPaginatedEntitiesInitialRequest = { + fields?: string[]; + limit?: number; + filter?: EntityFilterQuery; + sortFields?: EntityOrderQuery; + query?: string; +}; /** * A request type for {@link CatalogClient.getPaginatedEntities}. diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts index 5a26053fb9..8a2eb9c06d 100644 --- a/packages/catalog-client/src/utils.ts +++ b/packages/catalog-client/src/utils.ts @@ -23,9 +23,6 @@ import { export function isPaginatedEntitiesInitialRequest( request: GetPaginatedEntitiesInitialRequest, ): request is GetPaginatedEntitiesInitialRequest { - if (!request) { - return true; - } return !(request as GetPaginatedEntitiesCursorRequest).cursor; } From 9b71a220dcf4b24a680f71abff493acd08442b3a Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Sat, 28 Jan 2023 00:33:24 +0100 Subject: [PATCH 25/50] catalog-backend: cleanup test Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 76 ------------------- 1 file changed, 76 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index c2f1843845..bde55ef9bb 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1527,82 +1527,6 @@ describe('DefaultEntitiesCatalog', () => { }, 60_000, ); - - it.each(databases.eachSupportedId())( - 'can match relations', - async databaseId => { - await createDatabase(databaseId); - - await addEntityToSearch({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'one', - tags: ['java', 'rust'], - }, - spec: {}, - relations: [ - { - targetRef: 'targetRef', - type: 'ownedBy', - }, - ], - }); - await addEntityToSearch({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'two', - tags: ['java', 'node'], - }, - spec: {}, - relations: [ - { - targetRef: 'anotherTargetRef', - type: 'ownedBy', - }, - { - targetRef: 'tt', - type: 'somethingelse', - }, - ], - }); - await addEntityToSearch({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'three', - tags: ['go'], - }, - spec: {}, - relations: [ - { - targetRef: 'targetRef', - type: 'ownedBy', - }, - ], - }); - - const catalog = new DefaultEntitiesCatalog({ - database: knex, - logger: getVoidLogger(), - stitcher, - }); - - await expect( - catalog.facets({ - facets: ['relations.ownedBy'], - }), - ).resolves.toEqual({ - facets: { - 'relations.ownedBy': [ - { count: 1, value: 'anotherTargetRef' }, - { count: 2, value: 'targetRef' }, - ], - }, - }); - }, - ); }); }); From 9615da8726bd6461a35d5f745f603a7dee72029e Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Tue, 31 Jan 2023 23:07:21 +0100 Subject: [PATCH 26/50] catalog-backend: fix mysql query issue Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 43e0de026d..0cc791e5de 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -368,8 +368,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const normalizedQueryByName = cursor.query?.trim(); if (normalizedQueryByName) { - dbQuery.andWhereLike( - 'value', + dbQuery.andWhereRaw( + 'value like ?', `%${normalizedQueryByName.toLocaleLowerCase('en-US')}%`, ); } From 362145175e8dfaaf87b47cafacc68f90dce2cb72 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 09:56:33 +0100 Subject: [PATCH 27/50] catalog-backend: remove alpha methods Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index c5da9726c2..451f757e65 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -177,7 +177,6 @@ export interface EntitiesCatalog { /** * Fetch entities and scroll back and forth between entities. * - * @alpha * @param request */ paginatedEntities( @@ -215,8 +214,6 @@ export interface EntitiesCatalog { /** * The request shape for {@link EntitiesCatalog.paginatedEntities}. - * - * @alpha */ export type PaginatedEntitiesRequest = | PaginatedEntitiesInitialRequest @@ -226,8 +223,6 @@ export type PaginatedEntitiesRequest = * The initial request for {@link EntitiesCatalog.paginatedEntities}. * The request take immutable properties that are going to be bound * for the current and the next pagination requests. - * - * @alpha */ export interface PaginatedEntitiesInitialRequest { authorizationToken?: string; @@ -241,8 +236,6 @@ export interface PaginatedEntitiesInitialRequest { /** * Request for {@link EntitiesCatalog.paginatedEntities} used to * move forward or backward on the data. - * - * @alpha */ export interface PaginatedEntitiesCursorRequest { authorizationToken?: string; @@ -253,8 +246,6 @@ export interface PaginatedEntitiesCursorRequest { /** * The response shape for {@link EntitiesCatalog.paginatedEntities}. - * - * @alpha */ export interface PaginatedEntitiesResponse { /** @@ -277,8 +268,6 @@ export interface PaginatedEntitiesResponse { /** * The Cursor used internally by the catalog. - * - * @alpha */ export type Cursor = { /** From 0d137f6fa5d7c31a06b1401ee7965d020bfa5abe Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 10:00:18 +0100 Subject: [PATCH 28/50] catalog-client: export types as public Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 9 ++++----- packages/catalog-client/src/CatalogClient.ts | 2 -- packages/catalog-client/src/types/api.ts | 10 +++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 47c6ff7f36..2e6b5b9203 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -58,7 +58,6 @@ export interface CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; - // @alpha getPaginatedEntities?( request?: GetPaginatedEntitiesRequest, options?: CatalogRequestOptions, @@ -243,14 +242,14 @@ export interface GetEntityFacetsResponse { >; } -// @alpha +// @public export type GetPaginatedEntitiesCursorRequest = { fields?: string[]; limit?: number; cursor: string; }; -// @alpha +// @public export type GetPaginatedEntitiesInitialRequest = { fields?: string[]; limit?: number; @@ -259,12 +258,12 @@ export type GetPaginatedEntitiesInitialRequest = { query?: string; }; -// @alpha +// @public export type GetPaginatedEntitiesRequest = | GetPaginatedEntitiesInitialRequest | GetPaginatedEntitiesCursorRequest; -// @alpha +// @public export type GetPaginatedEntitiesResponse = { entities: Entity[]; totalItems: number; diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 7347535ca6..dca46c3ab6 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -207,8 +207,6 @@ export class CatalogClient implements CatalogApi { /** * {@inheritdoc CatalogApi.getPaginatedEntities} - * - * @alpha */ async getPaginatedEntities?( request: GetPaginatedEntitiesRequest = {}, diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 4e7f679072..0f0d0fe63c 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -380,7 +380,7 @@ export type ValidateEntityResponse = /** * The request type for {@link CatalogClient.getPaginatedEntities}. * - * @alpha + * @public */ export type GetPaginatedEntitiesRequest = | GetPaginatedEntitiesInitialRequest @@ -394,7 +394,7 @@ export type GetPaginatedEntitiesRequest = * The properties filter, sortField, query and sortFieldOrder, are going * to be immutable for the entire lifecycle of the following requests. * - * @alpha + * @public */ export type GetPaginatedEntitiesInitialRequest = { fields?: string[]; @@ -409,7 +409,7 @@ export type GetPaginatedEntitiesInitialRequest = { * The method takes this type in a pagination request, following * the initial request. * - * @alpha + * @public */ export type GetPaginatedEntitiesCursorRequest = { fields?: string[]; @@ -420,7 +420,7 @@ export type GetPaginatedEntitiesCursorRequest = { /** * The response type for {@link CatalogClient.getPaginatedEntities}. * - * @alpha + * @public */ export type GetPaginatedEntitiesResponse = { /* The list of entities for the current request */ @@ -503,7 +503,7 @@ export interface CatalogApi { * with a prev function useful for navigating backwards and a next function * in case more entities matching the filters of the first request are present in the catalog. * - * @alpha + * @public * * @param request - Request parameters * @param options - Additional options From cf5c58a89fcc3bf4917ab52de7b201ac81910372 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 13:30:52 +0100 Subject: [PATCH 29/50] catalog-backend: validate cursor Signed-off-by: Vincenzo Scamporlino --- .../service/DefaultEntitiesCatalog.test.ts | 20 +++++++++++ .../src/service/DefaultEntitiesCatalog.ts | 34 ++++++++++++++++--- plugins/catalog-backend/src/service/util.ts | 10 +++--- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index bde55ef9bb..a25cbf2240 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1294,6 +1294,26 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.totalItems).toBe(6); }, ); + + it('should throw in case of malformed cursor', async () => { + const catalog = new DefaultEntitiesCatalog({ + database: knex, + logger: getVoidLogger(), + stitcher, + }); + + const cursor = Buffer.from( + JSON.stringify({ filter: 'notavalidfilter' }), + 'utf8', + ).toString('base64'); + const request: PaginatedEntitiesCursorRequest = { + cursor, + }; + + await expect(catalog.paginatedEntities(request)).rejects.toThrow( + Error('Malformed cursor, could not be parsed'), + ); + }); }); describe('removeEntityByUid', () => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 0cc791e5de..67fe77cc8b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -23,6 +23,7 @@ import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { isEqual, chunk as lodashChunk } from 'lodash'; import { Logger } from 'winston'; +import { z } from 'zod'; import { EntitiesBatchRequest, EntitiesBatchResponse, @@ -658,11 +659,39 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } } +const entityFilterParser: z.ZodSchema = z.lazy(() => + z + .object({ + key: z.string(), + values: z.array(z.string()).optional(), + }) + .or(z.object({ not: entityFilterParser })) + .or(z.object({ anyOf: z.array(entityFilterParser) })) + .or(z.object({ allOf: z.array(entityFilterParser) })), +); + +export const cursorParser: z.ZodSchema = z.object({ + sortFields: z.array( + z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }), + ), + sortFieldValues: z.array(z.string().or(z.null())), + filter: entityFilterParser.optional(), + isPrevious: z.boolean(), + query: z.string().optional(), + firstSortFieldValues: z.array(z.string().or(z.null())).optional(), + totalItems: z.number().optional(), +}); + function encodeCursor(cursor: Cursor) { const json = JSON.stringify(cursor); return Buffer.from(json, 'utf8').toString('base64'); } +function decodeCursor(encodedCursor: string) { + const json = Buffer.from(encodedCursor, 'base64').toString('utf8'); + return cursorParser.parse(JSON.parse(json)); +} + function parseCursorFromRequest( request?: PaginatedEntitiesRequest, ): Partial { @@ -672,10 +701,7 @@ function parseCursorFromRequest( } if (isPaginatedEntitiesCursorRequest(request)) { try { - const json = Buffer.from(request.cursor, 'base64').toString('utf8'); - const cursor = JSON.parse(json); - // TODO(vinzscam): validate the shit - return cursor as unknown as Cursor; + return decodeCursor(request.cursor); } catch { throw new InputError('Malformed cursor, could not be parsed'); } diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index bb86654565..958b4153f8 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -19,9 +19,9 @@ import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; import { - EntitiesRequest, PaginatedEntitiesCursorRequest, PaginatedEntitiesInitialRequest, + PaginatedEntitiesRequest, } from '../catalog/types'; export async function requireRequestBody(req: Request): Promise { @@ -72,21 +72,19 @@ export function disallowReadonlyMode(readonly: boolean) { } export function isPaginatedEntitiesInitialRequest( - input: EntitiesRequest | undefined, + input: PaginatedEntitiesRequest | undefined, ): input is PaginatedEntitiesInitialRequest { if (!input) { return false; } - // TODO(vinzscam) expand this return !isPaginatedEntitiesCursorRequest(input); } export function isPaginatedEntitiesCursorRequest( - input: EntitiesRequest | undefined, + input: PaginatedEntitiesRequest | undefined, ): input is PaginatedEntitiesCursorRequest { if (!input) { return false; } - // TODO(vinzscam) expand this - return input?.hasOwnProperty('cursor') ?? false; + return !!(input as PaginatedEntitiesCursorRequest).cursor; } From fd173aac2ca6cf01d879e24e80dadba9e5cf711d Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 13:39:18 +0100 Subject: [PATCH 30/50] catalog-client: make paginated response serializable Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 9 ++---- .../catalog-client/src/CatalogClient.test.ts | 12 ++++---- packages/catalog-client/src/CatalogClient.ts | 30 ++++--------------- packages/catalog-client/src/types/api.ts | 12 +++----- 4 files changed, 18 insertions(+), 45 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 2e6b5b9203..e39fa7eb8d 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -128,7 +128,6 @@ export class CatalogClient implements CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; - // @alpha getPaginatedEntities?( request?: GetPaginatedEntitiesRequest, options?: CatalogRequestOptions, @@ -267,12 +266,8 @@ export type GetPaginatedEntitiesRequest = export type GetPaginatedEntitiesResponse = { entities: Entity[]; totalItems: number; - next?( - request?: Omit, - ): Promise; - prev?( - request?: Omit, - ): Promise; + nextCursor?: string; + prevCursor?: string; }; // @public diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 4e5065e089..ed27bb0016 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -310,8 +310,8 @@ describe('CatalogClient', () => { const response = await client.getPaginatedEntities?.({}, { token }); expect(response?.entities).toEqual(defaultClientResponse.entities); expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); - expect(response?.next).toBeDefined(); - expect(response?.prev).toBeDefined(); + expect(response?.nextCursor).toBeDefined(); + expect(response?.prevCursor).toBeDefined(); }); it('builds multiple entity search filters properly', async () => { @@ -429,15 +429,15 @@ describe('CatalogClient', () => { }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); - expect(response?.next).toBeDefined(); - expect(response?.prev).toBeDefined(); + expect(response?.nextCursor).toBeDefined(); + expect(response?.prevCursor).toBeDefined(); - await response?.next?.(); + await client.getPaginatedEntities?.({ cursor: response!.nextCursor! }); expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( '?cursor=nextcursor', ); - await response?.prev?.(); + await client.getPaginatedEntities?.({ cursor: response!.prevCursor! }); expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( '?cursor=prevcursor', ); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index dca46c3ab6..52ed2fd8fc 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -42,7 +42,6 @@ import { GetPaginatedEntitiesRequest, GetPaginatedEntitiesResponse, EntityFilterQuery, - GetPaginatedEntitiesCursorRequest, } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; @@ -247,18 +246,12 @@ export class CatalogClient implements CatalogApi { } const query = params.length ? `?${params.join('&')}` : ''; - const { entities, totalItems, nextCursor, prevCursor } = - await this.requestRequired<{ - entities: Entity[]; - totalItems: number; - nextCursor?: string; - prevCursor?: string; - }>('GET', `/v2beta1/entities${query}`, options); - - const next = this.getEntitiesFromCursor(nextCursor, options); - const prev = this.getEntitiesFromCursor(prevCursor, options); - - return { entities, totalItems, next, prev }; + return this.requestRequired<{ + entities: Entity[]; + totalItems: number; + nextCursor?: string; + prevCursor?: string; + }>('GET', `/v2beta1/entities${query}`, options); } /** @@ -546,15 +539,4 @@ export class CatalogClient implements CatalogApi { } return params; } - - private getEntitiesFromCursor( - cursor: string | undefined, - options?: CatalogRequestOptions, - ) { - if (!cursor) { - return undefined; - } - return (request: Omit = {}) => - this.getPaginatedEntities!({ ...request, cursor }, options); - } } diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 0f0d0fe63c..fbee3d6afb 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -427,14 +427,10 @@ export type GetPaginatedEntitiesResponse = { entities: Entity[]; /* The number of entities among all the requests */ totalItems: number; - /* A method returning a promise containing the next batch of entities. */ - next?( - request?: Omit, - ): Promise; - /* A method returning a promise containing the previous batch of entities. */ - prev?( - request?: Omit, - ): Promise; + /* The cursor for the next batch of entities */ + nextCursor?: string; + /* The cursor for the previous batch of entities */ + prevCursor?: string; }; /** From c5494cf09b835940fc1863b30a3a5cd7e4975435 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 13:42:05 +0100 Subject: [PATCH 31/50] catalog: move paginated endpoint Signed-off-by: Vincenzo Scamporlino --- .changeset/silly-suits-run.md | 2 +- packages/catalog-client/src/CatalogClient.test.ts | 10 +++++----- packages/catalog-client/src/CatalogClient.ts | 2 +- .../catalog-backend/src/service/createRouter.test.ts | 10 ++++------ plugins/catalog-backend/src/service/createRouter.ts | 2 +- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.changeset/silly-suits-run.md b/.changeset/silly-suits-run.md index ed0961a584..812e6ee933 100644 --- a/.changeset/silly-suits-run.md +++ b/.changeset/silly-suits-run.md @@ -3,6 +3,6 @@ '@backstage/plugin-catalog-backend': minor --- -Add /v2beta1/entities endpoint returning paginated entities. +Add /v2/entities endpoint returning paginated entities. The endpoint supports cursor base pagination and server side sorting of the entities by name or uid. diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index ed27bb0016..9dc60da193 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -300,7 +300,7 @@ describe('CatalogClient', () => { beforeEach(() => { server.use( - rest.get(`${mockBaseUrl}/v2beta1/entities`, (_, res, ctx) => { + rest.get(`${mockBaseUrl}/v2/entities`, (_, res, ctx) => { return res(ctx.json(defaultServiceResponse)); }), ); @@ -321,7 +321,7 @@ describe('CatalogClient', () => { res(ctx.json({ entities: [], totalItems: 0 })), ); - server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); const response = await client.getPaginatedEntities?.( { @@ -356,7 +356,7 @@ describe('CatalogClient', () => { res(ctx.json({ entities: [], totalItems: 0 })), ); - server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); await client.getPaginatedEntities?.({ fields: ['a', 'b'], @@ -379,7 +379,7 @@ describe('CatalogClient', () => { res(ctx.json({ entities: [], totalItems: 0 })), ); - server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); await client.getPaginatedEntities?.({ fields: ['a', 'b'], @@ -422,7 +422,7 @@ describe('CatalogClient', () => { ), ); - server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); const response = await client.getPaginatedEntities?.({ limit: 2, diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 52ed2fd8fc..8af2a40ccd 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -251,7 +251,7 @@ export class CatalogClient implements CatalogApi { totalItems: number; nextCursor?: string; prevCursor?: string; - }>('GET', `/v2beta1/entities${query}`, options); + }>('GET', `/v2/entities${query}`, options); } /** diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index c3b96fbfd8..0fe32c02b1 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -136,7 +136,7 @@ describe('createRouter readonly disabled', () => { }); }); - describe('GET /v2beta1/entities', () => { + describe('GET /v2/entities', () => { it('happy path: lists entities', async () => { const entities: Entity[] = [ { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, @@ -148,7 +148,7 @@ describe('createRouter readonly disabled', () => { nextCursor: 'something', }); - const response = await request(app).get('/v2beta1/entities'); + const response = await request(app).get('/v2/entities'); expect(response.status).toEqual(200); expect(response.body).toEqual({ entities, @@ -163,7 +163,7 @@ describe('createRouter readonly disabled', () => { totalItems: 0, }); const response = await request(app).get( - '/v2beta1/entities?filter=a=1,a=2,b=3&filter=c=4&sortField=metadata.name,asc&sortField=metadata.uid,desc', + '/v2/entities?filter=a=1,a=2,b=3&filter=c=4&sortField=metadata.name,asc&sortField=metadata.uid,desc', ); expect(response.status).toEqual(200); @@ -198,9 +198,7 @@ describe('createRouter readonly disabled', () => { nextCursor: 'next', }); - const response = await request(app).get( - '/v2beta1/entities?cursor=something', - ); + const response = await request(app).get('/v2/entities?cursor=something'); expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ cursor: 'something', diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index a7f5373a03..c4c2c87bad 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -131,7 +131,7 @@ export async function createRouter( // TODO(freben): encode the pageInfo in the response res.json(entities); }) - .get('/v2beta1/entities', async (req, res) => { + .get('/v2/entities', async (req, res) => { const response = await entitiesCatalog.paginatedEntities({ ...parsePaginatedEntitiesParams(req.query), authorizationToken: getBearerToken(req.header('authorization')), From 47bcc6cf33bf78b43cea99c9914bd00f2183711a Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 13:54:04 +0100 Subject: [PATCH 32/50] catalog-client: improve docs Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/src/types/api.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index fbee3d6afb..168dbbcaff 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -479,8 +479,7 @@ export interface CatalogApi { * filter: [{ kind: 'group' }], * limit: 20, * query: 'A', - * sortField: 'metadata.name', - * sortFieldOrder: 'asc' + * sortFields: { field: 'metadata.name' order: 'asc' }, * }); * ``` * @@ -488,16 +487,16 @@ export interface CatalogApi { * with 'A', ordered by name ascending. * * The response will contain a maximum of 20 entities. In case - * more than 20 entities exist, the response will contain a next function, invocable - * using: + * more than 20 entities exist, the response will contain a nextCursor + * property that can be used to fetch the next batch of entities. * * ``` - * const secondBatchResponse = await response.next({ limit: 20 }); + * const secondBatchResponse = await catalogClient + * .getPaginatedEntities({ cursor: response.nextCursor }); * ``` * - * secondBatchResponse will contain the next batch of maximum 20 entities, together - * with a prev function useful for navigating backwards and a next function - * in case more entities matching the filters of the first request are present in the catalog. + * secondBatchResponse will contain the next batch of (maximum) 20 entities, + * together with a prevCursor property, useful to fetch the previous batch. * * @public * From 9bb980d23f854d00ee65dc3fb69ecdaf3913c989 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 1 Feb 2023 20:58:51 +0100 Subject: [PATCH 33/50] catalog-client: make getPaginatedEntities non optional Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 4 ++-- packages/catalog-client/src/CatalogClient.test.ts | 14 +++++++------- packages/catalog-client/src/CatalogClient.ts | 2 +- packages/catalog-client/src/types/api.ts | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index e39fa7eb8d..1abc90d24c 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -58,7 +58,7 @@ export interface CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; - getPaginatedEntities?( + getPaginatedEntities( request?: GetPaginatedEntitiesRequest, options?: CatalogRequestOptions, ): Promise; @@ -128,7 +128,7 @@ export class CatalogClient implements CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; - getPaginatedEntities?( + getPaginatedEntities( request?: GetPaginatedEntitiesRequest, options?: CatalogRequestOptions, ): Promise; diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 9dc60da193..fde54adeb6 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -307,7 +307,7 @@ describe('CatalogClient', () => { }); it('should fetch entities from correct endpoint', async () => { - const response = await client.getPaginatedEntities?.({}, { token }); + const response = await client.getPaginatedEntities({}, { token }); expect(response?.entities).toEqual(defaultClientResponse.entities); expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); expect(response?.nextCursor).toBeDefined(); @@ -323,7 +323,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); - const response = await client.getPaginatedEntities?.( + const response = await client.getPaginatedEntities( { filter: [ { @@ -358,7 +358,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); - await client.getPaginatedEntities?.({ + await client.getPaginatedEntities({ fields: ['a', 'b'], limit: 100, query: 'query', @@ -381,7 +381,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); - await client.getPaginatedEntities?.({ + await client.getPaginatedEntities({ fields: ['a', 'b'], limit: 100, query: 'query', @@ -424,7 +424,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); - const response = await client.getPaginatedEntities?.({ + const response = await client.getPaginatedEntities({ limit: 2, }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); @@ -432,12 +432,12 @@ describe('CatalogClient', () => { expect(response?.nextCursor).toBeDefined(); expect(response?.prevCursor).toBeDefined(); - await client.getPaginatedEntities?.({ cursor: response!.nextCursor! }); + await client.getPaginatedEntities({ cursor: response!.nextCursor! }); expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( '?cursor=nextcursor', ); - await client.getPaginatedEntities?.({ cursor: response!.prevCursor! }); + await client.getPaginatedEntities({ cursor: response!.prevCursor! }); expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( '?cursor=prevcursor', ); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 8af2a40ccd..6d7188edbb 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -207,7 +207,7 @@ export class CatalogClient implements CatalogApi { /** * {@inheritdoc CatalogApi.getPaginatedEntities} */ - async getPaginatedEntities?( + async getPaginatedEntities( request: GetPaginatedEntitiesRequest = {}, options?: CatalogRequestOptions, ): Promise { diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 168dbbcaff..667bc3d8f0 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -503,7 +503,7 @@ export interface CatalogApi { * @param request - Request parameters * @param options - Additional options */ - getPaginatedEntities?( + getPaginatedEntities( request?: GetPaginatedEntitiesRequest, options?: CatalogRequestOptions, ): Promise; From b17bc0191385f235b927fb88afdcc62186301ce0 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 2 Feb 2023 11:10:44 +0100 Subject: [PATCH 34/50] catalog-backend: move endpoint to /entities/by-query Signed-off-by: Vincenzo Scamporlino --- .changeset/silly-suits-run.md | 2 +- packages/catalog-client/src/CatalogClient.test.ts | 10 +++++----- packages/catalog-client/src/CatalogClient.ts | 2 +- .../catalog-backend/src/service/createRouter.test.ts | 10 ++++++---- plugins/catalog-backend/src/service/createRouter.ts | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.changeset/silly-suits-run.md b/.changeset/silly-suits-run.md index 812e6ee933..2fc731f2a4 100644 --- a/.changeset/silly-suits-run.md +++ b/.changeset/silly-suits-run.md @@ -3,6 +3,6 @@ '@backstage/plugin-catalog-backend': minor --- -Add /v2/entities endpoint returning paginated entities. +Add /entities/by-query endpoint returning paginated entities. The endpoint supports cursor base pagination and server side sorting of the entities by name or uid. diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index fde54adeb6..cd8a0c48da 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -300,7 +300,7 @@ describe('CatalogClient', () => { beforeEach(() => { server.use( - rest.get(`${mockBaseUrl}/v2/entities`, (_, res, ctx) => { + rest.get(`${mockBaseUrl}/entities/by-query`, (_, res, ctx) => { return res(ctx.json(defaultServiceResponse)); }), ); @@ -321,7 +321,7 @@ describe('CatalogClient', () => { res(ctx.json({ entities: [], totalItems: 0 })), ); - server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); const response = await client.getPaginatedEntities( { @@ -356,7 +356,7 @@ describe('CatalogClient', () => { res(ctx.json({ entities: [], totalItems: 0 })), ); - server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); await client.getPaginatedEntities({ fields: ['a', 'b'], @@ -379,7 +379,7 @@ describe('CatalogClient', () => { res(ctx.json({ entities: [], totalItems: 0 })), ); - server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); await client.getPaginatedEntities({ fields: ['a', 'b'], @@ -422,7 +422,7 @@ describe('CatalogClient', () => { ), ); - server.use(rest.get(`${mockBaseUrl}/v2/entities`, mockedEndpoint)); + server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); const response = await client.getPaginatedEntities({ limit: 2, diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 6d7188edbb..757d6eaa15 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -251,7 +251,7 @@ export class CatalogClient implements CatalogApi { totalItems: number; nextCursor?: string; prevCursor?: string; - }>('GET', `/v2/entities${query}`, options); + }>('GET', `/entities/by-query${query}`, options); } /** diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 0fe32c02b1..8f5aeab2a0 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -136,7 +136,7 @@ describe('createRouter readonly disabled', () => { }); }); - describe('GET /v2/entities', () => { + describe('GET /entities/by-query', () => { it('happy path: lists entities', async () => { const entities: Entity[] = [ { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, @@ -148,7 +148,7 @@ describe('createRouter readonly disabled', () => { nextCursor: 'something', }); - const response = await request(app).get('/v2/entities'); + const response = await request(app).get('/entities/by-query'); expect(response.status).toEqual(200); expect(response.body).toEqual({ entities, @@ -163,7 +163,7 @@ describe('createRouter readonly disabled', () => { totalItems: 0, }); const response = await request(app).get( - '/v2/entities?filter=a=1,a=2,b=3&filter=c=4&sortField=metadata.name,asc&sortField=metadata.uid,desc', + '/entities/by-query?filter=a=1,a=2,b=3&filter=c=4&sortField=metadata.name,asc&sortField=metadata.uid,desc', ); expect(response.status).toEqual(200); @@ -198,7 +198,9 @@ describe('createRouter readonly disabled', () => { nextCursor: 'next', }); - const response = await request(app).get('/v2/entities?cursor=something'); + const response = await request(app).get( + '/entities/by-query?cursor=something', + ); expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ cursor: 'something', diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index c4c2c87bad..994b676345 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -131,7 +131,7 @@ export async function createRouter( // TODO(freben): encode the pageInfo in the response res.json(entities); }) - .get('/v2/entities', async (req, res) => { + .get('/entities/by-query', async (req, res) => { const response = await entitiesCatalog.paginatedEntities({ ...parsePaginatedEntitiesParams(req.query), authorizationToken: getBearerToken(req.header('authorization')), From cdbe702d35a62ec741600c26733399ed6b73615d Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 2 Feb 2023 11:17:49 +0100 Subject: [PATCH 35/50] catalog:client rename method to queryEntities Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 40 +++++++++---------- .../catalog-client/src/CatalogClient.test.ts | 16 ++++---- packages/catalog-client/src/CatalogClient.ts | 12 +++--- packages/catalog-client/src/types/api.ts | 30 +++++++------- packages/catalog-client/src/types/index.ts | 8 ++-- packages/catalog-client/src/utils.ts | 16 ++++---- 6 files changed, 61 insertions(+), 61 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 1abc90d24c..b8a6a17a09 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -58,10 +58,10 @@ export interface CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; - getPaginatedEntities( - request?: GetPaginatedEntitiesRequest, + queryEntities( + request?: QueryEntitiesRequest, options?: CatalogRequestOptions, - ): Promise; + ): Promise; refreshEntity( entityRef: string, options?: CatalogRequestOptions, @@ -128,10 +128,10 @@ export class CatalogClient implements CatalogApi { locationRef: string, options?: CatalogRequestOptions, ): Promise; - getPaginatedEntities( - request?: GetPaginatedEntitiesRequest, + queryEntities( + request?: QueryEntitiesRequest, options?: CatalogRequestOptions, - ): Promise; + ): Promise; refreshEntity( entityRef: string, options?: CatalogRequestOptions, @@ -242,14 +242,22 @@ export interface GetEntityFacetsResponse { } // @public -export type GetPaginatedEntitiesCursorRequest = { +type Location_2 = { + id: string; + type: string; + target: string; +}; +export { Location_2 as Location }; + +// @public +export type QueryEntitiesCursorRequest = { fields?: string[]; limit?: number; cursor: string; }; // @public -export type GetPaginatedEntitiesInitialRequest = { +export type QueryEntitiesInitialRequest = { fields?: string[]; limit?: number; filter?: EntityFilterQuery; @@ -258,26 +266,18 @@ export type GetPaginatedEntitiesInitialRequest = { }; // @public -export type GetPaginatedEntitiesRequest = - | GetPaginatedEntitiesInitialRequest - | GetPaginatedEntitiesCursorRequest; +export type QueryEntitiesRequest = + | QueryEntitiesInitialRequest + | QueryEntitiesCursorRequest; // @public -export type GetPaginatedEntitiesResponse = { +export type QueryEntitiesResponse = { entities: Entity[]; totalItems: number; nextCursor?: string; prevCursor?: string; }; -// @public -type Location_2 = { - id: string; - type: string; - target: string; -}; -export { Location_2 as Location }; - // @public export type ValidateEntityResponse = | { diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index cd8a0c48da..bc041ffd19 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -249,7 +249,7 @@ describe('CatalogClient', () => { }); }); - describe('getPaginatedEntities', () => { + describe('queryEntities', () => { const defaultServiceResponse = { entities: [ { @@ -307,7 +307,7 @@ describe('CatalogClient', () => { }); it('should fetch entities from correct endpoint', async () => { - const response = await client.getPaginatedEntities({}, { token }); + const response = await client.queryEntities({}, { token }); expect(response?.entities).toEqual(defaultClientResponse.entities); expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); expect(response?.nextCursor).toBeDefined(); @@ -323,7 +323,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); - const response = await client.getPaginatedEntities( + const response = await client.queryEntities( { filter: [ { @@ -358,7 +358,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); - await client.getPaginatedEntities({ + await client.queryEntities({ fields: ['a', 'b'], limit: 100, query: 'query', @@ -381,7 +381,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); - await client.getPaginatedEntities({ + await client.queryEntities({ fields: ['a', 'b'], limit: 100, query: 'query', @@ -424,7 +424,7 @@ describe('CatalogClient', () => { server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); - const response = await client.getPaginatedEntities({ + const response = await client.queryEntities({ limit: 2, }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); @@ -432,12 +432,12 @@ describe('CatalogClient', () => { expect(response?.nextCursor).toBeDefined(); expect(response?.prevCursor).toBeDefined(); - await client.getPaginatedEntities({ cursor: response!.nextCursor! }); + await client.queryEntities({ cursor: response!.nextCursor! }); expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( '?cursor=nextcursor', ); - await client.getPaginatedEntities({ cursor: response!.prevCursor! }); + await client.queryEntities({ cursor: response!.prevCursor! }); expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( '?cursor=prevcursor', ); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 757d6eaa15..adf23f861c 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -39,8 +39,8 @@ import { ValidateEntityResponse, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, - GetPaginatedEntitiesRequest, - GetPaginatedEntitiesResponse, + QueryEntitiesRequest, + QueryEntitiesResponse, EntityFilterQuery, } from './types/api'; import { DiscoveryApi } from './types/discovery'; @@ -205,12 +205,12 @@ export class CatalogClient implements CatalogApi { } /** - * {@inheritdoc CatalogApi.getPaginatedEntities} + * {@inheritdoc CatalogApi.queryEntities} */ - async getPaginatedEntities( - request: GetPaginatedEntitiesRequest = {}, + async queryEntities( + request: QueryEntitiesRequest = {}, options?: CatalogRequestOptions, - ): Promise { + ): Promise { const params: string[] = []; if (isPaginatedEntitiesInitialRequest(request)) { diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 667bc3d8f0..09a67c8bce 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -378,16 +378,16 @@ export type ValidateEntityResponse = | { valid: false; errors: SerializedError[] }; /** - * The request type for {@link CatalogClient.getPaginatedEntities}. + * The request type for {@link CatalogClient.queryEntities}. * * @public */ -export type GetPaginatedEntitiesRequest = - | GetPaginatedEntitiesInitialRequest - | GetPaginatedEntitiesCursorRequest; +export type QueryEntitiesRequest = + | QueryEntitiesInitialRequest + | QueryEntitiesCursorRequest; /** - * A request type for {@link CatalogClient.getPaginatedEntities}. + * A request type for {@link CatalogClient.queryEntities}. * The method takes this type in an initial pagination request, * when requesting the first batch of entities. * @@ -396,7 +396,7 @@ export type GetPaginatedEntitiesRequest = * * @public */ -export type GetPaginatedEntitiesInitialRequest = { +export type QueryEntitiesInitialRequest = { fields?: string[]; limit?: number; filter?: EntityFilterQuery; @@ -405,24 +405,24 @@ export type GetPaginatedEntitiesInitialRequest = { }; /** - * A request type for {@link CatalogClient.getPaginatedEntities}. + * A request type for {@link CatalogClient.queryEntities}. * The method takes this type in a pagination request, following * the initial request. * * @public */ -export type GetPaginatedEntitiesCursorRequest = { +export type QueryEntitiesCursorRequest = { fields?: string[]; limit?: number; cursor: string; }; /** - * The response type for {@link CatalogClient.getPaginatedEntities}. + * The response type for {@link CatalogClient.queryEntities}. * * @public */ -export type GetPaginatedEntitiesResponse = { +export type QueryEntitiesResponse = { /* The list of entities for the current request */ entities: Entity[]; /* The number of entities among all the requests */ @@ -475,7 +475,7 @@ export interface CatalogApi { * Example: * * ``` - * const response = await catalogClient.getPaginatedEntities({ + * const response = await catalogClient.queryEntities({ * filter: [{ kind: 'group' }], * limit: 20, * query: 'A', @@ -492,7 +492,7 @@ export interface CatalogApi { * * ``` * const secondBatchResponse = await catalogClient - * .getPaginatedEntities({ cursor: response.nextCursor }); + * .queryEntities({ cursor: response.nextCursor }); * ``` * * secondBatchResponse will contain the next batch of (maximum) 20 entities, @@ -503,10 +503,10 @@ export interface CatalogApi { * @param request - Request parameters * @param options - Additional options */ - getPaginatedEntities( - request?: GetPaginatedEntitiesRequest, + queryEntities( + request?: QueryEntitiesRequest, options?: CatalogRequestOptions, - ): Promise; + ): Promise; /** * Gets entity ancestor information, i.e. the hierarchy of parent entities diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 715639e1c3..9bf35886c8 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -33,9 +33,9 @@ export type { GetEntityFacetsResponse, Location, ValidateEntityResponse, - GetPaginatedEntitiesCursorRequest, - GetPaginatedEntitiesInitialRequest, - GetPaginatedEntitiesRequest, - GetPaginatedEntitiesResponse, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, + QueryEntitiesResponse, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts index 8a2eb9c06d..d6131b79d2 100644 --- a/packages/catalog-client/src/utils.ts +++ b/packages/catalog-client/src/utils.ts @@ -15,19 +15,19 @@ */ import { - GetPaginatedEntitiesCursorRequest, - GetPaginatedEntitiesInitialRequest, - GetPaginatedEntitiesRequest, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, } from './types/api'; export function isPaginatedEntitiesInitialRequest( - request: GetPaginatedEntitiesInitialRequest, -): request is GetPaginatedEntitiesInitialRequest { - return !(request as GetPaginatedEntitiesCursorRequest).cursor; + request: QueryEntitiesInitialRequest, +): request is QueryEntitiesInitialRequest { + return !(request as QueryEntitiesCursorRequest).cursor; } export function isPaginatedEntitiesCursorRequest( - request: GetPaginatedEntitiesRequest, -): request is GetPaginatedEntitiesCursorRequest { + request: QueryEntitiesRequest, +): request is QueryEntitiesCursorRequest { return !isPaginatedEntitiesInitialRequest(request); } From 5044821cb13e582b8e65b80fcea5b6395ce995ea Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 00:38:05 +0100 Subject: [PATCH 36/50] catalog-client: rename query entities request utilities Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/src/CatalogClient.ts | 4 ++-- packages/catalog-client/src/utils.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index adf23f861c..389f109989 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -45,7 +45,7 @@ import { } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; -import { isPaginatedEntitiesInitialRequest } from './utils'; +import { isQueryEntitiesInitialRequest } from './utils'; /** * A frontend and backend compatible client for communicating with the Backstage @@ -213,7 +213,7 @@ export class CatalogClient implements CatalogApi { ): Promise { const params: string[] = []; - if (isPaginatedEntitiesInitialRequest(request)) { + if (isQueryEntitiesInitialRequest(request)) { const { fields = [], filter, limit, sortFields, query } = request; params.push(...this.getParams(filter)); diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts index d6131b79d2..de21cea0d4 100644 --- a/packages/catalog-client/src/utils.ts +++ b/packages/catalog-client/src/utils.ts @@ -20,14 +20,14 @@ import { QueryEntitiesRequest, } from './types/api'; -export function isPaginatedEntitiesInitialRequest( +export function isQueryEntitiesInitialRequest( request: QueryEntitiesInitialRequest, ): request is QueryEntitiesInitialRequest { return !(request as QueryEntitiesCursorRequest).cursor; } -export function isPaginatedEntitiesCursorRequest( +export function isQueryEntitiesCursorRequest( request: QueryEntitiesRequest, ): request is QueryEntitiesCursorRequest { - return !isPaginatedEntitiesInitialRequest(request); + return !isQueryEntitiesInitialRequest(request); } From 9ccbf8a08dc5cc20af0ad05e78921d9d51f0ce16 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 00:39:03 +0100 Subject: [PATCH 37/50] catalog-backend: rename paginatedEntities to queryEntities Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 24 ++-- .../service/AuthorizedEntitiesCatalog.test.ts | 2 +- .../src/service/AuthorizedEntitiesCatalog.ts | 18 +-- .../service/DefaultEntitiesCatalog.test.ts | 130 +++++++++--------- .../src/service/DefaultEntitiesCatalog.ts | 20 +-- .../src/service/createRouter.test.ts | 20 +-- .../src/service/createRouter.ts | 6 +- .../src/service/request/index.ts | 2 +- ...st.ts => parseQueryEntitiesParams.test.ts} | 32 ++--- ...sParams.ts => parseQueryEntitiesParams.ts} | 36 +++-- plugins/catalog-backend/src/service/util.ts | 22 +-- 11 files changed, 154 insertions(+), 158 deletions(-) rename plugins/catalog-backend/src/service/request/{parsePaginatedEntitiesParams.test.ts => parseQueryEntitiesParams.test.ts} (81%) rename plugins/catalog-backend/src/service/request/{parsePaginatedEntitiesParams.ts => parseQueryEntitiesParams.ts} (74%) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 451f757e65..93720e5a68 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -179,9 +179,7 @@ export interface EntitiesCatalog { * * @param request */ - paginatedEntities( - request?: PaginatedEntitiesRequest, - ): Promise; + queryEntities(request?: QueryEntitiesRequest): Promise; /** * Removes a single entity. @@ -213,18 +211,18 @@ export interface EntitiesCatalog { } /** - * The request shape for {@link EntitiesCatalog.paginatedEntities}. + * The request shape for {@link EntitiesCatalog.queryEntities}. */ -export type PaginatedEntitiesRequest = - | PaginatedEntitiesInitialRequest - | PaginatedEntitiesCursorRequest; +export type QueryEntitiesRequest = + | QueryEntitiesInitialRequest + | QueryEntitiesCursorRequest; /** - * The initial request for {@link EntitiesCatalog.paginatedEntities}. + * The initial request for {@link EntitiesCatalog.queryEntities}. * The request take immutable properties that are going to be bound * for the current and the next pagination requests. */ -export interface PaginatedEntitiesInitialRequest { +export interface QueryEntitiesInitialRequest { authorizationToken?: string; fields?: (entity: Entity) => Entity; limit?: number; @@ -234,10 +232,10 @@ export interface PaginatedEntitiesInitialRequest { } /** - * Request for {@link EntitiesCatalog.paginatedEntities} used to + * Request for {@link EntitiesCatalog.queryEntities} used to * move forward or backward on the data. */ -export interface PaginatedEntitiesCursorRequest { +export interface QueryEntitiesCursorRequest { authorizationToken?: string; fields?: (entity: Entity) => Entity; limit?: number; @@ -245,9 +243,9 @@ export interface PaginatedEntitiesCursorRequest { } /** - * The response shape for {@link EntitiesCatalog.paginatedEntities}. + * The response shape for {@link EntitiesCatalog.queryEntities}. */ -export interface PaginatedEntitiesResponse { +export interface QueryEntitiesResponse { /** * The entities for the current pagination request */ diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 8ee10fdbcb..e559f5fd49 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -30,7 +30,7 @@ describe('AuthorizedEntitiesCatalog', () => { facets: jest.fn(), refresh: jest.fn(), listAncestors: jest.fn(), - paginatedEntities: jest.fn(), + queryEntities: jest.fn(), }; const fakePermissionApi = { authorize: jest.fn(), diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index 5ae31158f8..2a5388c3d5 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -35,11 +35,11 @@ import { EntityFacetsRequest, EntityFacetsResponse, EntityFilter, - PaginatedEntitiesRequest, - PaginatedEntitiesResponse, + QueryEntitiesRequest, + QueryEntitiesResponse, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; -import { isPaginatedEntitiesInitialRequest } from './util'; +import { isQueryEntitiesInitialRequest } from './util'; export class AuthorizedEntitiesCatalog implements EntitiesCatalog { constructor( @@ -109,9 +109,9 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { return this.entitiesCatalog.entitiesBatch(request); } - async paginatedEntities( - request?: PaginatedEntitiesRequest, - ): Promise { + async queryEntities( + request?: QueryEntitiesRequest, + ): Promise { const authorizeDecision = ( await this.permissionApi.authorizeConditional( [{ permission: catalogEntityReadPermission }], @@ -131,9 +131,9 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { authorizeDecision.conditions, ); - return this.entitiesCatalog.paginatedEntities({ + return this.entitiesCatalog.queryEntities({ ...request, - ...(isPaginatedEntitiesInitialRequest(request) && { + ...(isQueryEntitiesInitialRequest(request) && { filter: request?.filter ? { allOf: [permissionFilter, request.filter] } : permissionFilter, @@ -141,7 +141,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { }); } - return this.entitiesCatalog.paginatedEntities(request); + return this.entitiesCatalog.queryEntities(request); } async removeEntityByUid( diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index a25cbf2240..92b2ecb3c5 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -20,8 +20,8 @@ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { v4 as uuid, v4 } from 'uuid'; import { - PaginatedEntitiesCursorRequest, - PaginatedEntitiesInitialRequest, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, } from '../catalog/types'; import { applyDatabaseMigrations } from '../database/migrations'; import { @@ -723,7 +723,7 @@ describe('DefaultEntitiesCatalog', () => { ); }); - describe('paginatedEntities', () => { + describe('queryEntities', () => { it.each(databases.eachSupportedId())( 'should return paginated entities and scroll the items accordingly, %p', async databaseId => { @@ -764,89 +764,89 @@ describe('DefaultEntitiesCatalog', () => { const limit = 2; // initial request - const request1: PaginatedEntitiesInitialRequest = { + const request1: QueryEntitiesInitialRequest = { filter, limit, sortFields: [{ field: 'metadata.name', order: 'asc' }], }; - const response1 = await catalog.paginatedEntities(request1); + const response1 = await catalog.queryEntities(request1); expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); expect(response1.nextCursor).toBeDefined(); expect(response1.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(names.length); // second request (forward) - const request2: PaginatedEntitiesCursorRequest = { + const request2: QueryEntitiesCursorRequest = { cursor: response1.nextCursor!, limit, }; - const response2 = await catalog.paginatedEntities(request2); + const response2 = await catalog.queryEntities(request2); expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]); expect(response2.nextCursor).toBeDefined(); expect(response2.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(names.length); // third request (forward) - const request3: PaginatedEntitiesCursorRequest = { + const request3: QueryEntitiesCursorRequest = { cursor: response2.nextCursor!, limit, }; - const response3 = await catalog.paginatedEntities(request3); + const response3 = await catalog.queryEntities(request3); expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]); expect(response3.nextCursor).toBeDefined(); expect(response3.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(names.length); // fourth request (backwards) - const request4: PaginatedEntitiesCursorRequest = { + const request4: QueryEntitiesCursorRequest = { cursor: response3.prevCursor!, limit, }; - const response4 = await catalog.paginatedEntities(request4); + const response4 = await catalog.queryEntities(request4); expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]); expect(response4.nextCursor).toBeDefined(); expect(response4.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(names.length); // fifth request (backwards) - const request5: PaginatedEntitiesCursorRequest = { + const request5: QueryEntitiesCursorRequest = { cursor: response4.prevCursor!, limit, }; - const response5 = await catalog.paginatedEntities(request5); + const response5 = await catalog.queryEntities(request5); expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]); expect(response5.nextCursor).toBeDefined(); expect(response5.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(names.length); // sixth request (forward) - const request6: PaginatedEntitiesCursorRequest = { + const request6: QueryEntitiesCursorRequest = { cursor: response5.nextCursor!, limit, }; - const response6 = await catalog.paginatedEntities(request6); + const response6 = await catalog.queryEntities(request6); expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]); expect(response6.nextCursor).toBeDefined(); expect(response6.prevCursor).toBeDefined(); expect(response6.totalItems).toBe(names.length); // seventh request (forward) - const request7: PaginatedEntitiesCursorRequest = { + const request7: QueryEntitiesCursorRequest = { cursor: response6.nextCursor!, limit, }; - const response7 = await catalog.paginatedEntities(request7); + const response7 = await catalog.queryEntities(request7); expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]); expect(response7.nextCursor).toBeDefined(); expect(response7.prevCursor).toBeDefined(); expect(response7.totalItems).toBe(names.length); // seventh.2 request (forward with a different limit) - const request7bis: PaginatedEntitiesCursorRequest = { + const request7bis: QueryEntitiesCursorRequest = { cursor: response6.nextCursor!, limit: limit + 1, }; - const response7bis = await catalog.paginatedEntities(request7bis); + const response7bis = await catalog.queryEntities(request7bis); expect(response7bis.entities).toEqual([ entityFrom('E'), entityFrom('F'), @@ -857,11 +857,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response7bis.totalItems).toBe(names.length); // last request (forward) - const request8: PaginatedEntitiesCursorRequest = { + const request8: QueryEntitiesCursorRequest = { cursor: response7.nextCursor!, limit, }; - const response8 = await catalog.paginatedEntities(request8); + const response8 = await catalog.queryEntities(request8); expect(response8.entities).toEqual([entityFrom('G')]); expect(response8.nextCursor).toBeUndefined(); expect(response8.prevCursor).toBeDefined(); @@ -909,45 +909,45 @@ describe('DefaultEntitiesCatalog', () => { const limit = 2; // initial request - const request1: PaginatedEntitiesInitialRequest = { + const request1: QueryEntitiesInitialRequest = { filter, limit, sortFields: [{ field: 'metadata.name', order: 'desc' }], }; - const response1 = await catalog.paginatedEntities(request1); + const response1 = await catalog.queryEntities(request1); expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]); expect(response1.nextCursor).toBeDefined(); expect(response1.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(names.length); // second request (forward) - const request2: PaginatedEntitiesCursorRequest = { + const request2: QueryEntitiesCursorRequest = { cursor: response1.nextCursor!, limit, }; - const response2 = await catalog.paginatedEntities(request2); + const response2 = await catalog.queryEntities(request2); expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]); expect(response2.nextCursor).toBeDefined(); expect(response2.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(names.length); // third request (forward) - const request3: PaginatedEntitiesCursorRequest = { + const request3: QueryEntitiesCursorRequest = { cursor: response2.nextCursor!, limit, }; - const response3 = await catalog.paginatedEntities(request3); + const response3 = await catalog.queryEntities(request3); expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]); expect(response3.nextCursor).toBeDefined(); expect(response3.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(names.length); // fourth request (backwards) - const request4: PaginatedEntitiesCursorRequest = { + const request4: QueryEntitiesCursorRequest = { cursor: response3.prevCursor!, limit, }; - const response4 = await catalog.paginatedEntities(request4); + const response4 = await catalog.queryEntities(request4); expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]); expect(response4.nextCursor).toBeDefined(); @@ -955,44 +955,44 @@ describe('DefaultEntitiesCatalog', () => { expect(response4.totalItems).toBe(names.length); // fifth request (backwards) - const request5: PaginatedEntitiesCursorRequest = { + const request5: QueryEntitiesCursorRequest = { cursor: response4.prevCursor!, limit, }; - const response5 = await catalog.paginatedEntities(request5); + const response5 = await catalog.queryEntities(request5); expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]); expect(response5.nextCursor).toBeDefined(); expect(response5.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(names.length); // sixth request (forward) - const request6: PaginatedEntitiesCursorRequest = { + const request6: QueryEntitiesCursorRequest = { cursor: response5.nextCursor!, limit, }; - const response6 = await catalog.paginatedEntities(request6); + const response6 = await catalog.queryEntities(request6); expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]); expect(response6.nextCursor).toBeDefined(); expect(response6.prevCursor).toBeDefined(); expect(response6.totalItems).toBe(names.length); // seventh request (forward) - const request7: PaginatedEntitiesCursorRequest = { + const request7: QueryEntitiesCursorRequest = { cursor: response6.nextCursor!, limit, }; - const response7 = await catalog.paginatedEntities(request7); + const response7 = await catalog.queryEntities(request7); expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]); expect(response7.nextCursor).toBeDefined(); expect(response7.prevCursor).toBeDefined(); expect(response7.totalItems).toBe(names.length); // seventh.2 request (forward with a different limit) - const request7bis: PaginatedEntitiesCursorRequest = { + const request7bis: QueryEntitiesCursorRequest = { cursor: response6.nextCursor!, limit: limit + 1, }; - const response7bis = await catalog.paginatedEntities(request7bis); + const response7bis = await catalog.queryEntities(request7bis); expect(response7bis.entities).toEqual([ entityFrom('C'), entityFrom('B'), @@ -1003,11 +1003,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response7bis.totalItems).toBe(names.length); // last request (forward) - const request8: PaginatedEntitiesCursorRequest = { + const request8: QueryEntitiesCursorRequest = { cursor: response7.nextCursor!, limit, }; - const response8 = await catalog.paginatedEntities(request8); + const response8 = await catalog.queryEntities(request8); expect(response8.entities).toEqual([entityFrom('A')]); expect(response8.nextCursor).toBeUndefined(); expect(response8.prevCursor).toBeDefined(); @@ -1052,14 +1052,14 @@ describe('DefaultEntitiesCatalog', () => { key: 'spec.should_include_this', }; - const request: PaginatedEntitiesInitialRequest = { + const request: QueryEntitiesInitialRequest = { filter, limit: 100, sortFields: [{ field: 'metadata.name', order: 'asc' }], query: 'cAt ', }; - const response = await catalog.paginatedEntities(request); + const response = await catalog.queryEntities(request); expect(response.entities).toEqual([ entityFrom('atcatss'), entityFrom('cat'), @@ -1094,10 +1094,10 @@ describe('DefaultEntitiesCatalog', () => { stitcher, }); - const request: PaginatedEntitiesInitialRequest = { + const request: QueryEntitiesInitialRequest = { limit: 0, }; - const response = await catalog.paginatedEntities(request); + const response = await catalog.queryEntities(request); expect(response).toEqual({ totalItems: 20, entities: [] }); }, ); @@ -1125,11 +1125,11 @@ describe('DefaultEntitiesCatalog', () => { const limit = 2; // initial request - const request1: PaginatedEntitiesInitialRequest = { + const request1: QueryEntitiesInitialRequest = { limit, sortFields: [{ field: 'metadata.name', order: 'asc' }], }; - const response1 = await catalog.paginatedEntities(request1); + const response1 = await catalog.queryEntities(request1); expect(response1.entities).toMatchObject([ entityFrom('AA'), entityFrom('AA'), @@ -1139,11 +1139,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response1.totalItems).toBe(6); // second request (forward) - const request2: PaginatedEntitiesCursorRequest = { + const request2: QueryEntitiesCursorRequest = { cursor: response1.nextCursor!, limit, }; - const response2 = await catalog.paginatedEntities(request2); + const response2 = await catalog.queryEntities(request2); expect(response2.entities).toMatchObject([ entityFrom('AA'), entityFrom('AA'), @@ -1153,11 +1153,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response2.totalItems).toBe(6); // third request (forward) - const request3: PaginatedEntitiesCursorRequest = { + const request3: QueryEntitiesCursorRequest = { cursor: response2.nextCursor!, limit, }; - const response3 = await catalog.paginatedEntities(request3); + const response3 = await catalog.queryEntities(request3); expect(response3.entities).toEqual([ entityFrom('CC'), entityFrom('DD'), @@ -1167,11 +1167,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response3.totalItems).toBe(6); // forth request (backward) - const request4: PaginatedEntitiesCursorRequest = { + const request4: QueryEntitiesCursorRequest = { cursor: response3.prevCursor!, limit, }; - const response4 = await catalog.paginatedEntities(request4); + const response4 = await catalog.queryEntities(request4); expect(response4.entities).toMatchObject([ entityFrom('AA'), entityFrom('AA'), @@ -1181,11 +1181,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response4.totalItems).toBe(6); // fifth request (backward) - const request5: PaginatedEntitiesCursorRequest = { + const request5: QueryEntitiesCursorRequest = { cursor: response4.prevCursor!, limit, }; - const response5 = await catalog.paginatedEntities(request5); + const response5 = await catalog.queryEntities(request5); expect(response5.entities).toMatchObject([ entityFrom('AA'), entityFrom('AA'), @@ -1225,10 +1225,10 @@ describe('DefaultEntitiesCatalog', () => { const limit = 2; // initial request - const request1: PaginatedEntitiesInitialRequest = { + const request1: QueryEntitiesInitialRequest = { limit, }; - const response1 = await catalog.paginatedEntities(request1); + const response1 = await catalog.queryEntities(request1); expect(response1.entities).toMatchObject([ entityFrom('AA'), entityFrom('CC'), @@ -1238,11 +1238,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response1.totalItems).toBe(6); // second request (forward) - const request2: PaginatedEntitiesCursorRequest = { + const request2: QueryEntitiesCursorRequest = { cursor: response1.nextCursor!, limit, }; - const response2 = await catalog.paginatedEntities(request2); + const response2 = await catalog.queryEntities(request2); expect(response2.entities).toMatchObject([ entityFrom('DD'), entityFrom('AA', { namespace: 'namespace2' }), @@ -1252,11 +1252,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response2.totalItems).toBe(6); // third request (forward) - const request3: PaginatedEntitiesCursorRequest = { + const request3: QueryEntitiesCursorRequest = { cursor: response2.nextCursor!, limit, }; - const response3 = await catalog.paginatedEntities(request3); + const response3 = await catalog.queryEntities(request3); expect(response3.entities).toMatchObject([ entityFrom('AA', { namespace: 'namespace3' }), entityFrom('AA', { namespace: 'namespace4' }), @@ -1266,11 +1266,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response3.totalItems).toBe(6); // forth request (backward) - const request4: PaginatedEntitiesCursorRequest = { + const request4: QueryEntitiesCursorRequest = { cursor: response3.prevCursor!, limit, }; - const response4 = await catalog.paginatedEntities(request4); + const response4 = await catalog.queryEntities(request4); expect(response4.entities).toMatchObject([ entityFrom('DD'), entityFrom('AA', { namespace: 'namespace2' }), @@ -1280,11 +1280,11 @@ describe('DefaultEntitiesCatalog', () => { expect(response4.totalItems).toBe(6); // fifth request (backward) - const request5: PaginatedEntitiesCursorRequest = { + const request5: QueryEntitiesCursorRequest = { cursor: response4.prevCursor!, limit, }; - const response5 = await catalog.paginatedEntities(request5); + const response5 = await catalog.queryEntities(request5); expect(response5.entities).toMatchObject([ entityFrom('AA'), entityFrom('CC'), @@ -1306,11 +1306,11 @@ describe('DefaultEntitiesCatalog', () => { JSON.stringify({ filter: 'notavalidfilter' }), 'utf8', ).toString('base64'); - const request: PaginatedEntitiesCursorRequest = { + const request: QueryEntitiesCursorRequest = { cursor, }; - await expect(catalog.paginatedEntities(request)).rejects.toThrow( + await expect(catalog.queryEntities(request)).rejects.toThrow( Error('Malformed cursor, could not be parsed'), ); }); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 67fe77cc8b..fad5736f12 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -37,8 +37,8 @@ import { EntityFacetsResponse, EntityFilter, EntityPagination, - PaginatedEntitiesRequest, - PaginatedEntitiesResponse, + QueryEntitiesRequest, + QueryEntitiesResponse, EntityOrder, } from '../catalog/types'; import { @@ -53,8 +53,8 @@ import { import { Stitcher } from '../stitching/Stitcher'; import { - isPaginatedEntitiesCursorRequest, - isPaginatedEntitiesInitialRequest, + isQueryEntitiesCursorRequest, + isQueryEntitiesInitialRequest, } from './util'; const defaultSortField: EntityOrder = { @@ -332,9 +332,9 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { items }; } - async paginatedEntities( - request?: PaginatedEntitiesRequest, - ): Promise { + async queryEntities( + request?: QueryEntitiesRequest, + ): Promise { const db = this.database; const limit = request?.limit ?? 20; @@ -693,13 +693,13 @@ function decodeCursor(encodedCursor: string) { } function parseCursorFromRequest( - request?: PaginatedEntitiesRequest, + request?: QueryEntitiesRequest, ): Partial { - if (isPaginatedEntitiesInitialRequest(request)) { + if (isQueryEntitiesInitialRequest(request)) { const { filter, sortFields = [defaultSortField], query } = request; return { filter, sortFields, query }; } - if (isPaginatedEntitiesCursorRequest(request)) { + if (isQueryEntitiesCursorRequest(request)) { try { return decodeCursor(request.cursor); } catch { diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 8f5aeab2a0..d25d2b0cb4 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -52,7 +52,7 @@ describe('createRouter readonly disabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), - paginatedEntities: jest.fn(), + queryEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -142,7 +142,7 @@ describe('createRouter readonly disabled', () => { { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; - entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entitiesCatalog.queryEntities.mockResolvedValueOnce({ entities, totalItems: 100, nextCursor: 'something', @@ -158,7 +158,7 @@ describe('createRouter readonly disabled', () => { }); it('parses initial request', async () => { - entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entitiesCatalog.queryEntities.mockResolvedValueOnce({ entities: [], totalItems: 0, }); @@ -167,8 +167,8 @@ describe('createRouter readonly disabled', () => { ); expect(response.status).toEqual(200); - expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ + expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ filter: { anyOf: [ { @@ -192,7 +192,7 @@ describe('createRouter readonly disabled', () => { { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; - entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entitiesCatalog.queryEntities.mockResolvedValueOnce({ entities, totalItems: 100, nextCursor: 'next', @@ -201,8 +201,8 @@ describe('createRouter readonly disabled', () => { const response = await request(app).get( '/entities/by-query?cursor=something', ); - expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); - expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ + expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ cursor: 'something', }); expect(response.status).toEqual(200); @@ -639,7 +639,7 @@ describe('createRouter readonly enabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), - paginatedEntities: jest.fn(), + queryEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -830,7 +830,7 @@ describe('NextRouter permissioning', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), - paginatedEntities: jest.fn(), + queryEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 994b676345..7603f6955d 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -39,7 +39,7 @@ import { parseEntityFilterParams, parseEntityPaginationParams, parseEntityTransformParams, - parsePaginatedEntitiesParams, + parseQueryEntitiesParams, } from './request'; import { parseEntityFacetParams } from './request/parseEntityFacetParams'; import { parseEntityOrderParams } from './request/parseEntityOrderParams'; @@ -132,8 +132,8 @@ export async function createRouter( res.json(entities); }) .get('/entities/by-query', async (req, res) => { - const response = await entitiesCatalog.paginatedEntities({ - ...parsePaginatedEntitiesParams(req.query), + const response = await entitiesCatalog.queryEntities({ + ...parseQueryEntitiesParams(req.query), authorizationToken: getBearerToken(req.header('authorization')), }); diff --git a/plugins/catalog-backend/src/service/request/index.ts b/plugins/catalog-backend/src/service/request/index.ts index 5a55c11ca0..d9297fc122 100644 --- a/plugins/catalog-backend/src/service/request/index.ts +++ b/plugins/catalog-backend/src/service/request/index.ts @@ -19,4 +19,4 @@ export { basicEntityFilter } from './basicEntityFilter'; export { parseEntityFilterParams } from './parseEntityFilterParams'; export { parseEntityPaginationParams } from './parseEntityPaginationParams'; export { parseEntityTransformParams } from './parseEntityTransformParams'; -export { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams'; +export { parseQueryEntitiesParams } from './parseQueryEntitiesParams'; diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts similarity index 81% rename from plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts rename to plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts index 53d44c8ca5..ab702438c2 100644 --- a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -15,12 +15,12 @@ */ import { - PaginatedEntitiesCursorRequest, - PaginatedEntitiesInitialRequest, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, } from '../../catalog/types'; -import { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams'; +import { parseQueryEntitiesParams } from './parseQueryEntitiesParams'; -describe('parsePaginatedEntitiesParams', () => { +describe('parseQueryEntitiesParams', () => { describe('initial request', () => { it('should parse all the defined params', () => { const validRequest = { @@ -31,9 +31,9 @@ describe('parsePaginatedEntitiesParams', () => { sortField: ['metadata.name,desc'], query: 'query', }; - const parsedObj = parsePaginatedEntitiesParams( + const parsedObj = parseQueryEntitiesParams( validRequest, - ) as PaginatedEntitiesInitialRequest; + ) as QueryEntitiesInitialRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); expect(parsedObj.sortFields).toEqual([ @@ -45,9 +45,9 @@ describe('parsePaginatedEntitiesParams', () => { expect(parsedObj).not.toHaveProperty('cursor'); }); it('should ignore optional params', () => { - const parsedObj = parsePaginatedEntitiesParams( + const parsedObj = parseQueryEntitiesParams( {}, - ) as PaginatedEntitiesInitialRequest; + ) as QueryEntitiesInitialRequest; expect(parsedObj.limit).toBeUndefined(); expect(parsedObj.fields).toBeUndefined(); expect(parsedObj.sortFields).toBeUndefined(); @@ -66,7 +66,7 @@ describe('parsePaginatedEntitiesParams', () => { { fields: [4] }, { query: [] }, ])('should throw if some parameter is not valid %p', params => { - expect(() => parsePaginatedEntitiesParams(params)).toThrow(); + expect(() => parseQueryEntitiesParams(params)).toThrow(); }); }); @@ -78,9 +78,9 @@ describe('parsePaginatedEntitiesParams', () => { limit: '3', cursor: 'cursor', }; - const parsedObj = parsePaginatedEntitiesParams( + const parsedObj = parseQueryEntitiesParams( validRequest, - ) as PaginatedEntitiesCursorRequest; + ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); expect(parsedObj.cursor).toBe('cursor'); @@ -97,9 +97,9 @@ describe('parsePaginatedEntitiesParams', () => { sortFieldOrder: 'desc', query: 'query', }; - const parsedObj = parsePaginatedEntitiesParams( + const parsedObj = parseQueryEntitiesParams( validRequest, - ) as PaginatedEntitiesCursorRequest; + ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); expect(parsedObj.cursor).toBe('cursor'); @@ -110,9 +110,9 @@ describe('parsePaginatedEntitiesParams', () => { }); it('should ignore optional params', () => { - const parsedObj = parsePaginatedEntitiesParams( + const parsedObj = parseQueryEntitiesParams( {}, - ) as PaginatedEntitiesCursorRequest; + ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBeUndefined(); expect(parsedObj.fields).toBeUndefined(); }); @@ -124,7 +124,7 @@ describe('parsePaginatedEntitiesParams', () => { { cursor: [] }, { fields: [4] }, ])('should throw if some parameter is not valid %p', params => { - expect(() => parsePaginatedEntitiesParams(params)).toThrow(); + expect(() => parseQueryEntitiesParams(params)).toThrow(); }); }); }); diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts similarity index 74% rename from plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts rename to plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts index f5730d3297..858189b0d2 100644 --- a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -15,28 +15,27 @@ */ import { - PaginatedEntitiesCursorRequest, - PaginatedEntitiesInitialRequest, - PaginatedEntitiesRequest, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, } from '../../catalog/types'; import { parseIntegerParam, parseStringParam } from './common'; import { parseEntityFilterParams } from './parseEntityFilterParams'; import { parseEntitySortFieldParams } from './parseEntitySortFieldParams'; import { parseEntityTransformParams } from './parseEntityTransformParams'; -export function parsePaginatedEntitiesParams( +export function parseQueryEntitiesParams( params: Record, -): Omit { +): Omit { const fields = parseEntityTransformParams(params); const limit = parseIntegerParam(params.limit, 'limit'); const cursor = parseStringParam(params.cursor, 'cursor'); if (cursor) { - const response: Omit = - { - cursor, - fields, - limit, - }; + const response: Omit = { + cursor, + fields, + limit, + }; return response; } @@ -44,14 +43,13 @@ export function parsePaginatedEntitiesParams( const query = parseStringParam(params.query, 'query'); const sortFields = parseEntitySortFieldParams(params); - const response: Omit = - { - fields, - filter, - limit, - sortFields, - query, - }; + const response: Omit = { + fields, + filter, + limit, + sortFields, + query, + }; return response; } diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 958b4153f8..1ab154a86d 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -19,9 +19,9 @@ import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; import { - PaginatedEntitiesCursorRequest, - PaginatedEntitiesInitialRequest, - PaginatedEntitiesRequest, + QueryEntitiesCursorRequest, + QueryEntitiesInitialRequest, + QueryEntitiesRequest, } from '../catalog/types'; export async function requireRequestBody(req: Request): Promise { @@ -71,20 +71,20 @@ export function disallowReadonlyMode(readonly: boolean) { } } -export function isPaginatedEntitiesInitialRequest( - input: PaginatedEntitiesRequest | undefined, -): input is PaginatedEntitiesInitialRequest { +export function isQueryEntitiesInitialRequest( + input: QueryEntitiesRequest | undefined, +): input is QueryEntitiesInitialRequest { if (!input) { return false; } - return !isPaginatedEntitiesCursorRequest(input); + return !isQueryEntitiesCursorRequest(input); } -export function isPaginatedEntitiesCursorRequest( - input: PaginatedEntitiesRequest | undefined, -): input is PaginatedEntitiesCursorRequest { +export function isQueryEntitiesCursorRequest( + input: QueryEntitiesRequest | undefined, +): input is QueryEntitiesCursorRequest { if (!input) { return false; } - return !!(input as PaginatedEntitiesCursorRequest).cursor; + return !!(input as QueryEntitiesCursorRequest).cursor; } From 0df4da9e714b4d2fb924dcb9748ff751b5f66683 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 10:55:36 +0100 Subject: [PATCH 38/50] catalog-client: move cursors to pageinfo Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/src/CatalogClient.test.ts | 12 ++++++------ packages/catalog-client/src/CatalogClient.ts | 13 ++++++------- packages/catalog-client/src/types/api.ts | 10 ++++++---- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index bc041ffd19..61fe1e4e19 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -310,8 +310,8 @@ describe('CatalogClient', () => { const response = await client.queryEntities({}, { token }); expect(response?.entities).toEqual(defaultClientResponse.entities); expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); - expect(response?.nextCursor).toBeDefined(); - expect(response?.prevCursor).toBeDefined(); + expect(response?.pageInfo.nextCursor).toBeDefined(); + expect(response?.pageInfo.prevCursor).toBeDefined(); }); it('builds multiple entity search filters properly', async () => { @@ -429,15 +429,15 @@ describe('CatalogClient', () => { }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); - expect(response?.nextCursor).toBeDefined(); - expect(response?.prevCursor).toBeDefined(); + expect(response?.pageInfo.nextCursor).toBeDefined(); + expect(response?.pageInfo.prevCursor).toBeDefined(); - await client.queryEntities({ cursor: response!.nextCursor! }); + await client.queryEntities({ cursor: response!.pageInfo.nextCursor! }); expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( '?cursor=nextcursor', ); - await client.queryEntities({ cursor: response!.prevCursor! }); + await client.queryEntities({ cursor: response!.pageInfo.prevCursor! }); expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( '?cursor=prevcursor', ); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 389f109989..c8bf7e57c2 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -210,7 +210,7 @@ export class CatalogClient implements CatalogApi { async queryEntities( request: QueryEntitiesRequest = {}, options?: CatalogRequestOptions, - ): Promise { + ) { const params: string[] = []; if (isQueryEntitiesInitialRequest(request)) { @@ -246,12 +246,11 @@ export class CatalogClient implements CatalogApi { } const query = params.length ? `?${params.join('&')}` : ''; - return this.requestRequired<{ - entities: Entity[]; - totalItems: number; - nextCursor?: string; - prevCursor?: string; - }>('GET', `/entities/by-query${query}`, options); + return this.requestRequired( + 'GET', + `/entities/by-query${query}`, + options, + ); } /** diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 09a67c8bce..5ef127660b 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -427,10 +427,12 @@ export type QueryEntitiesResponse = { entities: Entity[]; /* The number of entities among all the requests */ totalItems: number; - /* The cursor for the next batch of entities */ - nextCursor?: string; - /* The cursor for the previous batch of entities */ - prevCursor?: string; + pageInfo: { + /* The cursor for the next batch of entities */ + nextCursor?: string; + /* The cursor for the previous batch of entities */ + prevCursor?: string; + }; }; /** From 53c70b2f5e394519abda92c56ef17b7ff0c7ea99 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 10:55:54 +0100 Subject: [PATCH 39/50] catalog-backend: move cursors to pageinfo Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 19 +- .../src/service/AuthorizedEntitiesCatalog.ts | 1 + .../service/DefaultEntitiesCatalog.test.ts | 164 +++++++++--------- .../src/service/DefaultEntitiesCatalog.ts | 2 +- .../src/service/createRouter.test.ts | 3 +- 5 files changed, 97 insertions(+), 92 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 93720e5a68..8450e022cc 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -250,14 +250,17 @@ export interface QueryEntitiesResponse { * The entities for the current pagination request */ entities: Entity[]; - /** - * The cursor of the next pagination request. - */ - nextCursor?: string; - /** - * The cursor of the previous pagination request. - */ - prevCursor?: string; + + pageInfo: { + /** + * The cursor of the next pagination request. + */ + nextCursor?: string; + /** + * The cursor of the previous pagination request. + */ + prevCursor?: string; + }; /** * the total number of entities matching the current filters. */ diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index 2a5388c3d5..eea9a2ca68 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -122,6 +122,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { if (authorizeDecision.result === AuthorizeResult.DENY) { return { entities: [], + pageInfo: {}, totalItems: 0, }; } diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 92b2ecb3c5..f54773bb2d 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -771,79 +771,79 @@ describe('DefaultEntitiesCatalog', () => { }; const response1 = await catalog.queryEntities(request1); expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); - expect(response1.nextCursor).toBeDefined(); - expect(response1.prevCursor).toBeUndefined(); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(names.length); // second request (forward) const request2: QueryEntitiesCursorRequest = { - cursor: response1.nextCursor!, + cursor: response1.pageInfo.nextCursor!, limit, }; const response2 = await catalog.queryEntities(request2); expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]); - expect(response2.nextCursor).toBeDefined(); - expect(response2.prevCursor).toBeDefined(); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(names.length); // third request (forward) const request3: QueryEntitiesCursorRequest = { - cursor: response2.nextCursor!, + cursor: response2.pageInfo.nextCursor!, limit, }; const response3 = await catalog.queryEntities(request3); expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]); - expect(response3.nextCursor).toBeDefined(); - expect(response3.prevCursor).toBeDefined(); + expect(response3.pageInfo.nextCursor).toBeDefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(names.length); // fourth request (backwards) const request4: QueryEntitiesCursorRequest = { - cursor: response3.prevCursor!, + cursor: response3.pageInfo.prevCursor!, limit, }; const response4 = await catalog.queryEntities(request4); expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]); - expect(response4.nextCursor).toBeDefined(); - expect(response4.prevCursor).toBeDefined(); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(names.length); // fifth request (backwards) const request5: QueryEntitiesCursorRequest = { - cursor: response4.prevCursor!, + cursor: response4.pageInfo.prevCursor!, limit, }; const response5 = await catalog.queryEntities(request5); expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]); - expect(response5.nextCursor).toBeDefined(); - expect(response5.prevCursor).toBeUndefined(); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(names.length); // sixth request (forward) const request6: QueryEntitiesCursorRequest = { - cursor: response5.nextCursor!, + cursor: response5.pageInfo.nextCursor!, limit, }; const response6 = await catalog.queryEntities(request6); expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]); - expect(response6.nextCursor).toBeDefined(); - expect(response6.prevCursor).toBeDefined(); + expect(response6.pageInfo.nextCursor).toBeDefined(); + expect(response6.pageInfo.prevCursor).toBeDefined(); expect(response6.totalItems).toBe(names.length); // seventh request (forward) const request7: QueryEntitiesCursorRequest = { - cursor: response6.nextCursor!, + cursor: response6.pageInfo.nextCursor!, limit, }; const response7 = await catalog.queryEntities(request7); expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]); - expect(response7.nextCursor).toBeDefined(); - expect(response7.prevCursor).toBeDefined(); + expect(response7.pageInfo.nextCursor).toBeDefined(); + expect(response7.pageInfo.prevCursor).toBeDefined(); expect(response7.totalItems).toBe(names.length); // seventh.2 request (forward with a different limit) const request7bis: QueryEntitiesCursorRequest = { - cursor: response6.nextCursor!, + cursor: response6.pageInfo.nextCursor!, limit: limit + 1, }; const response7bis = await catalog.queryEntities(request7bis); @@ -852,19 +852,19 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('F'), entityFrom('G'), ]); - expect(response7bis.nextCursor).toBeUndefined(); - expect(response7bis.prevCursor).toBeDefined(); + expect(response7bis.pageInfo.nextCursor).toBeUndefined(); + expect(response7bis.pageInfo.prevCursor).toBeDefined(); expect(response7bis.totalItems).toBe(names.length); // last request (forward) const request8: QueryEntitiesCursorRequest = { - cursor: response7.nextCursor!, + cursor: response7.pageInfo.nextCursor!, limit, }; const response8 = await catalog.queryEntities(request8); expect(response8.entities).toEqual([entityFrom('G')]); - expect(response8.nextCursor).toBeUndefined(); - expect(response8.prevCursor).toBeDefined(); + expect(response8.pageInfo.nextCursor).toBeUndefined(); + expect(response8.pageInfo.prevCursor).toBeDefined(); expect(response8.totalItems).toBe(names.length); }, ); @@ -916,80 +916,80 @@ describe('DefaultEntitiesCatalog', () => { }; const response1 = await catalog.queryEntities(request1); expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]); - expect(response1.nextCursor).toBeDefined(); - expect(response1.prevCursor).toBeUndefined(); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(names.length); // second request (forward) const request2: QueryEntitiesCursorRequest = { - cursor: response1.nextCursor!, + cursor: response1.pageInfo.nextCursor!, limit, }; const response2 = await catalog.queryEntities(request2); expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]); - expect(response2.nextCursor).toBeDefined(); - expect(response2.prevCursor).toBeDefined(); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(names.length); // third request (forward) const request3: QueryEntitiesCursorRequest = { - cursor: response2.nextCursor!, + cursor: response2.pageInfo.nextCursor!, limit, }; const response3 = await catalog.queryEntities(request3); expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]); - expect(response3.nextCursor).toBeDefined(); - expect(response3.prevCursor).toBeDefined(); + expect(response3.pageInfo.nextCursor).toBeDefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(names.length); // fourth request (backwards) const request4: QueryEntitiesCursorRequest = { - cursor: response3.prevCursor!, + cursor: response3.pageInfo.prevCursor!, limit, }; const response4 = await catalog.queryEntities(request4); expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]); - expect(response4.nextCursor).toBeDefined(); - expect(response4.prevCursor).toBeDefined(); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(names.length); // fifth request (backwards) const request5: QueryEntitiesCursorRequest = { - cursor: response4.prevCursor!, + cursor: response4.pageInfo.prevCursor!, limit, }; const response5 = await catalog.queryEntities(request5); expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]); - expect(response5.nextCursor).toBeDefined(); - expect(response5.prevCursor).toBeUndefined(); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(names.length); // sixth request (forward) const request6: QueryEntitiesCursorRequest = { - cursor: response5.nextCursor!, + cursor: response5.pageInfo.nextCursor!, limit, }; const response6 = await catalog.queryEntities(request6); expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]); - expect(response6.nextCursor).toBeDefined(); - expect(response6.prevCursor).toBeDefined(); + expect(response6.pageInfo.nextCursor).toBeDefined(); + expect(response6.pageInfo.prevCursor).toBeDefined(); expect(response6.totalItems).toBe(names.length); // seventh request (forward) const request7: QueryEntitiesCursorRequest = { - cursor: response6.nextCursor!, + cursor: response6.pageInfo.nextCursor!, limit, }; const response7 = await catalog.queryEntities(request7); expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]); - expect(response7.nextCursor).toBeDefined(); - expect(response7.prevCursor).toBeDefined(); + expect(response7.pageInfo.nextCursor).toBeDefined(); + expect(response7.pageInfo.prevCursor).toBeDefined(); expect(response7.totalItems).toBe(names.length); // seventh.2 request (forward with a different limit) const request7bis: QueryEntitiesCursorRequest = { - cursor: response6.nextCursor!, + cursor: response6.pageInfo.nextCursor!, limit: limit + 1, }; const response7bis = await catalog.queryEntities(request7bis); @@ -998,19 +998,19 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('B'), entityFrom('A'), ]); - expect(response7bis.nextCursor).toBeUndefined(); - expect(response7bis.prevCursor).toBeDefined(); + expect(response7bis.pageInfo.nextCursor).toBeUndefined(); + expect(response7bis.pageInfo.prevCursor).toBeDefined(); expect(response7bis.totalItems).toBe(names.length); // last request (forward) const request8: QueryEntitiesCursorRequest = { - cursor: response7.nextCursor!, + cursor: response7.pageInfo.nextCursor!, limit, }; const response8 = await catalog.queryEntities(request8); expect(response8.entities).toEqual([entityFrom('A')]); - expect(response8.nextCursor).toBeUndefined(); - expect(response8.prevCursor).toBeDefined(); + expect(response8.pageInfo.nextCursor).toBeUndefined(); + expect(response8.pageInfo.prevCursor).toBeDefined(); expect(response8.totalItems).toBe(names.length); }, ); @@ -1065,8 +1065,8 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('cat'), entityFrom('dogcat'), ]); - expect(response.nextCursor).toBeUndefined(); - expect(response.prevCursor).toBeUndefined(); + expect(response.pageInfo.nextCursor).toBeUndefined(); + expect(response.pageInfo.prevCursor).toBeUndefined(); expect(response.totalItems).toBe(3); }, ); @@ -1134,13 +1134,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA'), entityFrom('AA'), ]); - expect(response1.nextCursor).toBeDefined(); - expect(response1.prevCursor).toBeUndefined(); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(6); // second request (forward) const request2: QueryEntitiesCursorRequest = { - cursor: response1.nextCursor!, + cursor: response1.pageInfo.nextCursor!, limit, }; const response2 = await catalog.queryEntities(request2); @@ -1148,13 +1148,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA'), entityFrom('AA'), ]); - expect(response2.nextCursor).toBeDefined(); - expect(response2.prevCursor).toBeDefined(); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(6); // third request (forward) const request3: QueryEntitiesCursorRequest = { - cursor: response2.nextCursor!, + cursor: response2.pageInfo.nextCursor!, limit, }; const response3 = await catalog.queryEntities(request3); @@ -1162,13 +1162,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('CC'), entityFrom('DD'), ]); - expect(response3.nextCursor).toBeUndefined(); - expect(response3.prevCursor).toBeDefined(); + expect(response3.pageInfo.nextCursor).toBeUndefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(6); // forth request (backward) const request4: QueryEntitiesCursorRequest = { - cursor: response3.prevCursor!, + cursor: response3.pageInfo.prevCursor!, limit, }; const response4 = await catalog.queryEntities(request4); @@ -1176,13 +1176,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA'), entityFrom('AA'), ]); - expect(response4.nextCursor).toBeDefined(); - expect(response4.prevCursor).toBeDefined(); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(6); // fifth request (backward) const request5: QueryEntitiesCursorRequest = { - cursor: response4.prevCursor!, + cursor: response4.pageInfo.prevCursor!, limit, }; const response5 = await catalog.queryEntities(request5); @@ -1190,8 +1190,8 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA'), entityFrom('AA'), ]); - expect(response5.nextCursor).toBeDefined(); - expect(response5.prevCursor).toBeUndefined(); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(6); }, ); @@ -1233,13 +1233,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA'), entityFrom('CC'), ]); - expect(response1.nextCursor).toBeDefined(); - expect(response1.prevCursor).toBeUndefined(); + expect(response1.pageInfo.nextCursor).toBeDefined(); + expect(response1.pageInfo.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(6); // second request (forward) const request2: QueryEntitiesCursorRequest = { - cursor: response1.nextCursor!, + cursor: response1.pageInfo.nextCursor!, limit, }; const response2 = await catalog.queryEntities(request2); @@ -1247,13 +1247,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('DD'), entityFrom('AA', { namespace: 'namespace2' }), ]); - expect(response2.nextCursor).toBeDefined(); - expect(response2.prevCursor).toBeDefined(); + expect(response2.pageInfo.nextCursor).toBeDefined(); + expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(6); // third request (forward) const request3: QueryEntitiesCursorRequest = { - cursor: response2.nextCursor!, + cursor: response2.pageInfo.nextCursor!, limit, }; const response3 = await catalog.queryEntities(request3); @@ -1261,13 +1261,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA', { namespace: 'namespace3' }), entityFrom('AA', { namespace: 'namespace4' }), ]); - expect(response3.nextCursor).toBeUndefined(); - expect(response3.prevCursor).toBeDefined(); + expect(response3.pageInfo.nextCursor).toBeUndefined(); + expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(6); // forth request (backward) const request4: QueryEntitiesCursorRequest = { - cursor: response3.prevCursor!, + cursor: response3.pageInfo.prevCursor!, limit, }; const response4 = await catalog.queryEntities(request4); @@ -1275,13 +1275,13 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('DD'), entityFrom('AA', { namespace: 'namespace2' }), ]); - expect(response4.nextCursor).toBeDefined(); - expect(response4.prevCursor).toBeDefined(); + expect(response4.pageInfo.nextCursor).toBeDefined(); + expect(response4.pageInfo.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(6); // fifth request (backward) const request5: QueryEntitiesCursorRequest = { - cursor: response4.prevCursor!, + cursor: response4.pageInfo.prevCursor!, limit, }; const response5 = await catalog.queryEntities(request5); @@ -1289,8 +1289,8 @@ describe('DefaultEntitiesCatalog', () => { entityFrom('AA'), entityFrom('CC'), ]); - expect(response5.nextCursor).toBeDefined(); - expect(response5.prevCursor).toBeUndefined(); + expect(response5.pageInfo.nextCursor).toBeDefined(); + expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(6); }, ); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index fad5736f12..0f68a7a3c1 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -475,7 +475,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .map(e => JSON.parse(e.final_entity!)) .map(e => (request?.fields ? request.fields(e) : e)); - return { entities, prevCursor, nextCursor, totalItems }; + return { entities, pageInfo: { prevCursor, nextCursor }, totalItems }; } async removeEntityByUid(uid: string): Promise { diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index d25d2b0cb4..397d16f714 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -145,7 +145,7 @@ describe('createRouter readonly disabled', () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ entities, totalItems: 100, - nextCursor: 'something', + pageInfo: { nextCursor: 'something' }, }); const response = await request(app).get('/entities/by-query'); @@ -160,6 +160,7 @@ describe('createRouter readonly disabled', () => { it('parses initial request', async () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ entities: [], + pageInfo: {}, totalItems: 0, }); const response = await request(app).get( From 6567b7cd585bda87fd676d8bbd0a56c7599aa206 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 12:55:39 +0100 Subject: [PATCH 40/50] catalog-backend: entities to items Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 2 +- .../src/service/AuthorizedEntitiesCatalog.ts | 2 +- .../service/DefaultEntitiesCatalog.test.ts | 61 +++++++++---------- .../src/service/DefaultEntitiesCatalog.ts | 4 +- .../src/service/createRouter.test.ts | 16 ++--- 5 files changed, 41 insertions(+), 44 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 8450e022cc..5591573ee2 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -249,7 +249,7 @@ export interface QueryEntitiesResponse { /** * The entities for the current pagination request */ - entities: Entity[]; + items: Entity[]; pageInfo: { /** diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index eea9a2ca68..f9ec058879 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -121,7 +121,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { if (authorizeDecision.result === AuthorizeResult.DENY) { return { - entities: [], + items: [], pageInfo: {}, totalItems: 0, }; diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index f54773bb2d..ab20c0942a 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -770,7 +770,7 @@ describe('DefaultEntitiesCatalog', () => { sortFields: [{ field: 'metadata.name', order: 'asc' }], }; const response1 = await catalog.queryEntities(request1); - expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response1.items).toEqual([entityFrom('A'), entityFrom('B')]); expect(response1.pageInfo.nextCursor).toBeDefined(); expect(response1.pageInfo.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(names.length); @@ -781,7 +781,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response2 = await catalog.queryEntities(request2); - expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response2.items).toEqual([entityFrom('C'), entityFrom('D')]); expect(response2.pageInfo.nextCursor).toBeDefined(); expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(names.length); @@ -792,7 +792,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response3 = await catalog.queryEntities(request3); - expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response3.items).toEqual([entityFrom('E'), entityFrom('F')]); expect(response3.pageInfo.nextCursor).toBeDefined(); expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(names.length); @@ -803,7 +803,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response4 = await catalog.queryEntities(request4); - expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response4.items).toEqual([entityFrom('C'), entityFrom('D')]); expect(response4.pageInfo.nextCursor).toBeDefined(); expect(response4.pageInfo.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(names.length); @@ -814,7 +814,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response5 = await catalog.queryEntities(request5); - expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response5.items).toEqual([entityFrom('A'), entityFrom('B')]); expect(response5.pageInfo.nextCursor).toBeDefined(); expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(names.length); @@ -825,7 +825,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response6 = await catalog.queryEntities(request6); - expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response6.items).toEqual([entityFrom('C'), entityFrom('D')]); expect(response6.pageInfo.nextCursor).toBeDefined(); expect(response6.pageInfo.prevCursor).toBeDefined(); expect(response6.totalItems).toBe(names.length); @@ -836,7 +836,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response7 = await catalog.queryEntities(request7); - expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response7.items).toEqual([entityFrom('E'), entityFrom('F')]); expect(response7.pageInfo.nextCursor).toBeDefined(); expect(response7.pageInfo.prevCursor).toBeDefined(); expect(response7.totalItems).toBe(names.length); @@ -847,7 +847,7 @@ describe('DefaultEntitiesCatalog', () => { limit: limit + 1, }; const response7bis = await catalog.queryEntities(request7bis); - expect(response7bis.entities).toEqual([ + expect(response7bis.items).toEqual([ entityFrom('E'), entityFrom('F'), entityFrom('G'), @@ -862,7 +862,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response8 = await catalog.queryEntities(request8); - expect(response8.entities).toEqual([entityFrom('G')]); + expect(response8.items).toEqual([entityFrom('G')]); expect(response8.pageInfo.nextCursor).toBeUndefined(); expect(response8.pageInfo.prevCursor).toBeDefined(); expect(response8.totalItems).toBe(names.length); @@ -915,7 +915,7 @@ describe('DefaultEntitiesCatalog', () => { sortFields: [{ field: 'metadata.name', order: 'desc' }], }; const response1 = await catalog.queryEntities(request1); - expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response1.items).toEqual([entityFrom('G'), entityFrom('F')]); expect(response1.pageInfo.nextCursor).toBeDefined(); expect(response1.pageInfo.prevCursor).toBeUndefined(); expect(response1.totalItems).toBe(names.length); @@ -926,7 +926,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response2 = await catalog.queryEntities(request2); - expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response2.items).toEqual([entityFrom('E'), entityFrom('D')]); expect(response2.pageInfo.nextCursor).toBeDefined(); expect(response2.pageInfo.prevCursor).toBeDefined(); expect(response2.totalItems).toBe(names.length); @@ -937,7 +937,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response3 = await catalog.queryEntities(request3); - expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response3.items).toEqual([entityFrom('C'), entityFrom('B')]); expect(response3.pageInfo.nextCursor).toBeDefined(); expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(names.length); @@ -949,7 +949,7 @@ describe('DefaultEntitiesCatalog', () => { }; const response4 = await catalog.queryEntities(request4); - expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response4.items).toEqual([entityFrom('E'), entityFrom('D')]); expect(response4.pageInfo.nextCursor).toBeDefined(); expect(response4.pageInfo.prevCursor).toBeDefined(); expect(response4.totalItems).toBe(names.length); @@ -960,7 +960,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response5 = await catalog.queryEntities(request5); - expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response5.items).toEqual([entityFrom('G'), entityFrom('F')]); expect(response5.pageInfo.nextCursor).toBeDefined(); expect(response5.pageInfo.prevCursor).toBeUndefined(); expect(response5.totalItems).toBe(names.length); @@ -971,7 +971,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response6 = await catalog.queryEntities(request6); - expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response6.items).toEqual([entityFrom('E'), entityFrom('D')]); expect(response6.pageInfo.nextCursor).toBeDefined(); expect(response6.pageInfo.prevCursor).toBeDefined(); expect(response6.totalItems).toBe(names.length); @@ -982,7 +982,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response7 = await catalog.queryEntities(request7); - expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response7.items).toEqual([entityFrom('C'), entityFrom('B')]); expect(response7.pageInfo.nextCursor).toBeDefined(); expect(response7.pageInfo.prevCursor).toBeDefined(); expect(response7.totalItems).toBe(names.length); @@ -993,7 +993,7 @@ describe('DefaultEntitiesCatalog', () => { limit: limit + 1, }; const response7bis = await catalog.queryEntities(request7bis); - expect(response7bis.entities).toEqual([ + expect(response7bis.items).toEqual([ entityFrom('C'), entityFrom('B'), entityFrom('A'), @@ -1008,7 +1008,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response8 = await catalog.queryEntities(request8); - expect(response8.entities).toEqual([entityFrom('A')]); + expect(response8.items).toEqual([entityFrom('A')]); expect(response8.pageInfo.nextCursor).toBeUndefined(); expect(response8.pageInfo.prevCursor).toBeDefined(); expect(response8.totalItems).toBe(names.length); @@ -1060,7 +1060,7 @@ describe('DefaultEntitiesCatalog', () => { query: 'cAt ', }; const response = await catalog.queryEntities(request); - expect(response.entities).toEqual([ + expect(response.items).toEqual([ entityFrom('atcatss'), entityFrom('cat'), entityFrom('dogcat'), @@ -1130,7 +1130,7 @@ describe('DefaultEntitiesCatalog', () => { sortFields: [{ field: 'metadata.name', order: 'asc' }], }; const response1 = await catalog.queryEntities(request1); - expect(response1.entities).toMatchObject([ + expect(response1.items).toMatchObject([ entityFrom('AA'), entityFrom('AA'), ]); @@ -1144,7 +1144,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response2 = await catalog.queryEntities(request2); - expect(response2.entities).toMatchObject([ + expect(response2.items).toMatchObject([ entityFrom('AA'), entityFrom('AA'), ]); @@ -1158,10 +1158,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response3 = await catalog.queryEntities(request3); - expect(response3.entities).toEqual([ - entityFrom('CC'), - entityFrom('DD'), - ]); + expect(response3.items).toEqual([entityFrom('CC'), entityFrom('DD')]); expect(response3.pageInfo.nextCursor).toBeUndefined(); expect(response3.pageInfo.prevCursor).toBeDefined(); expect(response3.totalItems).toBe(6); @@ -1172,7 +1169,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response4 = await catalog.queryEntities(request4); - expect(response4.entities).toMatchObject([ + expect(response4.items).toMatchObject([ entityFrom('AA'), entityFrom('AA'), ]); @@ -1186,7 +1183,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response5 = await catalog.queryEntities(request5); - expect(response5.entities).toMatchObject([ + expect(response5.items).toMatchObject([ entityFrom('AA'), entityFrom('AA'), ]); @@ -1229,7 +1226,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response1 = await catalog.queryEntities(request1); - expect(response1.entities).toMatchObject([ + expect(response1.items).toMatchObject([ entityFrom('AA'), entityFrom('CC'), ]); @@ -1243,7 +1240,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response2 = await catalog.queryEntities(request2); - expect(response2.entities).toMatchObject([ + expect(response2.items).toMatchObject([ entityFrom('DD'), entityFrom('AA', { namespace: 'namespace2' }), ]); @@ -1257,7 +1254,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response3 = await catalog.queryEntities(request3); - expect(response3.entities).toMatchObject([ + expect(response3.items).toMatchObject([ entityFrom('AA', { namespace: 'namespace3' }), entityFrom('AA', { namespace: 'namespace4' }), ]); @@ -1271,7 +1268,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response4 = await catalog.queryEntities(request4); - expect(response4.entities).toMatchObject([ + expect(response4.items).toMatchObject([ entityFrom('DD'), entityFrom('AA', { namespace: 'namespace2' }), ]); @@ -1285,7 +1282,7 @@ describe('DefaultEntitiesCatalog', () => { limit, }; const response5 = await catalog.queryEntities(request5); - expect(response5.entities).toMatchObject([ + expect(response5.items).toMatchObject([ entityFrom('AA'), entityFrom('CC'), ]); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 0f68a7a3c1..ba242b7ab5 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -471,11 +471,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }) : undefined; - const entities = rows + const items = rows .map(e => JSON.parse(e.final_entity!)) .map(e => (request?.fields ? request.fields(e) : e)); - return { entities, pageInfo: { prevCursor, nextCursor }, totalItems }; + return { items, pageInfo: { prevCursor, nextCursor }, totalItems }; } async removeEntityByUid(uid: string): Promise { diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 397d16f714..a35115ac58 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -138,12 +138,12 @@ describe('createRouter readonly disabled', () => { describe('GET /entities/by-query', () => { it('happy path: lists entities', async () => { - const entities: Entity[] = [ + const items: Entity[] = [ { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - entities, + items, totalItems: 100, pageInfo: { nextCursor: 'something' }, }); @@ -151,7 +151,7 @@ describe('createRouter readonly disabled', () => { const response = await request(app).get('/entities/by-query'); expect(response.status).toEqual(200); expect(response.body).toEqual({ - entities, + items, totalItems: 100, nextCursor: 'something', }); @@ -159,7 +159,7 @@ describe('createRouter readonly disabled', () => { it('parses initial request', async () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ - entities: [], + items: [], pageInfo: {}, totalItems: 0, }); @@ -189,14 +189,14 @@ describe('createRouter readonly disabled', () => { }); it('parses cursor request', async () => { - const entities: Entity[] = [ + const items: Entity[] = [ { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - entities, + items, totalItems: 100, - nextCursor: 'next', + pageInfo: { nextCursor: 'next' }, }); const response = await request(app).get( @@ -208,7 +208,7 @@ describe('createRouter readonly disabled', () => { }); expect(response.status).toEqual(200); expect(response.body).toEqual({ - entities, + items, totalItems: 100, nextCursor: 'next', }); From 58195a4813822ecf9320f1576b49ac2783c9fbf7 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 12:56:05 +0100 Subject: [PATCH 41/50] catalog-client: entities to items Signed-off-by: Vincenzo Scamporlino --- .../catalog-client/src/CatalogClient.test.ts | 64 +++++++------------ packages/catalog-client/src/types/api.ts | 2 +- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 61fe1e4e19..4992b0d373 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -18,7 +18,11 @@ import { Entity } from '@backstage/catalog-model'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { CatalogClient } from './CatalogClient'; -import { CATALOG_FILTER_EXISTS, GetEntitiesResponse } from './types/api'; +import { + CATALOG_FILTER_EXISTS, + GetEntitiesResponse, + QueryEntitiesResponse, +} from './types/api'; import { DiscoveryApi } from './types/discovery'; const server = setupServer(); @@ -250,8 +254,8 @@ describe('CatalogClient', () => { }); describe('queryEntities', () => { - const defaultServiceResponse = { - entities: [ + const defaultResponse: QueryEntitiesResponse = { + items: [ { apiVersion: '1', kind: 'Component', @@ -269,47 +273,25 @@ describe('CatalogClient', () => { }, }, ], + pageInfo: { + nextCursor: 'next', + prevCursor: 'prev', + }, totalItems: 10, - nextCursor: 'next', - prevCursor: 'prev', - }; - - const defaultClientResponse = { - entities: [ - { - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'Test2', - namespace: 'test1', - }, - }, - { - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'Test1', - namespace: 'test1', - }, - }, - ], - totalItems: 10, - next: jest.fn(), - prev: jest.fn(), }; beforeEach(() => { server.use( rest.get(`${mockBaseUrl}/entities/by-query`, (_, res, ctx) => { - return res(ctx.json(defaultServiceResponse)); + return res(ctx.json(defaultResponse)); }), ); }); it('should fetch entities from correct endpoint', async () => { const response = await client.queryEntities({}, { token }); - expect(response?.entities).toEqual(defaultClientResponse.entities); - expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); + expect(response?.items).toEqual(defaultResponse.items); + expect(response?.totalItems).toEqual(defaultResponse.totalItems); expect(response?.pageInfo.nextCursor).toBeDefined(); expect(response?.pageInfo.prevCursor).toBeDefined(); }); @@ -318,7 +300,7 @@ describe('CatalogClient', () => { const mockedEndpoint = jest .fn() .mockImplementation((_req, res, ctx) => - res(ctx.json({ entities: [], totalItems: 0 })), + res(ctx.json({ items: [], totalItems: 0 })), ); server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); @@ -342,7 +324,7 @@ describe('CatalogClient', () => { { token }, ); - expect(response).toEqual({ entities: [], totalItems: 0 }); + expect(response).toEqual({ items: [], totalItems: 0 }); expect(mockedEndpoint).toHaveBeenCalledTimes(1); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c', @@ -353,7 +335,7 @@ describe('CatalogClient', () => { const mockedEndpoint = jest .fn() .mockImplementation((_req, res, ctx) => - res(ctx.json({ entities: [], totalItems: 0 })), + res(ctx.json({ items: [], totalItems: 0 })), ); server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); @@ -376,7 +358,7 @@ describe('CatalogClient', () => { const mockedEndpoint = jest .fn() .mockImplementation((_req, res, ctx) => - res(ctx.json({ entities: [], totalItems: 0 })), + res(ctx.json({ items: [], totalItems: 0 })), ); server.use(rest.get(`${mockBaseUrl}/entities/by-query`, mockedEndpoint)); @@ -397,7 +379,7 @@ describe('CatalogClient', () => { const mockedEndpoint = jest.fn().mockImplementation((_req, res, ctx) => res( ctx.json({ - entities: [ + items: [ { apiVersion: 'v1', kind: 'CustomKind', @@ -415,10 +397,12 @@ describe('CatalogClient', () => { }, }, ], - nextCursor: 'nextcursor', - prevCursor: 'prevcursor', + pageInfo: { + nextCursor: 'nextcursor', + prevCursor: 'prevcursor', + }, totalItems: 100, - }), + } as QueryEntitiesResponse), ), ); diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 5ef127660b..95998b3ed5 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -424,7 +424,7 @@ export type QueryEntitiesCursorRequest = { */ export type QueryEntitiesResponse = { /* The list of entities for the current request */ - entities: Entity[]; + items: Entity[]; /* The number of entities among all the requests */ totalItems: number; pageInfo: { From 7b370a5dadf45524510a2b809a937b64ede3daa6 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 13:18:27 +0100 Subject: [PATCH 42/50] catalog-client: sortFields to orderFields Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/src/CatalogClient.test.ts | 4 ++-- packages/catalog-client/src/CatalogClient.ts | 6 +++--- packages/catalog-client/src/types/api.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 4992b0d373..f11627d092 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -344,7 +344,7 @@ describe('CatalogClient', () => { fields: ['a', 'b'], limit: 100, query: 'query', - sortFields: [ + orderFields: [ { field: 'metadata.name', order: 'asc' }, { field: 'metadata.uid', order: 'desc' }, ], @@ -367,7 +367,7 @@ describe('CatalogClient', () => { fields: ['a', 'b'], limit: 100, query: 'query', - sortFields: [{ field: 'metadata.name', order: 'asc' }], + orderFields: [{ field: 'metadata.name', order: 'asc' }], cursor: 'cursor', }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index c8bf7e57c2..a380baf6e5 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -214,14 +214,14 @@ export class CatalogClient implements CatalogApi { const params: string[] = []; if (isQueryEntitiesInitialRequest(request)) { - const { fields = [], filter, limit, sortFields, query } = request; + const { fields = [], filter, limit, orderFields, query } = request; params.push(...this.getParams(filter)); if (limit !== undefined) { params.push(`limit=${limit}`); } - if (sortFields !== undefined) { - (Array.isArray(sortFields) ? sortFields : [sortFields]).forEach( + if (orderFields !== undefined) { + (Array.isArray(orderFields) ? orderFields : [orderFields]).forEach( ({ field, order }) => params.push(`sortField=${field},${order}`), ); } diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 95998b3ed5..4a727b7bf0 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -400,7 +400,7 @@ export type QueryEntitiesInitialRequest = { fields?: string[]; limit?: number; filter?: EntityFilterQuery; - sortFields?: EntityOrderQuery; + orderFields?: EntityOrderQuery; query?: string; }; From cf166f403021a7b5f5406eb37f7ebf760347643b Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 13:19:53 +0100 Subject: [PATCH 43/50] catalog-backend: sortFields to orderFields Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 6 +-- .../service/DefaultEntitiesCatalog.test.ts | 10 ++--- .../src/service/DefaultEntitiesCatalog.ts | 44 ++++++++++++------- .../src/service/createRouter.test.ts | 8 ++-- .../request/parseQueryEntitiesParams.test.ts | 4 +- .../request/parseQueryEntitiesParams.ts | 2 +- 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 5591573ee2..6b6ff5b92e 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -227,7 +227,7 @@ export interface QueryEntitiesInitialRequest { fields?: (entity: Entity) => Entity; limit?: number; filter?: EntityFilter; - sortFields?: EntityOrder[]; + orderFields?: EntityOrder[]; query?: string; } @@ -275,11 +275,11 @@ export type Cursor = { * An array of fields used for sorting the data. * For example, [ { field: 'metadata.name', order: 'asc' } ] */ - sortFields: EntityOrder[]; + orderFields: EntityOrder[]; /** * The values of the fields of a specific item used for paginating the data. */ - sortFieldValues: Array; + orderFieldValues: Array; /** * A filter to be applied to the full list of entities. */ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index ab20c0942a..2924c81106 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -767,7 +767,7 @@ describe('DefaultEntitiesCatalog', () => { const request1: QueryEntitiesInitialRequest = { filter, limit, - sortFields: [{ field: 'metadata.name', order: 'asc' }], + orderFields: [{ field: 'metadata.name', order: 'asc' }], }; const response1 = await catalog.queryEntities(request1); expect(response1.items).toEqual([entityFrom('A'), entityFrom('B')]); @@ -912,7 +912,7 @@ describe('DefaultEntitiesCatalog', () => { const request1: QueryEntitiesInitialRequest = { filter, limit, - sortFields: [{ field: 'metadata.name', order: 'desc' }], + orderFields: [{ field: 'metadata.name', order: 'desc' }], }; const response1 = await catalog.queryEntities(request1); expect(response1.items).toEqual([entityFrom('G'), entityFrom('F')]); @@ -1056,7 +1056,7 @@ describe('DefaultEntitiesCatalog', () => { filter, limit: 100, - sortFields: [{ field: 'metadata.name', order: 'asc' }], + orderFields: [{ field: 'metadata.name', order: 'asc' }], query: 'cAt ', }; const response = await catalog.queryEntities(request); @@ -1098,7 +1098,7 @@ describe('DefaultEntitiesCatalog', () => { limit: 0, }; const response = await catalog.queryEntities(request); - expect(response).toEqual({ totalItems: 20, entities: [] }); + expect(response).toEqual({ totalItems: 20, items: [], pageInfo: {} }); }, ); @@ -1127,7 +1127,7 @@ describe('DefaultEntitiesCatalog', () => { // initial request const request1: QueryEntitiesInitialRequest = { limit, - sortFields: [{ field: 'metadata.name', order: 'asc' }], + orderFields: [{ field: 'metadata.name', order: 'asc' }], }; const response1 = await catalog.queryEntities(request1); expect(response1.items).toMatchObject([ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index ba242b7ab5..959fba1851 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -338,26 +338,27 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const db = this.database; const limit = request?.limit ?? 20; - const cursor: Omit & { - sortFieldValues?: (string | null)[]; + const cursor: Omit & { + orderFieldValues?: (string | null)[]; } = { - sortFields: [defaultSortField], + orderFields: [defaultSortField], isPrevious: false, ...parseCursorFromRequest(request), }; const isFetchingBackwards = cursor.isPrevious; - if (cursor.sortFields.length > 1) { + if (cursor.orderFields.length > 1) { this.logger.warn(`Only one sort field is supported, ignoring the rest`); } const sortField: EntityOrder = { ...defaultSortField, - ...cursor.sortFields[0], + ...cursor.orderFields[0], }; - const [prevItemSortFieldValue, prevItemUid] = cursor.sortFieldValues || []; + const [prevItemOrderFieldValue, prevItemUid] = + cursor.orderFieldValues || []; const dbQuery = db('search') .join('final_entities', 'search.entity_id', 'final_entities.entity_id') @@ -379,14 +380,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const isOrderingDescending = sortField.order === 'desc'; - if (prevItemSortFieldValue) { + if (prevItemOrderFieldValue) { dbQuery.andWhere( 'value', isFetchingBackwards !== isOrderingDescending ? '<' : '>', - prevItemSortFieldValue, + prevItemOrderFieldValue, ); dbQuery.orWhere(function nested() { - this.where('value', '=', prevItemSortFieldValue).andWhere( + this.where('value', '=', prevItemOrderFieldValue).andWhere( 'search.entity_id', isFetchingBackwards !== isOrderingDescending ? '<' : '>', prevItemUid, @@ -451,7 +452,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { const nextCursor = hasMoreResults ? encodeCursor({ ...cursor, - sortFieldValues: sortFieldsFromRow(lastRow), + orderFieldValues: sortFieldsFromRow(lastRow), firstSortFieldValues, isPrevious: false, totalItems, @@ -464,7 +465,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues) ? encodeCursor({ ...cursor, - sortFieldValues: sortFieldsFromRow(firstRow), + orderFieldValues: sortFieldsFromRow(firstRow), firstSortFieldValues: cursor.firstSortFieldValues, isPrevious: true, totalItems, @@ -475,7 +476,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .map(e => JSON.parse(e.final_entity!)) .map(e => (request?.fields ? request.fields(e) : e)); - return { items, pageInfo: { prevCursor, nextCursor }, totalItems }; + return { + items, + pageInfo: { + ...(!!prevCursor && { prevCursor }), + ...(!!nextCursor && { nextCursor }), + }, + totalItems, + }; } async removeEntityByUid(uid: string): Promise { @@ -671,10 +679,10 @@ const entityFilterParser: z.ZodSchema = z.lazy(() => ); export const cursorParser: z.ZodSchema = z.object({ - sortFields: z.array( + orderFields: z.array( z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }), ), - sortFieldValues: z.array(z.string().or(z.null())), + orderFieldValues: z.array(z.string().or(z.null())), filter: entityFilterParser.optional(), isPrevious: z.boolean(), query: z.string().optional(), @@ -696,8 +704,12 @@ function parseCursorFromRequest( request?: QueryEntitiesRequest, ): Partial { if (isQueryEntitiesInitialRequest(request)) { - const { filter, sortFields = [defaultSortField], query } = request; - return { filter, sortFields, query }; + const { + filter, + orderFields: sortFields = [defaultSortField], + query, + } = request; + return { filter, orderFields: sortFields, query }; } if (isQueryEntitiesCursorRequest(request)) { try { diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index a35115ac58..91ede00d51 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -153,7 +153,9 @@ describe('createRouter readonly disabled', () => { expect(response.body).toEqual({ items, totalItems: 100, - nextCursor: 'something', + pageInfo: { + nextCursor: 'something', + }, }); }); @@ -181,7 +183,7 @@ describe('createRouter readonly disabled', () => { { allOf: [{ key: 'c', values: ['4'] }] }, ], }, - sortFields: [ + orderFields: [ { field: 'metadata.name', order: 'asc' }, { field: 'metadata.uid', order: 'desc' }, ], @@ -210,7 +212,7 @@ describe('createRouter readonly disabled', () => { expect(response.body).toEqual({ items, totalItems: 100, - nextCursor: 'next', + pageInfo: { nextCursor: 'next' }, }); }); }); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts index ab702438c2..3a8bf1b225 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -36,7 +36,7 @@ describe('parseQueryEntitiesParams', () => { ) as QueryEntitiesInitialRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); - expect(parsedObj.sortFields).toEqual([ + expect(parsedObj.orderFields).toEqual([ { field: 'metadata.name', order: 'desc' }, ]); expect(parsedObj.filter).toBeDefined(); @@ -50,7 +50,7 @@ describe('parseQueryEntitiesParams', () => { ) as QueryEntitiesInitialRequest; expect(parsedObj.limit).toBeUndefined(); expect(parsedObj.fields).toBeUndefined(); - expect(parsedObj.sortFields).toBeUndefined(); + expect(parsedObj.orderFields).toBeUndefined(); expect(parsedObj.filter).toBeUndefined(); expect(parsedObj.query).toBeUndefined(); expect(parsedObj).not.toHaveProperty('authorizationToken'); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts index 858189b0d2..0a3da2bd48 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -47,7 +47,7 @@ export function parseQueryEntitiesParams( fields, filter, limit, - sortFields, + orderFields: sortFields, query, }; From b02b6c3e053c008b9e66ab135f2852f15a697e07 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 13:58:05 +0100 Subject: [PATCH 44/50] catalog-client: move query to fullTextFilterTerm Signed-off-by: Vincenzo Scamporlino --- .../catalog-client/src/CatalogClient.test.ts | 10 +++++++--- packages/catalog-client/src/CatalogClient.ts | 17 +++++++++++++---- packages/catalog-client/src/types/api.ts | 5 ++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index f11627d092..c235ff1592 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -343,14 +343,16 @@ describe('CatalogClient', () => { await client.queryEntities({ fields: ['a', 'b'], limit: 100, - query: 'query', + fullTextFilter: { + term: 'query', + }, orderFields: [ { field: 'metadata.name', order: 'asc' }, { field: 'metadata.uid', order: 'desc' }, ], }); expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( - '?limit=100&sortField=metadata.name,asc&sortField=metadata.uid,desc&fields=a,b&query=query', + '?limit=100&sortField=metadata.name,asc&sortField=metadata.uid,desc&fields=a,b&fullTextFilterTerm=query', ); }); @@ -366,7 +368,9 @@ describe('CatalogClient', () => { await client.queryEntities({ fields: ['a', 'b'], limit: 100, - query: 'query', + fullTextFilter: { + term: 'query', + }, orderFields: [{ field: 'metadata.name', order: 'asc' }], cursor: 'cursor', }); diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index a380baf6e5..4984d394d4 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -214,7 +214,13 @@ export class CatalogClient implements CatalogApi { const params: string[] = []; if (isQueryEntitiesInitialRequest(request)) { - const { fields = [], filter, limit, orderFields, query } = request; + const { + fields = [], + filter, + limit, + orderFields, + fullTextFilter, + } = request; params.push(...this.getParams(filter)); if (limit !== undefined) { @@ -229,9 +235,12 @@ export class CatalogClient implements CatalogApi { params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); } - const normalizedQuery = query?.trim(); - if (normalizedQuery) { - params.push(`query=${normalizedQuery}`); + const normalizedFullTextFilterTerm = fullTextFilter?.term?.trim(); + if (normalizedFullTextFilterTerm) { + params.push(`fullTextFilterTerm=${normalizedFullTextFilterTerm}`); + } + if (fullTextFilter?.fields?.length) { + params.push(`fullTextFilterFields=${fullTextFilter.fields.join(',')}`); } } else { const { fields = [], limit, cursor } = request; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 4a727b7bf0..963e0521a0 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -401,7 +401,10 @@ export type QueryEntitiesInitialRequest = { limit?: number; filter?: EntityFilterQuery; orderFields?: EntityOrderQuery; - query?: string; + fullTextFilter?: { + term: string; + fields?: string[]; + }; }; /** From 5468b81811b82450f69514a497b8860647b3bcb7 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 13:59:50 +0100 Subject: [PATCH 45/50] catalog-backend: move query to fullTextFilterTerm Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 12 +++++-- .../service/DefaultEntitiesCatalog.test.ts | 2 +- .../src/service/DefaultEntitiesCatalog.ts | 10 +++--- .../src/service/createRouter.test.ts | 6 +++- ...ts => parseEntityOrderFieldParams.test.ts} | 34 +++++++++---------- ...rams.ts => parseEntityOrderFieldParams.ts} | 12 +++---- .../request/parseFullTextFilterFields.ts | 28 +++++++++++++++ .../request/parseQueryEntitiesParams.test.ts | 17 ++++++---- .../request/parseQueryEntitiesParams.ts | 19 ++++++++--- 9 files changed, 96 insertions(+), 44 deletions(-) rename plugins/catalog-backend/src/service/request/{parseEntitySortFieldParams.test.ts => parseEntityOrderFieldParams.test.ts} (53%) rename plugins/catalog-backend/src/service/request/{parseEntitySortFieldParams.ts => parseEntityOrderFieldParams.ts} (75%) create mode 100644 plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 6b6ff5b92e..d182ffdc47 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -228,7 +228,10 @@ export interface QueryEntitiesInitialRequest { limit?: number; filter?: EntityFilter; orderFields?: EntityOrder[]; - query?: string; + fullTextFilter?: { + term: string; + fields?: string[]; + }; } /** @@ -289,9 +292,12 @@ export type Cursor = { */ isPrevious: boolean; /** - * Used for filtering the data by name. + * Used for performing full text filtering on the given fields. */ - query?: string; + fullTextFilter?: { + term: string; + fields?: string[]; + }; /** * The value of the fields of the first returned item used for paginating the data. * The catalog uses this field internally to understand if the beginning diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 2924c81106..5bfb0e0077 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1057,7 +1057,7 @@ describe('DefaultEntitiesCatalog', () => { limit: 100, orderFields: [{ field: 'metadata.name', order: 'asc' }], - query: 'cAt ', + fullTextFilter: { term: 'cAt ' }, }; const response = await catalog.queryEntities(request); expect(response.items).toEqual([ diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 959fba1851..ba59bc55c0 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -368,11 +368,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id'); } - const normalizedQueryByName = cursor.query?.trim(); - if (normalizedQueryByName) { + const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim(); + if (normalizedFullTextFilterTerm) { dbQuery.andWhereRaw( 'value like ?', - `%${normalizedQueryByName.toLocaleLowerCase('en-US')}%`, + `%${normalizedFullTextFilterTerm.toLocaleLowerCase('en-US')}%`, ); } @@ -707,9 +707,9 @@ function parseCursorFromRequest( const { filter, orderFields: sortFields = [defaultSortField], - query, + fullTextFilter, } = request; - return { filter, orderFields: sortFields, query }; + return { filter, orderFields: sortFields, fullTextFilter }; } if (isQueryEntitiesCursorRequest(request)) { try { diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 91ede00d51..5121e800da 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -166,7 +166,7 @@ describe('createRouter readonly disabled', () => { totalItems: 0, }); const response = await request(app).get( - '/entities/by-query?filter=a=1,a=2,b=3&filter=c=4&sortField=metadata.name,asc&sortField=metadata.uid,desc', + '/entities/by-query?filter=a=1,a=2,b=3&filter=c=4&orderField=metadata.name,asc&orderField=metadata.uid,desc', ); expect(response.status).toEqual(200); @@ -187,6 +187,10 @@ describe('createRouter readonly disabled', () => { { field: 'metadata.name', order: 'asc' }, { field: 'metadata.uid', order: 'desc' }, ], + fullTextFilter: { + fields: undefined, + term: '', + }, }); }); diff --git a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts similarity index 53% rename from plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts rename to plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts index 35f7c2cac0..b732163394 100644 --- a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.test.ts @@ -14,31 +14,31 @@ * limitations under the License. */ -import { parseEntitySortFieldParams } from './parseEntitySortFieldParams'; +import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; -describe('parseEntitySortFieldParams', () => { - it('supports no sort fields', () => { - const result = parseEntitySortFieldParams({}); +describe('parseEntityOrderFieldParams', () => { + it('supports no order fields', () => { + const result = parseEntityOrderFieldParams({}); expect(result).toEqual(undefined); }); - it('supports single sortField', () => { - const result = parseEntitySortFieldParams({ - sortField: ['metadata.name,desc'], + it('supports single orderField', () => { + const result = parseEntityOrderFieldParams({ + orderField: ['metadata.name,desc'], })!; expect(result).toEqual([{ field: 'metadata.name', order: 'desc' }]); }); - it('supports single sortField without order', () => { - const result = parseEntitySortFieldParams({ - sortField: ['metadata.name'], + it('supports single orderField without order', () => { + const result = parseEntityOrderFieldParams({ + orderField: ['metadata.name'], })!; expect(result).toEqual([{ field: 'metadata.name' }]); }); - it('supports multiple sort fields', () => { - const result = parseEntitySortFieldParams({ - sortField: ['metadata.name,desc', 'metadata.uid,asc'], + it('supports multiple order fields', () => { + const result = parseEntityOrderFieldParams({ + orderField: ['metadata.name,desc', 'metadata.uid,asc'], }); expect(result).toEqual([ @@ -47,11 +47,11 @@ describe('parseEntitySortFieldParams', () => { ]); }); - it('throws if sortField order is not valid', () => { + it('throws if orderField order is not valid', () => { expect(() => - parseEntitySortFieldParams({ - sortField: ['metadata.name,desc', 'metadata.uid,invalid'], + parseEntityOrderFieldParams({ + orderField: ['metadata.name,desc', 'metadata.uid,invalid'], }), - ).toThrow(/Invalid sort field order/); + ).toThrow(/Invalid order field order/); }); }); diff --git a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts similarity index 75% rename from plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts rename to plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts index 5efa2de05e..5ab8cbfcf9 100644 --- a/plugins/catalog-backend/src/service/request/parseEntitySortFieldParams.ts +++ b/plugins/catalog-backend/src/service/request/parseEntityOrderFieldParams.ts @@ -18,19 +18,19 @@ import { InputError } from '@backstage/errors'; import { EntityOrder } from '../../catalog/types'; import { parseStringsParam } from './common'; -export function parseEntitySortFieldParams( +export function parseEntityOrderFieldParams( params: Record, ): EntityOrder[] | undefined { - const sortFieldStrings = parseStringsParam(params.sortField, 'sortField'); - if (!sortFieldStrings) { + const orderFieldStrings = parseStringsParam(params.orderField, 'orderField'); + if (!orderFieldStrings) { return undefined; } - return sortFieldStrings.map(sortFieldString => { - const [field, order] = sortFieldString.split(','); + return orderFieldStrings.map(orderFieldString => { + const [field, order] = orderFieldString.split(','); if (order !== undefined && !isOrder(order)) { - throw new InputError('Invalid sort field order, must be asc or desc'); + throw new InputError('Invalid order field order, must be asc or desc'); } return { field, order }; }); diff --git a/plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts b/plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts new file mode 100644 index 0000000000..02ba5864fc --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseFullTextFilterFields.ts @@ -0,0 +1,28 @@ +import { parseStringParam } from './common'; + +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function parseFullTextFilterFields(params: Record) { + const fullTextFilterFields = parseStringParam( + params.fullTextFilterFields, + 'fullTextFilterFields', + ); + if (!fullTextFilterFields) { + return undefined; + } + + return fullTextFilterFields.split(','); +} diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts index 3a8bf1b225..da2d7e474c 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -28,8 +28,9 @@ describe('parseQueryEntitiesParams', () => { fields: ['kind'], limit: '3', filter: ['a=1', 'b=2'], - sortField: ['metadata.name,desc'], - query: 'query', + orderField: ['metadata.name,desc'], + fullTextFilterTerm: 'query', + fullTextFilterFields: 'metadata.name,metadata.namespace', }; const parsedObj = parseQueryEntitiesParams( validRequest, @@ -40,7 +41,10 @@ describe('parseQueryEntitiesParams', () => { { field: 'metadata.name', order: 'desc' }, ]); expect(parsedObj.filter).toBeDefined(); - expect(parsedObj.query).toBe('query'); + expect(parsedObj.fullTextFilter).toEqual({ + term: 'query', + fields: ['metadata.name', 'metadata.namespace'], + }); expect(parsedObj).not.toHaveProperty('authorizationToken'); expect(parsedObj).not.toHaveProperty('cursor'); }); @@ -52,7 +56,7 @@ describe('parseQueryEntitiesParams', () => { expect(parsedObj.fields).toBeUndefined(); expect(parsedObj.orderFields).toBeUndefined(); expect(parsedObj.filter).toBeUndefined(); - expect(parsedObj.query).toBeUndefined(); + expect(parsedObj.fullTextFilter).toEqual({ term: '', fields: undefined }); expect(parsedObj).not.toHaveProperty('authorizationToken'); expect(parsedObj).not.toHaveProperty('cursor'); }); @@ -62,9 +66,10 @@ describe('parseQueryEntitiesParams', () => { limit: 'asd', }, { filter: 3 }, - { sortField: ['metadata.uid,diagonal'] }, + { orderField: ['metadata.uid,diagonal'] }, { fields: [4] }, - { query: [] }, + { fullTextFilterTerm: [] }, + { fullTextFilterFields: 3 }, ])('should throw if some parameter is not valid %p', params => { expect(() => parseQueryEntitiesParams(params)).toThrow(); }); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts index 0a3da2bd48..67a4629c1c 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -21,8 +21,9 @@ import { } from '../../catalog/types'; import { parseIntegerParam, parseStringParam } from './common'; import { parseEntityFilterParams } from './parseEntityFilterParams'; -import { parseEntitySortFieldParams } from './parseEntitySortFieldParams'; +import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; import { parseEntityTransformParams } from './parseEntityTransformParams'; +import { parseFullTextFilterFields } from './parseFullTextFilterFields'; export function parseQueryEntitiesParams( params: Record, @@ -40,15 +41,23 @@ export function parseQueryEntitiesParams( } const filter = parseEntityFilterParams(params); - const query = parseStringParam(params.query, 'query'); - const sortFields = parseEntitySortFieldParams(params); + const fullTextFilterTerm = parseStringParam( + params.fullTextFilterTerm, + 'fullTextFilterTerm', + ); + const fullTextFilterFields = parseFullTextFilterFields(params); + + const orderFields = parseEntityOrderFieldParams(params); const response: Omit = { fields, filter, limit, - orderFields: sortFields, - query, + orderFields, + fullTextFilter: { + term: fullTextFilterTerm || '', + fields: fullTextFilterFields, + }, }; return response; From 8c030b1826c9f6629612c0f87f900f04d285519d Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 14:04:27 +0100 Subject: [PATCH 46/50] catalog: split changeset Signed-off-by: Vincenzo Scamporlino --- .changeset/breezy-bees-care.md | 5 +++++ .changeset/silly-suits-run.md | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .changeset/breezy-bees-care.md diff --git a/.changeset/breezy-bees-care.md b/.changeset/breezy-bees-care.md new file mode 100644 index 0000000000..9a5b59fe03 --- /dev/null +++ b/.changeset/breezy-bees-care.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +Add `queryEntities` method to `CatalogApi`. diff --git a/.changeset/silly-suits-run.md b/.changeset/silly-suits-run.md index 2fc731f2a4..ec371e4077 100644 --- a/.changeset/silly-suits-run.md +++ b/.changeset/silly-suits-run.md @@ -1,8 +1,7 @@ --- -'@backstage/catalog-client': minor '@backstage/plugin-catalog-backend': minor --- Add /entities/by-query endpoint returning paginated entities. -The endpoint supports cursor base pagination and server side sorting of the entities by name or uid. +The endpoint supports cursor base pagination and server side sorting of the entities From 88740bed619452c57853fe13ee5d2c041a187f10 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 14:05:49 +0100 Subject: [PATCH 47/50] catalog-client: improve queryEntities example Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/src/types/api.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 963e0521a0..ee6579efca 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -477,14 +477,16 @@ export interface CatalogApi { * * @remarks * - * Example: + * @example * * ``` * const response = await catalogClient.queryEntities({ * filter: [{ kind: 'group' }], * limit: 20, - * query: 'A', - * sortFields: { field: 'metadata.name' order: 'asc' }, + * fullTextFilter: { + * term: 'A', + * } + * orderFields: { field: 'metadata.name' order: 'asc' }, * }); * ``` * From a7469138be3601324e38c94782d3fccda9040fcd Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 17:44:42 +0100 Subject: [PATCH 48/50] catalog-client: new cursors in api report Signed-off-by: Vincenzo Scamporlino --- packages/catalog-client/api-report.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index b8a6a17a09..22feceabec 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -261,8 +261,11 @@ export type QueryEntitiesInitialRequest = { fields?: string[]; limit?: number; filter?: EntityFilterQuery; - sortFields?: EntityOrderQuery; - query?: string; + orderFields?: EntityOrderQuery; + fullTextFilter?: { + term: string; + fields?: string[]; + }; }; // @public @@ -272,10 +275,12 @@ export type QueryEntitiesRequest = // @public export type QueryEntitiesResponse = { - entities: Entity[]; + items: Entity[]; totalItems: number; - nextCursor?: string; - prevCursor?: string; + pageInfo: { + nextCursor?: string; + prevCursor?: string; + }; }; // @public From 9e3915e40c70042255df507391647f9363aa0698 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 17:45:56 +0100 Subject: [PATCH 49/50] catalog-client: permission queryEntities cursor requests Signed-off-by: Vincenzo Scamporlino --- plugins/catalog-backend/src/catalog/types.ts | 8 +- .../service/AuthorizedEntitiesCatalog.test.ts | 149 ++++++++++++++++++ .../src/service/AuthorizedEntitiesCatalog.ts | 57 +++++-- .../service/DefaultEntitiesCatalog.test.ts | 20 --- .../src/service/DefaultEntitiesCatalog.ts | 34 ++-- .../src/service/createRouter.test.ts | 51 +++++- .../src/service/createRouter.ts | 23 ++- .../request/parseQueryEntitiesParams.test.ts | 28 +++- .../request/parseQueryEntitiesParams.ts | 4 +- plugins/catalog-backend/src/service/util.ts | 44 ++++++ 10 files changed, 340 insertions(+), 78 deletions(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index d182ffdc47..62c8fad605 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -179,7 +179,7 @@ export interface EntitiesCatalog { * * @param request */ - queryEntities(request?: QueryEntitiesRequest): Promise; + queryEntities(request: QueryEntitiesRequest): Promise; /** * Removes a single entity. @@ -242,7 +242,7 @@ export interface QueryEntitiesCursorRequest { authorizationToken?: string; fields?: (entity: Entity) => Entity; limit?: number; - cursor: string; + cursor: Cursor; } /** @@ -258,11 +258,11 @@ export interface QueryEntitiesResponse { /** * The cursor of the next pagination request. */ - nextCursor?: string; + nextCursor?: Cursor; /** * The cursor of the previous pagination request. */ - prevCursor?: string; + prevCursor?: Cursor; }; /** * the total number of entities matching the current filters. diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index e559f5fd49..2ee47dae19 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -20,6 +20,8 @@ import { createConditionTransformer } from '@backstage/plugin-permission-node'; import { isEntityKind } from '../permissions/rules/isEntityKind'; import { CatalogPermissionRule } from '../permissions/rules'; import { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog'; +import { Cursor, EntityFilter, QueryEntitiesResponse } from '../catalog/types'; +import { Entity } from '@backstage/catalog-model'; describe('AuthorizedEntitiesCatalog', () => { const fakeCatalog = { @@ -157,6 +159,153 @@ describe('AuthorizedEntitiesCatalog', () => { }); }); + describe('queryEntities', () => { + it('returns empty response on DENY', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.DENY }, + ]); + const catalog = createCatalog(); + + await expect( + catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }), + ).resolves.toEqual({ + items: [], + pageInfo: {}, + totalItems: 0, + }); + + expect(fakeCatalog.queryEntities).not.toHaveBeenCalled(); + }); + + it('calls underlying catalog method on ALLOW', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.ALLOW }, + ]); + const catalog = createCatalog(); + + await catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({ + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + }); + + it('calls underlying catalog method with correct filter on CONDITIONAL', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { + result: AuthorizeResult.CONDITIONAL, + conditions: { + rule: 'IS_ENTITY_KIND', + params: { kinds: ['b'] }, + }, + }, + ]); + + const requestFilter: EntityFilter = { key: 'name', values: ['name'] }; + + const entities = [ + { + kind: 'component', + namespace: 'default', + name: 'a', + } as unknown as Entity, + { + kind: 'component', + namespace: 'default', + name: 'b1', + } as unknown as Entity, + ]; + + fakeCatalog.queryEntities.mockResolvedValue({ + items: entities, + pageInfo: { + nextCursor: { + isPrevious: false, + orderFieldValues: ['xxx', null], + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + prevCursor: { + isPrevious: true, + orderFieldValues: ['a', null], + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + }, + totalItems: 4, + } as QueryEntitiesResponse); + const catalog = createCatalog(isEntityKind); + + let response = await catalog.queryEntities({ + authorizationToken: 'abcd', + filter: { key: 'name', values: ['name'] }, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenCalledWith({ + authorizationToken: 'abcd', + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }); + + expect(response).toEqual({ + items: entities, + totalItems: 4, + pageInfo: { + nextCursor: { + isPrevious: false, + filter: requestFilter, + orderFieldValues: ['xxx', null], + }, + prevCursor: { + isPrevious: true, + filter: requestFilter, + orderFieldValues: ['a', null], + }, + }, + }); + + const cursor: Cursor = { + filter: requestFilter, + orderFields: [{ field: 'name', order: 'asc' }], + isPrevious: false, + orderFieldValues: ['a', null], + }; + response = await catalog.queryEntities({ + authorizationToken: 'abcd', + cursor, + }); + + expect(fakeCatalog.queryEntities).toHaveBeenNthCalledWith(2, { + authorizationToken: 'abcd', + cursor: { + ...cursor, + filter: { allOf: [{ key: 'kind', values: ['b'] }, requestFilter] }, + }, + }); + + expect(response).toEqual({ + items: entities, + totalItems: 4, + pageInfo: { + nextCursor: { + isPrevious: false, + filter: requestFilter, + orderFieldValues: ['xxx', null], + }, + prevCursor: { + isPrevious: true, + filter: requestFilter, + orderFieldValues: ['a', null], + }, + }, + }); + }); + }); + describe('removeEntityByUid', () => { it('throws error on DENY', async () => { fakeCatalog.entities.mockResolvedValue({ diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index f9ec058879..2649b45260 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -26,6 +26,7 @@ import { } from '@backstage/plugin-permission-common'; import { ConditionTransformer } from '@backstage/plugin-permission-node'; import { + Cursor, EntitiesBatchRequest, EntitiesBatchResponse, EntitiesCatalog, @@ -39,7 +40,7 @@ import { QueryEntitiesResponse, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; -import { isQueryEntitiesInitialRequest } from './util'; +import { isQueryEntitiesCursorRequest } from './util'; export class AuthorizedEntitiesCatalog implements EntitiesCatalog { constructor( @@ -110,12 +111,12 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { } async queryEntities( - request?: QueryEntitiesRequest, + request: QueryEntitiesRequest, ): Promise { const authorizeDecision = ( await this.permissionApi.authorizeConditional( [{ permission: catalogEntityReadPermission }], - { token: request?.authorizationToken }, + { token: request.authorizationToken }, ) )[0]; @@ -132,14 +133,52 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { authorizeDecision.conditions, ); - return this.entitiesCatalog.queryEntities({ - ...request, - ...(isQueryEntitiesInitialRequest(request) && { - filter: request?.filter + let permissionedRequest: QueryEntitiesRequest; + let requestFilter: EntityFilter | undefined; + + if (isQueryEntitiesCursorRequest(request)) { + requestFilter = request.cursor.filter; + + permissionedRequest = { + ...request, + cursor: { + ...request.cursor, + filter: request.cursor.filter + ? { allOf: [permissionFilter, request.cursor.filter] } + : permissionFilter, + }, + }; + } else { + permissionedRequest = { + ...request, + filter: request.filter ? { allOf: [permissionFilter, request.filter] } : permissionFilter, - }), - }); + }; + requestFilter = request.filter; + } + + const response = await this.entitiesCatalog.queryEntities( + permissionedRequest, + ); + + const prevCursor: Cursor | undefined = response.pageInfo.prevCursor && { + ...response.pageInfo.prevCursor, + filter: requestFilter, + }; + + const nextCursor: Cursor | undefined = response.pageInfo.nextCursor && { + ...response.pageInfo.nextCursor, + filter: requestFilter, + }; + + return { + ...response, + pageInfo: { + prevCursor, + nextCursor, + }, + }; } return this.entitiesCatalog.queryEntities(request); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 5bfb0e0077..740f5b2558 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -1291,26 +1291,6 @@ describe('DefaultEntitiesCatalog', () => { expect(response5.totalItems).toBe(6); }, ); - - it('should throw in case of malformed cursor', async () => { - const catalog = new DefaultEntitiesCatalog({ - database: knex, - logger: getVoidLogger(), - stitcher, - }); - - const cursor = Buffer.from( - JSON.stringify({ filter: 'notavalidfilter' }), - 'utf8', - ).toString('base64'); - const request: QueryEntitiesCursorRequest = { - cursor, - }; - - await expect(catalog.queryEntities(request)).rejects.toThrow( - Error('Malformed cursor, could not be parsed'), - ); - }); }); describe('removeEntityByUid', () => { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index ba59bc55c0..a635a9a7d2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -333,10 +333,10 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } async queryEntities( - request?: QueryEntitiesRequest, + request: QueryEntitiesRequest, ): Promise { const db = this.database; - const limit = request?.limit ?? 20; + const limit = request.limit ?? 20; const cursor: Omit & { orderFieldValues?: (string | null)[]; @@ -449,32 +449,32 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { firstRow?.entity_id, ]; - const nextCursor = hasMoreResults - ? encodeCursor({ + const nextCursor: Cursor | undefined = hasMoreResults + ? { ...cursor, orderFieldValues: sortFieldsFromRow(lastRow), firstSortFieldValues, isPrevious: false, totalItems, - }) + } : undefined; - const prevCursor = + const prevCursor: Cursor | undefined = !isInitialRequest && rows.length > 0 && !isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues) - ? encodeCursor({ + ? { ...cursor, orderFieldValues: sortFieldsFromRow(firstRow), firstSortFieldValues: cursor.firstSortFieldValues, isPrevious: true, totalItems, - }) + } : undefined; const items = rows .map(e => JSON.parse(e.final_entity!)) - .map(e => (request?.fields ? request.fields(e) : e)); + .map(e => (request.fields ? request.fields(e) : e)); return { items, @@ -690,16 +690,6 @@ export const cursorParser: z.ZodSchema = z.object({ totalItems: z.number().optional(), }); -function encodeCursor(cursor: Cursor) { - const json = JSON.stringify(cursor); - return Buffer.from(json, 'utf8').toString('base64'); -} - -function decodeCursor(encodedCursor: string) { - const json = Buffer.from(encodedCursor, 'base64').toString('utf8'); - return cursorParser.parse(JSON.parse(json)); -} - function parseCursorFromRequest( request?: QueryEntitiesRequest, ): Partial { @@ -712,11 +702,7 @@ function parseCursorFromRequest( return { filter, orderFields: sortFields, fullTextFilter }; } if (isQueryEntitiesCursorRequest(request)) { - try { - return decodeCursor(request.cursor); - } catch { - throw new InputError('Malformed cursor, could not be parsed'); - } + return request.cursor; } return {}; } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 5121e800da..f2f87fa800 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -25,7 +25,7 @@ import { } from '@backstage/catalog-model'; import express from 'express'; import request from 'supertest'; -import { EntitiesCatalog } from '../catalog/types'; +import { Cursor, EntitiesCatalog } from '../catalog/types'; import { LocationInput, LocationService, RefreshService } from './types'; import { basicEntityFilter } from './request'; import { createRouter } from './createRouter'; @@ -37,6 +37,7 @@ import { import { RESOURCE_TYPE_CATALOG_ENTITY } from '@backstage/plugin-catalog-common'; import { CatalogProcessingOrchestrator } from '../processing/types'; import { z } from 'zod'; +import { encodeCursor } from './util'; describe('createRouter readonly disabled', () => { let entitiesCatalog: jest.Mocked; @@ -145,7 +146,7 @@ describe('createRouter readonly disabled', () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ items, totalItems: 100, - pageInfo: { nextCursor: 'something' }, + pageInfo: { nextCursor: mockCursor() }, }); const response = await request(app).get('/entities/by-query'); @@ -154,7 +155,7 @@ describe('createRouter readonly disabled', () => { items, totalItems: 100, pageInfo: { - nextCursor: 'something', + nextCursor: expect.any(String), }, }); }); @@ -202,23 +203,50 @@ describe('createRouter readonly disabled', () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ items, totalItems: 100, - pageInfo: { nextCursor: 'next' }, + pageInfo: { nextCursor: mockCursor() }, }); + const cursor = mockCursor({ totalItems: 100, isPrevious: false }); + const response = await request(app).get( - '/entities/by-query?cursor=something', + `/entities/by-query?cursor=${encodeCursor(cursor)}`, ); expect(entitiesCatalog.queryEntities).toHaveBeenCalledTimes(1); expect(entitiesCatalog.queryEntities).toHaveBeenCalledWith({ - cursor: 'something', + cursor, }); expect(response.status).toEqual(200); expect(response.body).toEqual({ items, totalItems: 100, - pageInfo: { nextCursor: 'next' }, + pageInfo: { nextCursor: expect.any(String) }, }); }); + + it('should throw in case of malformed cursor', async () => { + const items: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.queryEntities.mockResolvedValueOnce({ + items, + totalItems: 100, + pageInfo: { nextCursor: mockCursor() }, + }); + + let response = await request(app).get( + `/entities/by-query?cursor=${Buffer.from( + JSON.stringify({ bad: 'cursor' }), + 'utf8', + ).toString('base64')}`, + ); + expect(response.status).toEqual(400); + expect(response.body.error.message).toMatch(/Malformed cursor/); + + response = await request(app).get(`/entities/by-query?cursor=badcursor`); + expect(response.status).toEqual(400); + expect(response.body.error.message).toMatch(/Malformed cursor/); + }); }); describe('GET /entities/by-uid/:uid', () => { @@ -908,3 +936,12 @@ describe('NextRouter permissioning', () => { }); }); }); + +function mockCursor(partialCursor?: Partial): Cursor { + return { + orderFields: [], + orderFieldValues: [], + isPrevious: false, + ...partialCursor, + }; +} diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 7603f6955d..849096c601 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -46,6 +46,7 @@ import { parseEntityOrderParams } from './request/parseEntityOrderParams'; import { LocationService, RefreshOptions, RefreshService } from './types'; import { disallowReadonlyMode, + encodeCursor, locationInput, validateRequestBody, } from './util'; @@ -132,12 +133,24 @@ export async function createRouter( res.json(entities); }) .get('/entities/by-query', async (req, res) => { - const response = await entitiesCatalog.queryEntities({ - ...parseQueryEntitiesParams(req.query), - authorizationToken: getBearerToken(req.header('authorization')), - }); + const { items, pageInfo, totalItems } = + await entitiesCatalog.queryEntities({ + ...parseQueryEntitiesParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); - res.json(response); + res.json({ + items, + totalItems, + pageInfo: { + ...(pageInfo.nextCursor && { + nextCursor: encodeCursor(pageInfo.nextCursor), + }), + ...(pageInfo.prevCursor && { + prevCursor: encodeCursor(pageInfo.prevCursor), + }), + }, + }); }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts index da2d7e474c..43ba218eac 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.test.ts @@ -15,9 +15,11 @@ */ import { + Cursor, QueryEntitiesCursorRequest, QueryEntitiesInitialRequest, } from '../../catalog/types'; +import { encodeCursor } from '../util'; import { parseQueryEntitiesParams } from './parseQueryEntitiesParams'; describe('parseQueryEntitiesParams', () => { @@ -77,29 +79,40 @@ describe('parseQueryEntitiesParams', () => { describe('cursor request', () => { it('should parse all the defined params', () => { + const cursor: Cursor = { + totalItems: 100, + orderFields: [], + orderFieldValues: [], + isPrevious: false, + }; const validRequest = { authorizationToken: 'to_not_be_returned', fields: ['kind'], limit: '3', - cursor: 'cursor', + cursor: encodeCursor(cursor), }; const parsedObj = parseQueryEntitiesParams( validRequest, ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); - expect(parsedObj.cursor).toBe('cursor'); + expect(parsedObj.cursor).toEqual(cursor); }); it('should ignore unknown params', () => { + const cursor: Cursor = { + totalItems: 100, + orderFields: [], + orderFieldValues: [], + isPrevious: false, + }; const validRequest = { authorizationToken: 'to_not_be_returned', fields: ['kind'], limit: '3', - cursor: 'cursor', + cursor: encodeCursor(cursor), filter: ['a=1', 'b=2'], - sortField: 'sortField', - sortFieldOrder: 'desc', + orderField: 'orderField,desc', query: 'query', }; const parsedObj = parseQueryEntitiesParams( @@ -107,10 +120,9 @@ describe('parseQueryEntitiesParams', () => { ) as QueryEntitiesCursorRequest; expect(parsedObj.limit).toBe(3); expect(parsedObj.fields).toBeDefined(); - expect(parsedObj.cursor).toBe('cursor'); + expect(parsedObj.cursor).toEqual(cursor); expect(parsedObj).not.toHaveProperty('filter'); - expect(parsedObj).not.toHaveProperty('sortField'); - expect(parsedObj).not.toHaveProperty('sortFieldOrder'); + expect(parsedObj).not.toHaveProperty('orderField'); expect(parsedObj).not.toHaveProperty('query'); }); diff --git a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts index 67a4629c1c..ae8b649d13 100644 --- a/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts +++ b/plugins/catalog-backend/src/service/request/parseQueryEntitiesParams.ts @@ -19,6 +19,7 @@ import { QueryEntitiesInitialRequest, QueryEntitiesRequest, } from '../../catalog/types'; +import { decodeCursor } from '../util'; import { parseIntegerParam, parseStringParam } from './common'; import { parseEntityFilterParams } from './parseEntityFilterParams'; import { parseEntityOrderFieldParams } from './parseEntityOrderFieldParams'; @@ -32,8 +33,9 @@ export function parseQueryEntitiesParams( const limit = parseIntegerParam(params.limit, 'limit'); const cursor = parseStringParam(params.cursor, 'cursor'); if (cursor) { + const decodedCursor = decodeCursor(cursor); const response: Omit = { - cursor, + cursor: decodedCursor, fields, limit, }; diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 1ab154a86d..699bca2fc9 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -19,6 +19,8 @@ import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; import { + Cursor, + EntityFilter, QueryEntitiesCursorRequest, QueryEntitiesInitialRequest, QueryEntitiesRequest, @@ -88,3 +90,45 @@ export function isQueryEntitiesCursorRequest( } return !!(input as QueryEntitiesCursorRequest).cursor; } + +const entityFilterParser: z.ZodSchema = z.lazy(() => + z + .object({ + key: z.string(), + values: z.array(z.string()).optional(), + }) + .or(z.object({ not: entityFilterParser })) + .or(z.object({ anyOf: z.array(entityFilterParser) })) + .or(z.object({ allOf: z.array(entityFilterParser) })), +); + +export const cursorParser: z.ZodSchema = z.object({ + orderFields: z.array( + z.object({ field: z.string(), order: z.enum(['asc', 'desc']) }), + ), + orderFieldValues: z.array(z.string().or(z.null())), + filter: entityFilterParser.optional(), + isPrevious: z.boolean(), + query: z.string().optional(), + firstSortFieldValues: z.array(z.string().or(z.null())).optional(), + totalItems: z.number().optional(), +}); + +export function encodeCursor(cursor: Cursor) { + const json = JSON.stringify(cursor); + return Buffer.from(json, 'utf8').toString('base64'); +} + +export function decodeCursor(encodedCursor: string) { + try { + const data = Buffer.from(encodedCursor, 'base64').toString('utf8'); + const result = cursorParser.safeParse(JSON.parse(data)); + + if (!result.success) { + throw new InputError(`Malformed cursor: ${result.error}`); + } + return result.data; + } catch (e) { + throw new InputError(`Malformed cursor: ${e}`); + } +} From bd82c2242cd97477f829909810cf36698123b723 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Fri, 3 Feb 2023 17:48:44 +0100 Subject: [PATCH 50/50] catalog-backend define default limit Signed-off-by: Vincenzo Scamporlino --- .../catalog-backend/src/service/DefaultEntitiesCatalog.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index a635a9a7d2..f983d7496b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -62,6 +62,8 @@ const defaultSortField: EntityOrder = { order: 'asc', }; +const DEFAULT_LIMIT = 20; + function parsePagination(input?: EntityPagination): { limit?: number; offset?: number; @@ -336,7 +338,8 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { request: QueryEntitiesRequest, ): Promise { const db = this.database; - const limit = request.limit ?? 20; + + const limit = request.limit ?? DEFAULT_LIMIT; const cursor: Omit & { orderFieldValues?: (string | null)[];