diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index faf7451acd..9f5ad642f2 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -174,6 +174,16 @@ export interface EntitiesCatalog { */ entitiesBatch(request: EntitiesBatchRequest): Promise; + /** + * Fetch entities and scroll back and forth between entities. + * + * @alpha + * @param request + */ + paginatedEntities( + request?: PaginatedEntitiesRequest, + ): Promise; + /** * Removes a single entity. * @@ -202,3 +212,108 @@ export interface EntitiesCatalog { */ facets(request: EntityFacetsRequest): Promise; } + +/** + * The request shape for {@link EntitiesCatalog.paginatedEntities}. + * + * @alpha + */ +export type PaginatedEntitiesRequest = + | PaginatedEntitiesInitialRequest + | PaginatedEntitiesCursorRequest; + +/** + * The initial request for {@link EntitiesCatalog.paginatedEntities}. + * The request take immutable properties that are going to be bound + * for the current and the next pagination requests. + * + * @alpha + */ +export interface PaginatedEntitiesInitialRequest { + authorizationToken?: string; + fields?: (entity: Entity) => Entity; + limit?: number; + filter?: EntityFilter; + sortField?: string; + query?: string; + sortFieldOrder?: 'asc' | 'desc' | undefined; +} + +/** + * Request for {@link EntitiesCatalog.paginatedEntities} used to + * move forward or backward on the data. + * + * @alpha + */ +export interface PaginatedEntitiesCursorRequest { + authorizationToken?: string; + fields?: (entity: Entity) => Entity; + limit?: number; + cursor: string; +} + +/** + * The response shape for {@link EntitiesCatalog.paginatedEntities}. + * + * @alpha + */ +export interface PaginatedEntitiesResponse { + /** + * The entities for the current pagination request + */ + entities: Entity[]; + /** + * The cursor of the next pagination request. + */ + nextCursor?: string; + /** + * The cursor of the previous pagination request. + */ + prevCursor?: string; + /** + * the total number of entities matching the current filters. + */ + totalItems: number; +} + +/** + * The Cursor used internally by the catalog. + * + * @alpha + */ +export type Cursor = { + /** + * the id of the field used for sorting the data. + * For example, metadata.name + */ + sortField: string; + /** + * The value of the last item returned + * by the request. This is used for performing + * cursor based pagination. + */ + sortFieldId: string; + /** + * In which order the data should be paginated. + */ + sortFieldOrder: 'asc' | 'desc'; + filter?: EntityFilter; + /** + * true if the cursor is a previous cursor. + */ + isPrevious: boolean; + /** + * Filter the data by name. + */ + query?: string; + /** + * Sort field id of the first item. + * The catalog uses this field internally for understanding when the beginning + * of the list has been reached when performing cursor based pagination. + */ + firstFieldId: string; + /** + * The number of items that match the provided filters + */ + totalItems?: number; +}; diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index c62e9f9a84..5ae31158f8 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -35,8 +35,11 @@ import { EntityFacetsRequest, EntityFacetsResponse, EntityFilter, + PaginatedEntitiesRequest, + PaginatedEntitiesResponse, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; +import { isPaginatedEntitiesInitialRequest } from './util'; export class AuthorizedEntitiesCatalog implements EntitiesCatalog { constructor( @@ -106,6 +109,41 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { return this.entitiesCatalog.entitiesBatch(request); } + async paginatedEntities( + request?: PaginatedEntitiesRequest, + ): Promise { + const authorizeDecision = ( + await this.permissionApi.authorizeConditional( + [{ permission: catalogEntityReadPermission }], + { token: request?.authorizationToken }, + ) + )[0]; + + if (authorizeDecision.result === AuthorizeResult.DENY) { + return { + entities: [], + totalItems: 0, + }; + } + + if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) { + const permissionFilter: EntityFilter = this.transformConditions( + authorizeDecision.conditions, + ); + + return this.entitiesCatalog.paginatedEntities({ + ...request, + ...(isPaginatedEntitiesInitialRequest(request) && { + filter: request?.filter + ? { allOf: [permissionFilter, request.filter] } + : permissionFilter, + }), + }); + } + + return this.entitiesCatalog.paginatedEntities(request); + } + async removeEntityByUid( uid: string, options?: { authorizationToken?: string }, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 6735709662..a9592ddf86 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -18,6 +18,10 @@ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; +import { + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, +} from '../catalog/types'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -674,6 +678,376 @@ describe('DefaultEntitiesCatalog', () => { ); }); + describe('paginatedEntities', () => { + it.each(databases.eachSupportedId())( + 'should return paginated entities and scroll the items accordingly, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name }, + spec: { should_include_this: 'yes' }, + }; + } + + const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; + const entities: Entity[] = names.map(entityFrom); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities + .concat(notFoundEntities) + .map(e => addEntityToSearch(knex, e)), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const filter = { + key: 'spec.should_include_this', + }; + + const limit = 2; + + // initial request + const request1: PaginatedEntitiesInitialRequest = { + filter, + limit, + sortField: 'metadata.name', + }; + const response1 = await catalog.paginatedEntities(request1); + expect(response1.entities).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response1.nextCursor).toBeDefined(); + expect(response1.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // second request (forward) + const request2: PaginatedEntitiesCursorRequest = { + cursor: response1.nextCursor!, + limit, + }; + const response2 = await catalog.paginatedEntities(request2); + expect(response2.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response2.nextCursor).toBeDefined(); + expect(response2.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // third request (forward) + const request3: PaginatedEntitiesCursorRequest = { + cursor: response2.nextCursor!, + limit, + }; + const response3 = await catalog.paginatedEntities(request3); + expect(response3.entities).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response3.nextCursor).toBeDefined(); + expect(response3.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fourth request (backwards) + const request4: PaginatedEntitiesCursorRequest = { + cursor: response3.prevCursor!, + limit, + }; + const response4 = await catalog.paginatedEntities(request4); + expect(response4.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response4.nextCursor).toBeDefined(); + expect(response4.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fifth request (backwards) + const request5: PaginatedEntitiesCursorRequest = { + cursor: response4.prevCursor!, + limit, + }; + const response5 = await catalog.paginatedEntities(request5); + expect(response5.entities).toEqual([entityFrom('A'), entityFrom('B')]); + expect(response5.nextCursor).toBeDefined(); + expect(response5.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // sixth request (forward) + const request6: PaginatedEntitiesCursorRequest = { + cursor: response5.nextCursor!, + limit, + }; + const response6 = await catalog.paginatedEntities(request6); + expect(response6.entities).toEqual([entityFrom('C'), entityFrom('D')]); + expect(response6.nextCursor).toBeDefined(); + expect(response6.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh request (forward) + const request7: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit, + }; + const response7 = await catalog.paginatedEntities(request7); + expect(response7.entities).toEqual([entityFrom('E'), entityFrom('F')]); + expect(response7.nextCursor).toBeDefined(); + expect(response7.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh.2 request (forward with a different limit) + const request7bis: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit: limit + 1, + }; + const response7bis = await catalog.paginatedEntities(request7bis); + expect(response7bis.entities).toEqual([ + entityFrom('E'), + entityFrom('F'), + entityFrom('G'), + ]); + expect(response7bis.nextCursor).toBeUndefined(); + expect(response7bis.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // last request (forward) + const request8: PaginatedEntitiesCursorRequest = { + cursor: response7.nextCursor!, + limit, + }; + const response8 = await catalog.paginatedEntities(request8); + expect(response8.entities).toEqual([entityFrom('G')]); + expect(response8.nextCursor).toBeUndefined(); + expect(response8.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + }, + ); + + it.each(databases.eachSupportedId())( + 'should return paginated entities ordered in descending order and scroll the items accordingly, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name }, + spec: { should_include_this: 'yes' }, + }; + } + + const names = ['B', 'F', 'A', 'G', 'D', 'C', 'E']; + const entities: Entity[] = names.map(entityFrom); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities + .concat(notFoundEntities) + .map(e => addEntityToSearch(knex, e)), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const filter = { + key: 'spec.should_include_this', + }; + + const limit = 2; + + // initial request + const request1: PaginatedEntitiesInitialRequest = { + filter, + limit, + sortField: 'metadata.name', + sortFieldOrder: 'desc', + }; + const response1 = await catalog.paginatedEntities(request1); + expect(response1.entities).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response1.nextCursor).toBeDefined(); + expect(response1.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // second request (forward) + const request2: PaginatedEntitiesCursorRequest = { + cursor: response1.nextCursor!, + limit, + }; + const response2 = await catalog.paginatedEntities(request2); + expect(response2.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response2.nextCursor).toBeDefined(); + expect(response2.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // third request (forward) + const request3: PaginatedEntitiesCursorRequest = { + cursor: response2.nextCursor!, + limit, + }; + const response3 = await catalog.paginatedEntities(request3); + expect(response3.entities).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response3.nextCursor).toBeDefined(); + expect(response3.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fourth request (backwards) + const request4: PaginatedEntitiesCursorRequest = { + cursor: response3.prevCursor!, + limit, + }; + const response4 = await catalog.paginatedEntities(request4); + + expect(response4.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response4.nextCursor).toBeDefined(); + expect(response4.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // fifth request (backwards) + const request5: PaginatedEntitiesCursorRequest = { + cursor: response4.prevCursor!, + limit, + }; + const response5 = await catalog.paginatedEntities(request5); + expect(response5.entities).toEqual([entityFrom('G'), entityFrom('F')]); + expect(response5.nextCursor).toBeDefined(); + expect(response5.prevCursor).toBeUndefined(); + expect(response1.totalItems).toBe(names.length); + + // sixth request (forward) + const request6: PaginatedEntitiesCursorRequest = { + cursor: response5.nextCursor!, + limit, + }; + const response6 = await catalog.paginatedEntities(request6); + expect(response6.entities).toEqual([entityFrom('E'), entityFrom('D')]); + expect(response6.nextCursor).toBeDefined(); + expect(response6.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh request (forward) + const request7: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit, + }; + const response7 = await catalog.paginatedEntities(request7); + expect(response7.entities).toEqual([entityFrom('C'), entityFrom('B')]); + expect(response7.nextCursor).toBeDefined(); + expect(response7.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // seventh.2 request (forward with a different limit) + const request7bis: PaginatedEntitiesCursorRequest = { + cursor: response6.nextCursor!, + limit: limit + 1, + }; + const response7bis = await catalog.paginatedEntities(request7bis); + expect(response7bis.entities).toEqual([ + entityFrom('C'), + entityFrom('B'), + entityFrom('A'), + ]); + expect(response7bis.nextCursor).toBeUndefined(); + expect(response7bis.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + + // last request (forward) + const request8: PaginatedEntitiesCursorRequest = { + cursor: response7.nextCursor!, + limit, + }; + const response8 = await catalog.paginatedEntities(request8); + expect(response8.entities).toEqual([entityFrom('A')]); + expect(response8.nextCursor).toBeUndefined(); + expect(response8.prevCursor).toBeDefined(); + expect(response1.totalItems).toBe(names.length); + }, + ); + + it.each(databases.eachSupportedId())( + 'should filter the results when query is provided, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + function entityFrom(name: string) { + return { + apiVersion: 'a', + kind: 'k', + metadata: { name }, + spec: { should_include_this: 'yes' }, + }; + } + + const names = ['lion', 'cat', 'atcatss', 'dog', 'dogcat', 'aa', 's']; + const entities: Entity[] = names.map(entityFrom); + + const notFoundEntities: Entity[] = [ + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something' }, + spec: {}, + }, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'something else' }, + spec: {}, + }, + ]; + + await Promise.all( + entities + .concat(notFoundEntities) + .map(e => addEntityToSearch(knex, e)), + ); + + const catalog = new DefaultEntitiesCatalog(knex); + + const filter = { + key: 'spec.should_include_this', + }; + + const request: PaginatedEntitiesInitialRequest = { + filter, + limit: 100, + sortField: 'metadata.name', + query: 'cAt ', + }; + const response = await catalog.paginatedEntities(request); + expect(response.entities).toEqual([ + entityFrom('atcatss'), + entityFrom('cat'), + entityFrom('dogcat'), + ]); + expect(response.nextCursor).toBeUndefined(); + expect(response.prevCursor).toBeUndefined(); + expect(response.totalItems).toBe(3); + }, + ); + }); + describe('removeEntityByUid', () => { it.each(databases.eachSupportedId())( 'also clears parent hashes, %p', @@ -889,5 +1263,89 @@ describe('DefaultEntitiesCatalog', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'can match relations', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'one', + tags: ['java', 'rust'], + }, + spec: {}, + relations: [ + { + targetRef: 'targetRef', + type: 'ownedBy', + target: { kind: 'k', namespace: 'default', name: 'targetRef' }, + }, + ], + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'two', + tags: ['java', 'node'], + }, + spec: {}, + relations: [ + { + targetRef: 'anotherTargetRef', + type: 'ownedBy', + target: { + kind: 'k', + namespace: 'default', + name: 'anotherTargetRef', + }, + }, + { + targetRef: 'tt', + type: 'somethingelse', + target: { + kind: 'k', + namespace: 'default', + name: 'tt', + }, + }, + ], + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'three', + tags: ['go'], + }, + spec: {}, + relations: [ + { + targetRef: 'targetRef', + type: 'ownedBy', + target: { kind: 'k', namespace: 'default', name: 'targetRef' }, + }, + ], + }); + + const catalog = new DefaultEntitiesCatalog(knex); + + await expect( + catalog.facets({ + facets: ['relations.ownedBy'], + }), + ).resolves.toEqual({ + facets: { + 'relations.ownedBy': [ + { count: 1, value: 'anotherTargetRef' }, + { count: 2, value: 'targetRef' }, + ], + }, + }); + }, + ); }); }); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index d598881371..b860c16c86 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -25,6 +25,7 @@ import { Knex } from 'knex'; import { EntitiesBatchRequest, EntitiesBatchResponse, + Cursor, EntitiesCatalog, EntitiesRequest, EntitiesResponse, @@ -34,6 +35,8 @@ import { EntityFacetsResponse, EntityFilter, EntityPagination, + PaginatedEntitiesRequest, + PaginatedEntitiesResponse, } from '../catalog/types'; import { DbFinalEntitiesRow, @@ -43,8 +46,17 @@ import { DbRelationsRow, DbSearchRow, } from '../database/tables'; + import { Stitcher } from '../stitching/Stitcher'; +import { + isPaginatedEntitiesCursorRequest, + isPaginatedEntitiesInitialRequest, +} from './util'; + +const defaultSortField = 'metadata.name'; +const defaultSortFieldOrder = 'asc'; + function parsePagination(input?: EntityPagination): { limit?: number; offset?: number; @@ -97,7 +109,7 @@ function addCondition( // make a lot of sense. However, it had abysmal performance on sqlite // when datasets grew large, so we're using IN instead. const matchQuery = db('search') - .select('search.entity_id') + .select(entityIdField) .where({ key: filter.key.toLowerCase() }) .andWhere(function keyFilter() { if (filter.values) { @@ -310,6 +322,141 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { items }; } + async paginatedEntities( + request?: PaginatedEntitiesRequest, + ): Promise { + const db = this.database; + const limit = request?.limit ?? 20; + + const cursor: Omit & { sortFieldId?: string } = { + firstFieldId: '', + sortField: defaultSortField, + sortFieldOrder: defaultSortFieldOrder, + isPrevious: false, + ...parseCursorFromRequest(request), + }; + + const allowedSortFields = ['metadata.name', 'metadata.uid']; + if (!allowedSortFields.includes(cursor.sortField)) { + throw new InputError( + `Invalid sortField. Allowed values are ${allowedSortFields.join( + ', ', + )}.`, + ); + } + + const isFetchingBackwards = cursor.isPrevious; + + /** + * page number is used for quickly + * detecting the initial batch of items + * when navigating backwards without performing + * extra operations on the database. + */ + /* + const currentPage = isFetchingBackwards + ? cursor.previousPage - 1 + : cursor.previousPage + 1; + */ + + const dbQuery = db('search') + .join('final_entities', 'search.entity_id', 'final_entities.entity_id') + .where('key', cursor.sortField); + + if (cursor.filter) { + parseFilter(cursor.filter, dbQuery, db, false, 'search.entity_id'); + } + + const normalizedQueryByName = cursor.query?.trim(); + if (normalizedQueryByName) { + dbQuery.andWhereLike( + 'value', + `%${normalizedQueryByName.toLocaleLowerCase('en-US')}%`, + ); + } + + const countQuery = dbQuery.clone(); + + const isOrderingDescending = cursor.sortFieldOrder === 'desc'; + if (cursor.sortFieldId) { + dbQuery.andWhere( + 'value', + isFetchingBackwards !== isOrderingDescending ? '<' : '>', + cursor.sortFieldId, + ); + } + + dbQuery + .orderBy( + 'value', + isFetchingBackwards + ? invertOrder(cursor.sortFieldOrder) + : cursor.sortFieldOrder, + ) + // fetch an extra item for + // checking if there are more items + .limit(isFetchingBackwards ? limit : limit + 1); + + countQuery.count('search.entity_id', { as: 'count' }); + + const [rows, [{ count }]] = await Promise.all([ + dbQuery, + // for performance reasons we invoke the countQuery + // only on the first request. + // The result is then embedded into the cursor + // for subsequent requests. + typeof cursor.totalItems === 'undefined' + ? countQuery + : [{ count: cursor.totalItems }], + ]); + + const totalItems = Number(count); + + if (isFetchingBackwards) { + rows.reverse(); + } + const hasMoreResults = + limit > 0 && (isFetchingBackwards || rows.length > limit); + + // discard the extra item only when fetching forward. + if (rows.length > limit) { + rows.length -= 1; + } + + const isInitialRequest = cursor.firstFieldId === ''; + + const firstFieldId = cursor.firstFieldId || rows[0].value; + + const nextCursor = hasMoreResults + ? encodeCursor({ + ...cursor, + sortFieldId: rows[rows.length - 1].value, + firstFieldId, + isPrevious: false, + totalItems, + }) + : undefined; + + const prevCursor = + !isInitialRequest && + rows.length > 0 && + rows[0].value !== cursor.firstFieldId + ? encodeCursor({ + ...cursor, + sortFieldId: rows[0].value, + firstFieldId: cursor.firstFieldId, + isPrevious: true, + totalItems, + }) + : undefined; + + const entities = rows + .map(e => JSON.parse(e.final_entity!)) + .map(e => (request?.fields ? request.fields(e) : e)); + + return { entities, prevCursor, nextCursor, totalItems }; + } + async removeEntityByUid(uid: string): Promise { const dbConfig = this.database.client.config; @@ -490,3 +637,37 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { return { facets }; } } + +function encodeCursor(cursor: Cursor) { + const json = JSON.stringify(cursor); + return Buffer.from(json, 'utf8').toString('base64'); +} + +function parseCursorFromRequest( + request?: PaginatedEntitiesRequest, +): Partial { + if (isPaginatedEntitiesInitialRequest(request)) { + const { + filter, + sortField = defaultSortField, + sortFieldOrder = defaultSortFieldOrder, + query, + } = request; + return { filter, sortField, sortFieldOrder, query }; + } + if (isPaginatedEntitiesCursorRequest(request)) { + try { + const json = Buffer.from(request.cursor, 'base64').toString('utf8'); + const cursor = JSON.parse(json); + // TODO(vinzscam): validate the shit + return cursor as unknown as Cursor; + } catch { + throw new InputError('Malformed cursor, could not be parsed'); + } + } + return {}; +} + +function invertOrder(order: Cursor['sortFieldOrder']) { + return order === 'asc' ? 'desc' : 'asc'; +} diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 6379d65043..85fede25ca 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -52,6 +52,7 @@ describe('createRouter readonly disabled', () => { removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), + paginatedEntities: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -135,6 +136,80 @@ describe('createRouter readonly disabled', () => { }); }); + describe('GET /v2beta1/entities', () => { + it('happy path: lists entities', async () => { + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entities, + totalItems: 100, + nextCursor: 'something', + }); + + const response = await request(app).get('/v2beta1/entities'); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + entities, + totalItems: 100, + nextCursor: 'something', + }); + }); + + it('parses initial request', async () => { + entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entities: [], + totalItems: 0, + }); + const response = await request(app).get( + '/v2beta1/entities?filter=a=1,a=2,b=3&filter=c=4', + ); + + expect(response.status).toEqual(200); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ + filter: { + anyOf: [ + { + allOf: [ + { key: 'a', values: ['1', '2'] }, + { key: 'b', values: ['3'] }, + ], + }, + { allOf: [{ key: 'c', values: ['4'] }] }, + ], + }, + }); + }); + + it('parses cursor request', async () => { + const entities: Entity[] = [ + { apiVersion: 'a', kind: 'b', metadata: { name: 'n' } }, + ]; + + entitiesCatalog.paginatedEntities.mockResolvedValueOnce({ + entities, + totalItems: 100, + nextCursor: 'next', + }); + + const response = await request(app).get( + '/v2beta1/entities?cursor=something', + ); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledTimes(1); + expect(entitiesCatalog.paginatedEntities).toHaveBeenCalledWith({ + cursor: 'something', + }); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + entities, + totalItems: 100, + nextCursor: 'next', + }); + }); + }); + describe('GET /entities/by-uid/:uid', () => { it('can fetch entity by uid', async () => { const entity: Entity = { diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 62a0f970c9..a7f5373a03 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -39,6 +39,7 @@ import { parseEntityFilterParams, parseEntityPaginationParams, parseEntityTransformParams, + parsePaginatedEntitiesParams, } from './request'; import { parseEntityFacetParams } from './request/parseEntityFacetParams'; import { parseEntityOrderParams } from './request/parseEntityOrderParams'; @@ -130,6 +131,14 @@ export async function createRouter( // TODO(freben): encode the pageInfo in the response res.json(entities); }) + .get('/v2beta1/entities', async (req, res) => { + const response = await entitiesCatalog.paginatedEntities({ + ...parsePaginatedEntitiesParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); + + res.json(response); + }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; const { entities } = await entitiesCatalog.entities({ diff --git a/plugins/catalog-backend/src/service/request/index.ts b/plugins/catalog-backend/src/service/request/index.ts index 0b09355e72..5a55c11ca0 100644 --- a/plugins/catalog-backend/src/service/request/index.ts +++ b/plugins/catalog-backend/src/service/request/index.ts @@ -19,3 +19,4 @@ export { basicEntityFilter } from './basicEntityFilter'; export { parseEntityFilterParams } from './parseEntityFilterParams'; export { parseEntityPaginationParams } from './parseEntityPaginationParams'; export { parseEntityTransformParams } from './parseEntityTransformParams'; +export { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams'; diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts new file mode 100644 index 0000000000..838b931888 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.test.ts @@ -0,0 +1,132 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, +} from '../../catalog/types'; +import { parsePaginatedEntitiesParams } from './parsePaginatedEntitiesParams'; + +describe('parsePaginatedEntitiesParams', () => { + describe('initial request', () => { + it('should parse all the defined params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + filter: ['a=1', 'b=2'], + sortField: 'sortField', + sortFieldOrder: 'desc', + query: 'query', + }; + const parsedObj = parsePaginatedEntitiesParams( + validRequest, + ) as PaginatedEntitiesInitialRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.sortField).toBe('sortField'); + expect(parsedObj.sortFieldOrder).toBe('desc'); + expect(parsedObj.filter).toBeDefined(); + expect(parsedObj.query).toBe('query'); + expect(parsedObj).not.toHaveProperty('authorizationToken'); + expect(parsedObj).not.toHaveProperty('cursor'); + }); + it('should ignore optional params', () => { + const parsedObj = parsePaginatedEntitiesParams( + {}, + ) as PaginatedEntitiesInitialRequest; + expect(parsedObj.limit).toBeUndefined(); + expect(parsedObj.fields).toBeUndefined(); + expect(parsedObj.sortField).toBeUndefined(); + expect(parsedObj.sortFieldOrder).toBeUndefined(); + expect(parsedObj.filter).toBeUndefined(); + expect(parsedObj.query).toBeUndefined(); + expect(parsedObj).not.toHaveProperty('authorizationToken'); + expect(parsedObj).not.toHaveProperty('cursor'); + }); + + it.each([ + { + limit: 'asd', + }, + { filter: 3 }, + { sortField: [] }, + { sortFieldOrder: 'something' }, + { fields: [4] }, + { query: [] }, + ])('should throw if some parameter is not valid %p', params => { + expect(() => parsePaginatedEntitiesParams(params)).toThrow(); + }); + }); + + describe('cursor request', () => { + it('should parse all the defined params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + cursor: 'cursor', + }; + const parsedObj = parsePaginatedEntitiesParams( + validRequest, + ) as PaginatedEntitiesCursorRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.cursor).toBe('cursor'); + }); + + it('should ignore unknown params', () => { + const validRequest = { + authorizationToken: 'to_not_be_returned', + fields: ['kind'], + limit: '3', + cursor: 'cursor', + filter: ['a=1', 'b=2'], + sortField: 'sortField', + sortFieldOrder: 'desc', + query: 'query', + }; + const parsedObj = parsePaginatedEntitiesParams( + validRequest, + ) as PaginatedEntitiesCursorRequest; + expect(parsedObj.limit).toBe(3); + expect(parsedObj.fields).toBeDefined(); + expect(parsedObj.cursor).toBe('cursor'); + expect(parsedObj).not.toHaveProperty('filter'); + expect(parsedObj).not.toHaveProperty('sortField'); + expect(parsedObj).not.toHaveProperty('sortFieldOrder'); + expect(parsedObj).not.toHaveProperty('query'); + }); + + it('should ignore optional params', () => { + const parsedObj = parsePaginatedEntitiesParams( + {}, + ) as PaginatedEntitiesCursorRequest; + expect(parsedObj.limit).toBeUndefined(); + expect(parsedObj.fields).toBeUndefined(); + }); + + it.each([ + { + limit: 'asd', + }, + { cursor: [] }, + { fields: [4] }, + ])('should throw if some parameter is not valid %p', params => { + expect(() => parsePaginatedEntitiesParams(params)).toThrow(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts new file mode 100644 index 0000000000..84d519ec0b --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parsePaginatedEntitiesParams.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, + PaginatedEntitiesRequest, +} from '../../catalog/types'; +import { parseIntegerParam, parseStringParam } from './common'; +import { parseEntityFilterParams } from './parseEntityFilterParams'; +import { parseEntityTransformParams } from './parseEntityTransformParams'; + +export function parsePaginatedEntitiesParams( + params: Record, +): Omit { + const fields = parseEntityTransformParams(params); + const limit = parseIntegerParam(params.limit, 'limit'); + const cursor = parseStringParam(params.cursor, 'cursor'); + if (cursor) { + const response: Omit = + { + cursor, + fields, + limit, + }; + return response; + } + + const filter = parseEntityFilterParams(params); + const query = parseStringParam(params.query, 'query'); + const sortField = parseStringParam(params.sortField, 'sortField'); + const sortFieldOrder = parseSortFieldOrder( + params.sortFieldOrder, + 'sortFieldOrder', + ); + + const response: Omit = + { + fields, + filter, + limit, + sortField, + sortFieldOrder, + query, + }; + + return response; +} + +function parseSortFieldOrder(sortFieldOrder: unknown, ctx: string) { + const isSortFieldOrder = + sortFieldOrder === undefined || + sortFieldOrder === 'asc' || + sortFieldOrder === 'desc'; + + if (isSortFieldOrder) { + return sortFieldOrder; + } + throw new InputError(`Invalid ${ctx}, not asc or desc`); +} diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index 9a914aadcb..bb86654565 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -18,6 +18,11 @@ import { InputError, NotAllowedError } from '@backstage/errors'; import { Request } from 'express'; import lodash from 'lodash'; import { z } from 'zod'; +import { + EntitiesRequest, + PaginatedEntitiesCursorRequest, + PaginatedEntitiesInitialRequest, +} from '../catalog/types'; export async function requireRequestBody(req: Request): Promise { const contentType = req.header('content-type'); @@ -65,3 +70,23 @@ export function disallowReadonlyMode(readonly: boolean) { throw new NotAllowedError('This operation not allowed in readonly mode'); } } + +export function isPaginatedEntitiesInitialRequest( + input: EntitiesRequest | undefined, +): input is PaginatedEntitiesInitialRequest { + if (!input) { + return false; + } + // TODO(vinzscam) expand this + return !isPaginatedEntitiesCursorRequest(input); +} + +export function isPaginatedEntitiesCursorRequest( + input: EntitiesRequest | undefined, +): input is PaginatedEntitiesCursorRequest { + if (!input) { + return false; + } + // TODO(vinzscam) expand this + return input?.hasOwnProperty('cursor') ?? false; +}