From 6a1e7fbe52b7e66b31dcee04b83dcb0ad550b198 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Thu, 23 Jun 2022 18:46:25 +0200 Subject: [PATCH] CatalogClient: add getPaginatedEntities method Signed-off-by: Vincenzo Scamporlino --- .../catalog-client/src/CatalogClient.test.ts | 194 ++++++++++++++++++ packages/catalog-client/src/CatalogClient.ts | 163 ++++++++++----- packages/catalog-client/src/index.ts | 1 + packages/catalog-client/src/types/api.ts | 110 +++++++++- packages/catalog-client/src/types/index.ts | 3 + packages/catalog-client/src/utils.ts | 36 ++++ 6 files changed, 452 insertions(+), 55 deletions(-) create mode 100644 packages/catalog-client/src/utils.ts diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 2e9d2a0679..8a449c730e 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -249,6 +249,200 @@ describe('CatalogClient', () => { }); }); + describe('getPaginatedEntities', () => { + const defaultServiceResponse = { + entities: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test1', + namespace: 'test1', + }, + }, + ], + totalItems: 10, + nextCursor: 'next', + prevCursor: 'prev', + }; + + const defaultClientResponse = { + entities: [ + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }, + { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test1', + namespace: 'test1', + }, + }, + ], + totalItems: 10, + next: jest.fn(), + prev: jest.fn(), + }; + + beforeEach(() => { + server.use( + rest.get(`${mockBaseUrl}/v2beta1/entities`, (_, res, ctx) => { + return res(ctx.json(defaultServiceResponse)); + }), + ); + }); + + it('should fetch entities from correct endpoint', async () => { + const response = await client.getPaginatedEntities?.({}, { token }); + expect(response?.entities).toEqual(defaultClientResponse.entities); + expect(response?.totalItems).toEqual(defaultClientResponse.totalItems); + expect(response?.next).toBeDefined(); + expect(response?.prev).toBeDefined(); + }); + + it('builds multiple entity search filters properly', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ entities: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + const response = await client.getPaginatedEntities?.( + { + filter: [ + { + a: '1', + b: ['2', '3'], + รถ: '=', + }, + { + a: '2', + }, + { + c: CATALOG_FILTER_EXISTS, + }, + ], + }, + { token }, + ); + + expect(response).toEqual({ entities: [], totalItems: 0 }); + expect(mockedEndpoint).toHaveBeenCalledTimes(1); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?filter=a=1,b=2,b=3,%C3%B6=%3D&filter=a=2&filter=c', + ); + }); + + it('should send query params correctly on initial request', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ entities: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + await client.getPaginatedEntities?.({ + fields: ['a', 'b'], + limit: 100, + query: 'query', + sortField: 'metadata.name', + sortFieldOrder: 'asc', + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?limit=100&sortField=metadata.name&sortFieldOrder=asc&fields=a,b&query=query', + ); + }); + + it('should ignore initial query params if cursor is passed', async () => { + const mockedEndpoint = jest + .fn() + .mockImplementation((_req, res, ctx) => + res(ctx.json({ entities: [], totalItems: 0 })), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + await client.getPaginatedEntities?.({ + fields: ['a', 'b'], + limit: 100, + query: 'query', + sortField: 'metadata.name', + sortFieldOrder: 'asc', + cursor: 'cursor', + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe( + '?cursor=cursor&limit=100&fields=a,b', + ); + }); + + it('should return paginated functions if next and prev cursors are present', async () => { + const mockedEndpoint = jest.fn().mockImplementation((_req, res, ctx) => + res( + ctx.json({ + entities: [ + { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'e1', + }, + }, + { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'e2', + }, + }, + ], + nextCursor: 'nextcursor', + prevCursor: 'prevcursor', + totalItems: 100, + }), + ), + ); + + server.use(rest.get(`${mockBaseUrl}/v2beta1/entities`, mockedEndpoint)); + + const response = await client.getPaginatedEntities?.({ + limit: 2, + }); + expect(mockedEndpoint.mock.calls[0][0].url.search).toBe('?limit=2'); + + expect(response?.next).toBeDefined(); + expect(response?.prev).toBeDefined(); + + await response?.next?.(); + expect(mockedEndpoint.mock.calls[1][0].url.search).toBe( + '?cursor=nextcursor', + ); + + await response?.prev?.(); + expect(mockedEndpoint.mock.calls[2][0].url.search).toBe( + '?cursor=prevcursor', + ); + }); + }); + describe('getEntityByRef', () => { const existingEntity: Entity = { apiVersion: 'v1', diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index af4262b842..03fdd600e9 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -39,9 +39,14 @@ import { ValidateEntityResponse, GetEntitiesByRefsRequest, GetEntitiesByRefsResponse, + GetPaginatedEntitiesRequest, + GetPaginatedEntitiesResponse, + EntitiesFilter, + GetPaginatedEntitiesCursorRequest, } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; +import { isPaginatedEntitiesInitialRequest } from './utils'; /** * A frontend and backend compatible client for communicating with the Backstage @@ -107,30 +112,7 @@ export class CatalogClient implements CatalogApi { limit, after, } = request ?? {}; - const params: string[] = []; - - // filter param can occur multiple times, for example - // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' - // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters - // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of [filter].flat()) { - const filterParts: string[] = []; - for (const [key, value] of Object.entries(filterItem)) { - for (const v of [value].flat()) { - if (v === CATALOG_FILTER_EXISTS) { - filterParts.push(encodeURIComponent(key)); - } else if (typeof v === 'string') { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); - } - } - } - - if (filterParts.length) { - params.push(`filter=${filterParts.join(',')}`); - } - } + const params = this.getParams(filter); if (fields.length) { params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); @@ -223,6 +205,70 @@ export class CatalogClient implements CatalogApi { return { items }; } + /** + * {@inheritdoc CatalogApi.getPaginatedEntities} + */ + async getPaginatedEntities?( + request?: GetPaginatedEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise { + const params: string[] = []; + + if (isPaginatedEntitiesInitialRequest(request)) { + const { + fields = [], + filter, + limit, + sortField, + sortFieldOrder, + query, + } = request ?? {}; + params.push(...this.getParams(filter)); + + if (limit !== undefined) { + params.push(`limit=${limit}`); + } + if (sortField !== undefined) { + params.push(`sortField=${sortField}`); + } + if (sortFieldOrder !== undefined) { + params.push(`sortFieldOrder=${sortFieldOrder}`); + } + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + + const normalizedQuery = query?.trim(); + if (normalizedQuery) { + params.push(`query=${normalizedQuery}`); + } + } else { + const { fields = [], limit, cursor } = request; + + params.push(`cursor=${cursor}`); + if (limit !== undefined) { + params.push(`limit=${limit}`); + } + if (fields.length) { + params.push(`fields=${fields.map(encodeURIComponent).join(',')}`); + } + } + + const query = params.length ? `?${params.join('&')}` : ''; + const { entities, totalItems, nextCursor, prevCursor } = + await this.requestRequired<{ + entities: Entity[]; + totalItems: number; + nextCursor?: string; + prevCursor?: string; + }>('GET', `/v2beta1/entities${query}`, options); + + const next = this.getEntitiesFromCursor(nextCursor, options); + const prev = this.getEntitiesFromCursor(prevCursor, options); + + return { entities, totalItems, next, prev }; + } + /** * {@inheritdoc CatalogApi.getEntityByRef} */ @@ -290,30 +336,7 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise { const { filter = [], facets } = request; - const params: string[] = []; - - // filter param can occur multiple times, for example - // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' - // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters - // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of [filter].flat()) { - const filterParts: string[] = []; - for (const [key, value] of Object.entries(filterItem)) { - for (const v of [value].flat()) { - if (v === CATALOG_FILTER_EXISTS) { - filterParts.push(encodeURIComponent(key)); - } else if (typeof v === 'string') { - filterParts.push( - `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, - ); - } - } - } - - if (filterParts.length) { - params.push(`filter=${filterParts.join(',')}`); - } - } + const params = this.getParams(filter); for (const facet of facets) { params.push(`facet=${encodeURIComponent(facet)}`); @@ -466,11 +489,11 @@ export class CatalogClient implements CatalogApi { } } - private async requestRequired( + private async requestRequired( method: string, path: string, options?: CatalogRequestOptions, - ): Promise { + ): Promise { const url = `${await this.discoveryApi.getBaseUrl('catalog')}${path}`; const headers: Record = options?.token ? { Authorization: `Bearer ${options.token}` } @@ -481,7 +504,7 @@ export class CatalogClient implements CatalogApi { throw await ResponseError.fromResponse(response); } - return await response.json(); + return response.json(); } private async requestOptional( @@ -504,4 +527,42 @@ export class CatalogClient implements CatalogApi { return await response.json(); } + + private getParams(filter: EntitiesFilter = []) { + const params: string[] = []; + // filter param can occur multiple times, for example + // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' + // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters + // the "inner array" defined within a `filter` param corresponds to "allOf" filters + for (const filterItem of [filter].flat()) { + const filterParts: string[] = []; + for (const [key, value] of Object.entries(filterItem)) { + for (const v of [value].flat()) { + if (v === CATALOG_FILTER_EXISTS) { + filterParts.push(encodeURIComponent(key)); + } else if (typeof v === 'string') { + filterParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, + ); + } + } + } + + if (filterParts.length) { + params.push(`filter=${filterParts.join(',')}`); + } + } + return params; + } + + private getEntitiesFromCursor( + cursor: string | undefined, + options?: CatalogRequestOptions, + ) { + if (!cursor) { + return undefined; + } + return (request: Omit = {}) => + this.getPaginatedEntities!({ ...request, cursor }, options); + } } diff --git a/packages/catalog-client/src/index.ts b/packages/catalog-client/src/index.ts index 2e81942c2f..f0b10a3eef 100644 --- a/packages/catalog-client/src/index.ts +++ b/packages/catalog-client/src/index.ts @@ -22,3 +22,4 @@ export { CatalogClient } from './CatalogClient'; export * from './types'; +export * from './utils'; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index d662a9d106..1390a0576e 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -287,10 +287,7 @@ export interface GetEntityFacetsRequest { * (exported from this package), which means that you assert on the existence * of that key, no matter what its value is. */ - filter?: - | Record[] - | Record - | undefined; + filter?: EntitiesFilter; /** * Dot separated paths for the facets to extract from each entity. * @@ -359,6 +356,11 @@ export type AddLocationRequest = { dryRun?: boolean; }; +export type EntitiesFilter = + | Record[] + | Record + | undefined; + /** * The response type for {@link CatalogClient.addLocation}. * @@ -380,6 +382,69 @@ export type ValidateEntityResponse = | { valid: true } | { valid: false; errors: SerializedError[] }; +/** + * The request type for {@link CatalogClient.getPaginatedEntities}. + * + * @alpha + */ +export type GetPaginatedEntitiesRequest = + | GetPaginatedEntitiesInitialRequest + | GetPaginatedEntitiesCursorRequest; + +/** + * A request type for {@link CatalogClient.getPaginatedEntities}. + * The method takes this type in an initial pagination request, + * when requesting the first batch of entities. + * + * The properties filter, sortField, query and sortFieldOrder, are going + * to be immutable for the entire lifecycle of the following requests. + * + * @alpha + */ +export type GetPaginatedEntitiesInitialRequest = + | { + fields?: string[]; + limit?: number; + filter?: EntitiesFilter; + sortField?: string; + query?: string; + sortFieldOrder?: 'asc' | 'desc'; + } + | undefined; + +/** + * A request type for {@link CatalogClient.getPaginatedEntities}. + * The method takes this type in a pagination request, following + * the initial request. + * + * @alpha + */ +export type GetPaginatedEntitiesCursorRequest = { + fields?: string[]; + limit?: number; + cursor: string; +}; + +/** + * The response type for {@link CatalogClient.getPaginatedEntities}. + * + * @alpha + */ +export type GetPaginatedEntitiesResponse = { + /* The list of entities for the current request */ + entities: Entity[]; + /* The number of entities among all the requests */ + totalItems: number; + /* A method returning a promise containing the next batch of entities. */ + next?( + request?: Omit, + ): Promise; + /* A method returning a promise containing the previous batch of entities. */ + prev?( + request?: Omit, + ): Promise; +}; + /** * A client for interacting with the Backstage software catalog through its API. * @@ -414,6 +479,43 @@ export interface CatalogApi { options?: CatalogRequestOptions, ): Promise; + /** + * Gets paginated entities from the catalog. + * The method takes + * @alpha + * @remarks + * + * Example: + * + * const response = await catalogClient.getPaginatedEntities({ + * filter: [{ kind: 'group' }], + * limit: 20, + * query: 'A', + * sortField: 'metadata.name', + * sortFieldOrder: 'asc' + * }); + * + * this will match all entities of type group having a name starting + * with 'A', ordered by name ascending. + * + * The response will contain a maximum of 20 entities. In case + * more than 20 entities exist, the response will contain a next function, invocable + * using: + * + * const secondBatchResponse = await response.next({ limit: 20 }); + * + * secondBatchResponse will contain the next batch of maximum 20 entities, together + * with a prev function useful for navigating backwards and a next function + * in case more entities matching the filters of the first request are present in the catalog. + * + * @param request - Request parameters + * @param options - Additional options + */ + getPaginatedEntities?( + request?: GetPaginatedEntitiesRequest, + options?: CatalogRequestOptions, + ): Promise; + /** * Gets entity ancestor information, i.e. the hierarchy of parent entities * whose processing resulted in a given entity appearing in the catalog. diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 81b985eb42..7da07979fc 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -33,5 +33,8 @@ export type { GetEntityFacetsResponse, Location, ValidateEntityResponse, + GetPaginatedEntitiesCursorRequest, + GetPaginatedEntitiesInitialRequest, + GetPaginatedEntitiesResponse, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-client/src/utils.ts b/packages/catalog-client/src/utils.ts new file mode 100644 index 0000000000..5a26053fb9 --- /dev/null +++ b/packages/catalog-client/src/utils.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + GetPaginatedEntitiesCursorRequest, + GetPaginatedEntitiesInitialRequest, + GetPaginatedEntitiesRequest, +} from './types/api'; + +export function isPaginatedEntitiesInitialRequest( + request: GetPaginatedEntitiesInitialRequest, +): request is GetPaginatedEntitiesInitialRequest { + if (!request) { + return true; + } + return !(request as GetPaginatedEntitiesCursorRequest).cursor; +} + +export function isPaginatedEntitiesCursorRequest( + request: GetPaginatedEntitiesRequest, +): request is GetPaginatedEntitiesCursorRequest { + return !isPaginatedEntitiesInitialRequest(request); +}