From 00d90b520a954d84c2f049272b9281176382201c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 14 Nov 2022 17:02:24 +0100 Subject: [PATCH] implement by-refs batch fetching in the catalog client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/chilly-ads-lay.md | 5 + packages/catalog-client/api-report.md | 34 +++- .../catalog-client/src/CatalogClient.test.ts | 29 +++ packages/catalog-client/src/CatalogClient.ts | 36 ++++ packages/catalog-client/src/types/api.ts | 188 ++++++++++++------ packages/catalog-client/src/types/index.ts | 6 +- 6 files changed, 234 insertions(+), 64 deletions(-) create mode 100644 .changeset/chilly-ads-lay.md diff --git a/.changeset/chilly-ads-lay.md b/.changeset/chilly-ads-lay.md new file mode 100644 index 0000000000..844eaa8dc7 --- /dev/null +++ b/.changeset/chilly-ads-lay.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': minor +--- + +**BREAKING PRODUCERS**: Added a new `getEntitiesByRefs` endpoint to `CatalogApi`, for efficient batch fetching of entities by ref. diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 85fc389af7..2f76be79e2 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -34,6 +34,10 @@ export interface CatalogApi { request?: GetEntitiesRequest, options?: CatalogRequestOptions, ): Promise; + getEntitiesByRefs( + request: GetEntitiesByRefsRequest, + options?: CatalogRequestOptions, + ): Promise; getEntityAncestors( request: GetEntityAncestorsRequest, options?: CatalogRequestOptions, @@ -91,6 +95,10 @@ export class CatalogClient implements CatalogApi { request?: GetEntitiesRequest, options?: CatalogRequestOptions, ): Promise; + getEntitiesByRefs( + request: GetEntitiesByRefsRequest, + options?: CatalogRequestOptions, + ): Promise; getEntityAncestors( request: GetEntityAncestorsRequest, options?: CatalogRequestOptions, @@ -145,14 +153,30 @@ export interface CatalogRequestOptions { export const ENTITY_STATUS_CATALOG_PROCESSING_TYPE = 'backstage.io/catalog-processing'; +// @public +export type EntityFieldsQuery = string[]; + +// @public +export type EntityFilterQuery = + | Record[] + | Record; + +// @public +export interface GetEntitiesByRefsRequest { + entityRefs: string[]; + fields?: EntityFieldsQuery | undefined; +} + +// @public +export interface GetEntitiesByRefsResponse { + items: Array; +} + // @public export interface GetEntitiesRequest { after?: string; - fields?: string[] | undefined; - filter?: - | Record[] - | Record - | undefined; + fields?: EntityFieldsQuery; + filter?: EntityFilterQuery; limit?: number; offset?: number; } diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 5a344f49a3..1206d557c1 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -195,6 +195,35 @@ describe('CatalogClient', () => { }); }); + describe('getEntitiesByRefs', () => { + it('encodes and decodes the query correctly', async () => { + const entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'Test2', + namespace: 'test1', + }, + }; + server.use( + rest.post(`${mockBaseUrl}/entities/by-refs`, async (req, res, ctx) => { + expect(req.url.searchParams.get('fields')).toBe('a,b'); + await expect(req.json()).resolves.toEqual({ + entityRefs: ['k:n/a', 'k:n/b'], + }); + return res(ctx.json({ items: [entity, null] })); + }), + ); + + const response = await client.getEntitiesByRefs( + { entityRefs: ['k:n/a', 'k:n/b'], fields: ['a', 'b'] }, + { token }, + ); + + expect(response).toEqual({ items: [entity, null] }); + }); + }); + describe('getEntityByRef', () => { const existingEntity: Entity = { apiVersion: 'v1', diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index d67392d80d..d15f382ddd 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -37,6 +37,8 @@ import { GetEntityFacetsRequest, GetEntityFacetsResponse, ValidateEntityResponse, + GetEntitiesByRefsRequest, + GetEntitiesByRefsResponse, } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; @@ -169,6 +171,40 @@ export class CatalogClient implements CatalogApi { return { items: entities.sort(refCompare) }; } + /** + * {@inheritdoc CatalogApi.getEntitiesByRefs} + */ + async getEntitiesByRefs( + request: GetEntitiesByRefsRequest, + options?: CatalogRequestOptions, + ): Promise { + const params: string[] = []; + if (request.fields?.length) { + params.push(`fields=${request.fields.map(encodeURIComponent).join(',')}`); + } + + const baseUrl = await this.discoveryApi.getBaseUrl('catalog'); + const query = params.length ? `?${params.join('&')}` : ''; + const url = `${baseUrl}/entities/by-refs${query}`; + + const response = await this.fetchApi.fetch(url, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'POST', + body: JSON.stringify({ entityRefs: request.entityRefs }), + }); + + if (!response.ok) { + throw await ResponseError.fromResponse(response); + } + + const { items } = await response.json(); + + return { items }; + } + /** * {@inheritdoc CatalogApi.getEntityByRef} */ diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index d3d02022f3..771adaaeee 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -29,6 +29,75 @@ export const CATALOG_FILTER_EXISTS = Symbol.for( 'CATALOG_FILTER_EXISTS_0e15b590c0b343a2bae3e787e84c2111', ); +/** + * A key-value based filter expression for entities. + * + * @remarks + * + * Each key of a record is a dot-separated path into the entity structure, e.g. + * `metadata.name`. + * + * The values are literal values to match against. As a value you can also pass + * in the symbol `CATALOG_FILTER_EXISTS` (exported from this package), which + * means that you assert on the existence of that key, no matter what its value + * is. + * + * All matching of keys and values is case insensitive. + * + * If multiple filter sets are given as an array, then there is effectively an + * OR between each filter set. + * + * Within one filter set, there is effectively an AND between the various keys. + * + * Within one key, if there are more than one value, then there is effectively + * an OR between them. + * + * Example: For an input of + * + * ``` + * [ + * { kind: ['API', 'Component'] }, + * { 'metadata.name': 'a', 'metadata.namespace': 'b' } + * ] + * ``` + * + * This effectively means + * + * ``` + * (kind = EITHER 'API' OR 'Component') + * OR + * (metadata.name = 'a' AND metadata.namespace = 'b' ) + * ``` + * + * @public + */ +export type EntityFilterQuery = + | Record[] + | Record; + +/** + * A set of dot-separated paths into an entity's keys, showing what parts of an + * entity to include in a response, and excluding all others. + * + * @remarks + * + * Example: For an input of `['kind', 'metadata.annotations']`, then response + * objects will be shaped like + * + * ``` + * { + * "kind": "Component", + * "metadata": { + * "annotations": { + * "foo": "bar" + * } + * } + * } + * ``` + * @public + */ +export type EntityFieldsQuery = string[]; + /** * The request type for {@link CatalogClient.getEntities}. * @@ -36,67 +105,14 @@ export const CATALOG_FILTER_EXISTS = Symbol.for( */ export interface GetEntitiesRequest { /** - * If given, return only entities that match the given patterns. - * - * @remarks - * - * If multiple filter sets are given as an array, then there is effectively an - * OR between each filter set. - * - * Within one filter set, there is effectively an AND between the various - * keys. - * - * Within one key, if there are more than one value, then there is effectively - * an OR between them. - * - * Example: For an input of - * - * ``` - * [ - * { kind: ['API', 'Component'] }, - * { 'metadata.name': 'a', 'metadata.namespace': 'b' } - * ] - * ``` - * - * This effectively means - * - * ``` - * (kind = EITHER 'API' OR 'Component') - * OR - * (metadata.name = 'a' AND metadata.namespace = 'b' ) - * ``` - * - * Each key is a dot separated path in each object. - * - * As a value you can also pass in the symbol `CATALOG_FILTER_EXISTS` - * (exported from this package), which means that you assert on the existence - * of that key, no matter what its value is. + * If given, return only entities that match the given filter. */ - filter?: - | Record[] - | Record - | undefined; + filter?: EntityFilterQuery; /** - * If given, return only the parts of each entity that match those dot - * separated paths in each object. - * - * @remarks - * - * Example: For an input of `['kind', 'metadata.annotations']`, then response - * objects will be shaped like - * - * ``` - * { - * "kind": "Component", - * "metadata": { - * "annotations": { - * "foo": "bar" - * } - * } - * } - * ``` + * If given, return only the parts of each entity that match the field + * declarations. */ - fields?: string[] | undefined; + fields?: EntityFieldsQuery; /** * If given, skips over the first N items in the result set. */ @@ -121,6 +137,45 @@ export interface GetEntitiesResponse { items: Entity[]; } +/** + * The request type for {@link CatalogClient.getEntitiesByRefs}. + * + * @public + */ +export interface GetEntitiesByRefsRequest { + /** + * The list of entity refs to fetch. + * + * @remarks + * + * The returned list of entities will be in the same order as the refs, and + * null will be returned in those positions that were not found. + */ + entityRefs: string[]; + /** + * If given, return only the parts of each entity that match the field + * declarations. + */ + fields?: EntityFieldsQuery | undefined; +} + +/** + * The response type for {@link CatalogClient.getEntitiesByRefs}. + * + * @public + */ +export interface GetEntitiesByRefsResponse { + /** + * The returned list of entities. + * + * @remarks + * + * The list will be in the same order as the refs given in the request, and + * null will be returned in those positions that were not found. + */ + items: Array; +} + /** * The request type for {@link CatalogClient.getEntityAncestors}. * @@ -296,6 +351,23 @@ export interface CatalogApi { options?: CatalogRequestOptions, ): Promise; + /** + * Gets a batch of entities, by their entity refs. + * + * @remarks + * + * The output list of entities is of the same size and in the same order as + * the requested list of entity refs. Entries that are not found are returned + * as null. + * + * @param request - Request parameters + * @param options - Additional options + */ + getEntitiesByRefs( + request: GetEntitiesByRefsRequest, + options?: CatalogRequestOptions, + ): Promise; + /** * Gets entity ancestor information, i.e. the hierarchy of parent entities * whose processing resulted in a given entity appearing in the catalog. diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 86f7b18ab0..5ac058e349 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -20,13 +20,17 @@ export type { AddLocationResponse, CatalogApi, CatalogRequestOptions, + EntityFieldsQuery, + EntityFilterQuery, + GetEntitiesByRefsRequest, + GetEntitiesByRefsResponse, GetEntitiesRequest, GetEntitiesResponse, GetEntityAncestorsRequest, GetEntityAncestorsResponse, - Location, GetEntityFacetsRequest, GetEntityFacetsResponse, + Location, ValidateEntityResponse, } from './api'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status';