diff --git a/.changeset/fast-lies-remain.md b/.changeset/fast-lies-remain.md new file mode 100644 index 0000000000..74be117920 --- /dev/null +++ b/.changeset/fast-lies-remain.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added new `POST /entities/by-refs` endpoint, which allows you to efficiently +batch-fetch entities by their entity ref. This can be useful e.g. in graphql +resolvers or similar contexts where you need to fetch many entities at the same +time. diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 34060814aa..84673a3900 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -80,6 +80,36 @@ export type EntitiesResponse = { pageInfo: PageInfo; }; +/** + * A request for a batch of entities. + */ +export interface EntitiesBatchRequest { + /** + * The refs for which to fetch entities. + */ + entityRefs: string[]; + /** + * Any additional filters to apply in the selection of the entities. + */ + filter?: EntityFilter; + /** + * Strips out only the parts of the entity bodies to include in the response. + */ + fields?: (entity: Entity) => Entity; + /** + * The optional token that authorizes the action. + */ + authorizationToken?: string; +} + +export interface EntitiesBatchResponse { + /** + * The list of entities, in the same order as the refs in the request. Entries + * that are null signify that no entity existed with that ref. + */ + items: Array; +} + export type EntityAncestryResponse = { rootEntityRef: string; items: Array<{ @@ -130,6 +160,11 @@ export interface EntitiesCatalog { */ entities(request?: EntitiesRequest): Promise; + /** + * Fetches a batch of entities. + */ + entitiesBatch(request: EntitiesBatchRequest): Promise; + /** * Removes a single entity. * diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 66764dc89f..cbaf876071 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -24,6 +24,7 @@ import { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog'; describe('AuthorizedEntitiesCatalog', () => { const fakeCatalog = { entities: jest.fn(), + entitiesBatch: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), @@ -92,6 +93,67 @@ describe('AuthorizedEntitiesCatalog', () => { }); }); + describe('entitiesBatch', () => { + it('returns empty response on DENY', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.DENY }, + ]); + const catalog = createCatalog(); + + await expect( + catalog.entitiesBatch({ + entityRefs: ['component:default/component-a'], + authorizationToken: 'abcd', + }), + ).resolves.toEqual({ + items: [null], + }); + + expect(fakeCatalog.entitiesBatch).not.toHaveBeenCalled(); + }); + + it('calls underlying catalog method with correct filter on CONDITIONAL', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { + result: AuthorizeResult.CONDITIONAL, + conditions: { + rule: 'IS_ENTITY_KIND', + params: { kinds: ['b'] }, + }, + }, + ]); + const catalog = createCatalog(isEntityKind); + + await catalog.entitiesBatch({ + entityRefs: ['component:default/component-a'], + authorizationToken: 'abcd', + }); + + expect(fakeCatalog.entitiesBatch).toHaveBeenCalledWith({ + entityRefs: ['component:default/component-a'], + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + }); + + it('calls underlying catalog method on ALLOW', async () => { + fakePermissionApi.authorizeConditional.mockResolvedValue([ + { result: AuthorizeResult.ALLOW }, + ]); + const catalog = createCatalog(); + + await catalog.entitiesBatch({ + entityRefs: ['component:default/component-a'], + authorizationToken: 'abcd', + }); + + expect(fakeCatalog.entitiesBatch).toHaveBeenCalledWith({ + entityRefs: ['component:default/component-a'], + authorizationToken: 'abcd', + }); + }); + }); + describe('removeEntityByUid', () => { it('throws error on DENY', async () => { fakeCatalog.entities.mockResolvedValue({ diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index 0e861a39e1..c62e9f9a84 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -26,6 +26,8 @@ import { } from '@backstage/plugin-permission-common'; import { ConditionTransformer } from '@backstage/plugin-permission-node'; import { + EntitiesBatchRequest, + EntitiesBatchResponse, EntitiesCatalog, EntitiesRequest, EntitiesResponse, @@ -73,6 +75,37 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { return this.entitiesCatalog.entities(request); } + async entitiesBatch( + request: EntitiesBatchRequest, + ): Promise { + const authorizeDecision = ( + await this.permissionApi.authorizeConditional( + [{ permission: catalogEntityReadPermission }], + { token: request?.authorizationToken }, + ) + )[0]; + + if (authorizeDecision.result === AuthorizeResult.DENY) { + return { + items: new Array(request.entityRefs.length).fill(null), + }; + } + + if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) { + const permissionFilter: EntityFilter = this.transformConditions( + authorizeDecision.conditions, + ); + return this.entitiesCatalog.entitiesBatch({ + ...request, + filter: request?.filter + ? { allOf: [permissionFilter, request.filter] } + : permissionFilter, + }); + } + + return this.entitiesCatalog.entitiesBatch(request); + } + async removeEntityByUid( uid: string, options?: { authorizationToken?: string }, diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index 49d4fe9e19..515a862515 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -534,6 +534,60 @@ describe('DefaultEntitiesCatalog', () => { ); }); + describe('entitiesBatch', () => { + it.each(databases.eachSupportedId())( + 'queries for entities by ref, including duplicates, and gracefully returns null for missing entities', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntity( + knex, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'one' }, + spec: {}, + relations: [], + }, + [], + ); + await addEntity( + knex, + { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'two' }, + spec: {}, + relations: [], + }, + [], + ); + + const catalog = new DefaultEntitiesCatalog(knex, stitcher); + + const { items } = await catalog.entitiesBatch({ + entityRefs: [ + 'k:default/two', + 'k:default/one', + 'k:default/two', + 'not-even-a-ref', + 'k:default/does-not-exist', + 'k:default/two', + ], + }); + + expect(items.map(e => e && stringifyEntityRef(e))).toEqual([ + 'k:default/two', + 'k:default/one', + 'k:default/two', + null, + null, + 'k:default/two', + ]); + }, + ); + }); + describe('removeEntityByUid', () => { it.each(databases.eachSupportedId())( 'also clears parent hashes', diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index dc84b12484..81de6484c3 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -23,6 +23,8 @@ import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import lodash from 'lodash'; import { + EntitiesBatchRequest, + EntitiesBatchResponse, EntitiesCatalog, EntitiesRequest, EntitiesResponse, @@ -237,6 +239,38 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } + async entitiesBatch( + request: EntitiesBatchRequest, + ): Promise { + const lookup = new Map(); + + for (const chunk of lodash.chunk(request.entityRefs, 200)) { + let query = this.database('final_entities') + .innerJoin('refresh_state', { + 'refresh_state.entity_id': 'final_entities.entity_id', + }) + .select({ + entityRef: 'refresh_state.entity_ref', + entity: 'final_entities.final_entity', + }) + .whereIn('refresh_state.entity_ref', chunk); + if (request?.filter) { + query = parseFilter(request.filter, query, this.database); + } + for (const row of await query) { + lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); + } + } + + let items = request.entityRefs.map(ref => lookup.get(ref) ?? null); + + if (request.fields) { + items = items.map(e => e && request.fields!(e)); + } + + return { items }; + } + async removeEntityByUid(uid: string): Promise { // Clear the hashed state of the immediate parents of the deleted entity. // This makes sure that when they get reprocessed, their output is written diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index f383df7364..541359feb6 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -48,6 +48,7 @@ describe('createRouter readonly disabled', () => { beforeAll(async () => { entitiesCatalog = { entities: jest.fn(), + entitiesBatch: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), @@ -257,6 +258,38 @@ describe('createRouter readonly disabled', () => { }); }); + describe('POST /entities/by-refs', () => { + it.each([ + '', + 'not json', + '[', + '[]', + '{}', + '{"unknown":7}', + '{"entityRefs":7}', + '{"entityRefs":[7]}', + ])('properly rejects malformed request body, %p', async p => { + await expect( + request(app) + .post('/entities/by-refs') + .set('Content-Type', 'application/json') + .send(p), + ).resolves.toMatchObject({ status: 400 }); + }); + + it('can fetch entities by refs', async () => { + const entity: Entity = {} as any; + entitiesCatalog.entitiesBatch.mockResolvedValue({ items: [entity] }); + const response = await request(app) + .post('/entities/by-refs') + .set('Content-Type', 'application/json') + .send('{"entityRefs":["a"]}'); + expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledTimes(1); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ items: [entity] }); + }); + }); + describe('GET /locations', () => { it('happy path: lists locations', async () => { const locations: Location[] = [ @@ -517,6 +550,7 @@ describe('createRouter readonly enabled', () => { beforeAll(async () => { entitiesCatalog = { entities: jest.fn(), + entitiesBatch: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), @@ -706,6 +740,7 @@ describe('NextRouter permissioning', () => { beforeAll(async () => { entitiesCatalog = { entities: jest.fn(), + entitiesBatch: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), facets: jest.fn(), diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 5267519c33..4680dbb899 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -28,24 +28,25 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import yn from 'yn'; +import { z } from 'zod'; import { EntitiesCatalog } from '../catalog/types'; import { LocationAnalyzer } from '../ingestion/types'; +import { CatalogProcessingOrchestrator } from '../processing/types'; +import { validateEntityEnvelope } from '../processing/util'; import { basicEntityFilter, + entitiesBatchRequest, parseEntityFilterParams, parseEntityPaginationParams, parseEntityTransformParams, } from './request'; +import { parseEntityFacetParams } from './request/parseEntityFacetParams'; +import { LocationService, RefreshOptions, RefreshService } from './types'; import { disallowReadonlyMode, locationInput, validateRequestBody, } from './util'; -import { z } from 'zod'; -import { parseEntityFacetParams } from './request/parseEntityFacetParams'; -import { RefreshOptions, LocationService, RefreshService } from './types'; -import { CatalogProcessingOrchestrator } from '../processing/types'; -import { validateEntityEnvelope } from '../processing/util'; /** * Options used by {@link createRouter}. @@ -173,6 +174,16 @@ export async function createRouter( res.status(200).json(response); }, ) + .post('/entities/by-refs', async (req, res) => { + const request = entitiesBatchRequest(req); + const token = getBearerToken(req.header('authorization')); + const response = await entitiesCatalog.entitiesBatch({ + entityRefs: request.entityRefs, + fields: parseEntityTransformParams(req.query), + authorizationToken: token, + }); + res.status(200).json(response); + }) .get('/entity-facets', async (req, res) => { const response = await entitiesCatalog.facets({ filter: parseEntityFilterParams(req.query), diff --git a/plugins/catalog-backend/src/service/request/entitiesBatchRequest.ts b/plugins/catalog-backend/src/service/request/entitiesBatchRequest.ts new file mode 100644 index 0000000000..b3a91f9491 --- /dev/null +++ b/plugins/catalog-backend/src/service/request/entitiesBatchRequest.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { Request } from 'express'; +import { z } from 'zod'; + +const schema = z.object({ + entityRefs: z.array(z.string()), +}); + +export function entitiesBatchRequest(req: Request) { + try { + return schema.parse(req.body); + } catch (error) { + throw new InputError( + `Malformed request body (did you remember to specify an application/json content type?), ${error.message}`, + ); + } +} diff --git a/plugins/catalog-backend/src/service/request/index.ts b/plugins/catalog-backend/src/service/request/index.ts index 4ab579fbb6..0b09355e72 100644 --- a/plugins/catalog-backend/src/service/request/index.ts +++ b/plugins/catalog-backend/src/service/request/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export { entitiesBatchRequest } from './entitiesBatchRequest'; export { basicEntityFilter } from './basicEntityFilter'; export { parseEntityFilterParams } from './parseEntityFilterParams'; export { parseEntityPaginationParams } from './parseEntityPaginationParams';