paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

added a by-refs batch endpoint for entities

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2022-10-26 17:06:01 +02:00
parent 3a5aefb4f8
commit 16891a212c
10 changed files with 311 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/fast-lies-remain.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend': minor
---
Added new `POST /entities/by-refs` endpoint, which allows you to efficiently
batch-fetch entities by their entity ref. This can be useful e.g. in graphql
resolvers or similar contexts where you need to fetch many entities at the same
time.
plugins/catalog-backend/src/catalog/types.ts
+35
View File
@@ -80,6 +80,36 @@ export type EntitiesResponse = {
pageInfo: PageInfo;
};
/**
* A request for a batch of entities.
*/
export interface EntitiesBatchRequest {
/**
* The refs for which to fetch entities.
*/
entityRefs: string[];
/**
* Any additional filters to apply in the selection of the entities.
*/
filter?: EntityFilter;
/**
* Strips out only the parts of the entity bodies to include in the response.
*/
fields?: (entity: Entity) => Entity;
/**
* The optional token that authorizes the action.
*/
authorizationToken?: string;
}
export interface EntitiesBatchResponse {
/**
* The list of entities, in the same order as the refs in the request. Entries
* that are null signify that no entity existed with that ref.
*/
items: Array<Entity | null>;
}
export type EntityAncestryResponse = {
rootEntityRef: string;
items: Array<{
@@ -130,6 +160,11 @@ export interface EntitiesCatalog {
*/
entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
/**
* Fetches a batch of entities.
*/
entitiesBatch(request: EntitiesBatchRequest): Promise<EntitiesBatchResponse>;
/**
* Removes a single entity.
*
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts
+62
View File
@@ -24,6 +24,7 @@ import { AuthorizedEntitiesCatalog } from './AuthorizedEntitiesCatalog';
describe('AuthorizedEntitiesCatalog', () => {
const fakeCatalog = {
entities: jest.fn(),
entitiesBatch: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
@@ -92,6 +93,67 @@ describe('AuthorizedEntitiesCatalog', () => {
});
});
describe('entitiesBatch', () => {
it('returns empty response on DENY', async () => {
fakePermissionApi.authorizeConditional.mockResolvedValue([
{ result: AuthorizeResult.DENY },
]);
const catalog = createCatalog();
await expect(
catalog.entitiesBatch({
entityRefs: ['component:default/component-a'],
authorizationToken: 'abcd',
}),
).resolves.toEqual({
items: [null],
});
expect(fakeCatalog.entitiesBatch).not.toHaveBeenCalled();
});
it('calls underlying catalog method with correct filter on CONDITIONAL', async () => {
fakePermissionApi.authorizeConditional.mockResolvedValue([
{
result: AuthorizeResult.CONDITIONAL,
conditions: {
rule: 'IS_ENTITY_KIND',
params: { kinds: ['b'] },
},
},
]);
const catalog = createCatalog(isEntityKind);
await catalog.entitiesBatch({
entityRefs: ['component:default/component-a'],
authorizationToken: 'abcd',
});
expect(fakeCatalog.entitiesBatch).toHaveBeenCalledWith({
entityRefs: ['component:default/component-a'],
authorizationToken: 'abcd',
filter: { key: 'kind', values: ['b'] },
});
});
it('calls underlying catalog method on ALLOW', async () => {
fakePermissionApi.authorizeConditional.mockResolvedValue([
{ result: AuthorizeResult.ALLOW },
]);
const catalog = createCatalog();
await catalog.entitiesBatch({
entityRefs: ['component:default/component-a'],
authorizationToken: 'abcd',
});
expect(fakeCatalog.entitiesBatch).toHaveBeenCalledWith({
entityRefs: ['component:default/component-a'],
authorizationToken: 'abcd',
});
});
});
describe('removeEntityByUid', () => {
it('throws error on DENY', async () => {
fakeCatalog.entities.mockResolvedValue({
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts
+33
View File
@@ -26,6 +26,8 @@ import {
} from '@backstage/plugin-permission-common';
import { ConditionTransformer } from '@backstage/plugin-permission-node';
import {
EntitiesBatchRequest,
EntitiesBatchResponse,
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
@@ -73,6 +75,37 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
return this.entitiesCatalog.entities(request);
}
async entitiesBatch(
request: EntitiesBatchRequest,
): Promise<EntitiesBatchResponse> {
const authorizeDecision = (
await this.permissionApi.authorizeConditional(
[{ permission: catalogEntityReadPermission }],
{ token: request?.authorizationToken },
)
)[0];
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
items: new Array(request.entityRefs.length).fill(null),
};
}
if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) {
const permissionFilter: EntityFilter = this.transformConditions(
authorizeDecision.conditions,
);
return this.entitiesCatalog.entitiesBatch({
...request,
filter: request?.filter
? { allOf: [permissionFilter, request.filter] }
: permissionFilter,
});
}
return this.entitiesCatalog.entitiesBatch(request);
}
async removeEntityByUid(
uid: string,
options?: { authorizationToken?: string },
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+54
View File
@@ -534,6 +534,60 @@ describe('DefaultEntitiesCatalog', () => {
);
});
describe('entitiesBatch', () => {
it.each(databases.eachSupportedId())(
'queries for entities by ref, including duplicates, and gracefully returns null for missing entities',
async databaseId => {
const { knex } = await createDatabase(databaseId);
await addEntity(
knex,
{
apiVersion: 'a',
kind: 'k',
metadata: { name: 'one' },
spec: {},
relations: [],
},
[],
);
await addEntity(
knex,
{
apiVersion: 'a',
kind: 'k',
metadata: { name: 'two' },
spec: {},
relations: [],
},
[],
);
const catalog = new DefaultEntitiesCatalog(knex, stitcher);
const { items } = await catalog.entitiesBatch({
entityRefs: [
'k:default/two',
'k:default/one',
'k:default/two',
'not-even-a-ref',
'k:default/does-not-exist',
'k:default/two',
],
});
expect(items.map(e => e && stringifyEntityRef(e))).toEqual([
'k:default/two',
'k:default/one',
'k:default/two',
null,
null,
'k:default/two',
]);
},
);
});
describe('removeEntityByUid', () => {
it.each(databases.eachSupportedId())(
'also clears parent hashes',
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts
+34
View File
@@ -23,6 +23,8 @@ import { InputError, NotFoundError } from '@backstage/errors';
import { Knex } from 'knex';
import lodash from 'lodash';
import {
EntitiesBatchRequest,
EntitiesBatchResponse,
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
@@ -237,6 +239,38 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
};
}
async entitiesBatch(
request: EntitiesBatchRequest,
): Promise<EntitiesBatchResponse> {
const lookup = new Map<string, Entity>();
for (const chunk of lodash.chunk(request.entityRefs, 200)) {
let query = this.database<DbFinalEntitiesRow>('final_entities')
.innerJoin<DbRefreshStateRow>('refresh_state', {
'refresh_state.entity_id': 'final_entities.entity_id',
})
.select({
entityRef: 'refresh_state.entity_ref',
entity: 'final_entities.final_entity',
})
.whereIn('refresh_state.entity_ref', chunk);
if (request?.filter) {
query = parseFilter(request.filter, query, this.database);
}
for (const row of await query) {
lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null);
}
}
let items = request.entityRefs.map(ref => lookup.get(ref) ?? null);
if (request.fields) {
items = items.map(e => e && request.fields!(e));
}
return { items };
}
async removeEntityByUid(uid: string): Promise<void> {
// Clear the hashed state of the immediate parents of the deleted entity.
// This makes sure that when they get reprocessed, their output is written
plugins/catalog-backend/src/service/createRouter.test.ts
+35
View File
@@ -48,6 +48,7 @@ describe('createRouter readonly disabled', () => {
beforeAll(async () => {
entitiesCatalog = {
entities: jest.fn(),
entitiesBatch: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
@@ -257,6 +258,38 @@ describe('createRouter readonly disabled', () => {
});
});
describe('POST /entities/by-refs', () => {
it.each([
'',
'not json',
'[',
'[]',
'{}',
'{"unknown":7}',
'{"entityRefs":7}',
'{"entityRefs":[7]}',
])('properly rejects malformed request body, %p', async p => {
await expect(
request(app)
.post('/entities/by-refs')
.set('Content-Type', 'application/json')
.send(p),
).resolves.toMatchObject({ status: 400 });
});
it('can fetch entities by refs', async () => {
const entity: Entity = {} as any;
entitiesCatalog.entitiesBatch.mockResolvedValue({ items: [entity] });
const response = await request(app)
.post('/entities/by-refs')
.set('Content-Type', 'application/json')
.send('{"entityRefs":["a"]}');
expect(entitiesCatalog.entitiesBatch).toHaveBeenCalledTimes(1);
expect(response.status).toEqual(200);
expect(response.body).toEqual({ items: [entity] });
});
});
describe('GET /locations', () => {
it('happy path: lists locations', async () => {
const locations: Location[] = [
@@ -517,6 +550,7 @@ describe('createRouter readonly enabled', () => {
beforeAll(async () => {
entitiesCatalog = {
entities: jest.fn(),
entitiesBatch: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
@@ -706,6 +740,7 @@ describe('NextRouter permissioning', () => {
beforeAll(async () => {
entitiesCatalog = {
entities: jest.fn(),
entitiesBatch: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
plugins/catalog-backend/src/service/createRouter.ts
+16 -5
View File
@@ -28,24 +28,25 @@ import express from 'express';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import yn from 'yn';
import { z } from 'zod';
import { EntitiesCatalog } from '../catalog/types';
import { LocationAnalyzer } from '../ingestion/types';
import { CatalogProcessingOrchestrator } from '../processing/types';
import { validateEntityEnvelope } from '../processing/util';
import {
basicEntityFilter,
entitiesBatchRequest,
parseEntityFilterParams,
parseEntityPaginationParams,
parseEntityTransformParams,
} from './request';
import { parseEntityFacetParams } from './request/parseEntityFacetParams';
import { LocationService, RefreshOptions, RefreshService } from './types';
import {
disallowReadonlyMode,
locationInput,
validateRequestBody,
} from './util';
import { z } from 'zod';
import { parseEntityFacetParams } from './request/parseEntityFacetParams';
import { RefreshOptions, LocationService, RefreshService } from './types';
import { CatalogProcessingOrchestrator } from '../processing/types';
import { validateEntityEnvelope } from '../processing/util';
/**
* Options used by {@link createRouter}.
@@ -173,6 +174,16 @@ export async function createRouter(
res.status(200).json(response);
},
)
.post('/entities/by-refs', async (req, res) => {
const request = entitiesBatchRequest(req);
const token = getBearerToken(req.header('authorization'));
const response = await entitiesCatalog.entitiesBatch({
entityRefs: request.entityRefs,
fields: parseEntityTransformParams(req.query),
authorizationToken: token,
});
res.status(200).json(response);
})
.get('/entity-facets', async (req, res) => {
const response = await entitiesCatalog.facets({
filter: parseEntityFilterParams(req.query),
plugins/catalog-backend/src/service/request/entitiesBatchRequest.ts
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { Request } from 'express';
import { z } from 'zod';
const schema = z.object({
entityRefs: z.array(z.string()),
});
export function entitiesBatchRequest(req: Request) {
try {
return schema.parse(req.body);
} catch (error) {
throw new InputError(
`Malformed request body (did you remember to specify an application/json content type?), ${error.message}`,
);
}
}
plugins/catalog-backend/src/service/request/index.ts
+1
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
export { entitiesBatchRequest } from './entitiesBatchRequest';
export { basicEntityFilter } from './basicEntityFilter';
export { parseEntityFilterParams } from './parseEntityFilterParams';
export { parseEntityPaginationParams } from './parseEntityPaginationParams';