From 02bd2cb19c2faa755518b8eb42fdcf5d8c1c29e9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Dec 2024 12:00:33 +0100 Subject: [PATCH] catalog-backend: avoid JSON parsing when possible Signed-off-by: Patrik Oldsberg --- .changeset/curly-teachers-marry.md | 5 ++ .../config/vocabularies/Backstage/accept.txt | 1 + plugins/catalog-backend/src/catalog/types.ts | 20 ++++- .../service/AuthorizedEntitiesCatalog.test.ts | 2 +- .../src/service/AuthorizedEntitiesCatalog.ts | 11 ++- .../src/service/CatalogBuilder.ts | 12 ++- .../service/DefaultEntitiesCatalog.test.ts | 55 ++++++++---- .../src/service/DefaultEntitiesCatalog.ts | 48 +++++----- .../src/service/createRouter.test.ts | 34 +++---- .../src/service/createRouter.ts | 36 +++++--- .../src/service/response/index.ts | 22 +++++ .../src/service/response/process.ts | 62 +++++++++++++ .../src/service/response/write.ts | 89 +++++++++++++++++++ plugins/catalog-backend/src/service/util.ts | 56 +++++++----- .../src/tests/integration.test.ts | 8 +- 15 files changed, 361 insertions(+), 100 deletions(-) create mode 100644 .changeset/curly-teachers-marry.md create mode 100644 plugins/catalog-backend/src/service/response/index.ts create mode 100644 plugins/catalog-backend/src/service/response/process.ts create mode 100644 plugins/catalog-backend/src/service/response/write.ts diff --git a/.changeset/curly-teachers-marry.md b/.changeset/curly-teachers-marry.md new file mode 100644 index 0000000000..21f41da291 --- /dev/null +++ b/.changeset/curly-teachers-marry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added a new `catalog.enableRawJsonResponse` configuration option that avoids JSON deserialization and serialization if possible when reading entities. This can significantly improve the overall performance of the catalog, but it removes the backwards compatibility processing that ensures that both `entity.relation[].target` and `entity.relation[].targetRef` are present in returned entities. diff --git a/.github/vale/config/vocabularies/Backstage/accept.txt b/.github/vale/config/vocabularies/Backstage/accept.txt index e0389f0adb..fc0fd5021e 100644 --- a/.github/vale/config/vocabularies/Backstage/accept.txt +++ b/.github/vale/config/vocabularies/Backstage/accept.txt @@ -97,6 +97,7 @@ deliverables denormalized dependabot deps +deserialization destructured destructuring Deutsche diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index b3e52fb480..d00c38f536 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -52,8 +52,22 @@ export type EntitiesRequest = { credentials: BackstageCredentials; }; +/** + * Encapsulates either a deserialized or serialized entities to be sent in a response. + * @internal + */ +export type EntitiesResponseItems = + | { + type: 'objects'; + entities: (Entity | null)[]; + } + | { + type: 'raw'; + entities: (string | null)[]; + }; + export type EntitiesResponse = { - entities: Entity[]; + entities: EntitiesResponseItems; pageInfo: PageInfo; }; @@ -86,7 +100,7 @@ export interface EntitiesBatchResponse { * The list of entities, in the same order as the refs in the request. Entries * that are null signify that no entity existed with that ref. */ - items: Array; + items: EntitiesResponseItems; } export type EntityAncestryResponse = { @@ -224,7 +238,7 @@ export interface QueryEntitiesResponse { /** * The entities for the current pagination request */ - items: Entity[]; + items: EntitiesResponseItems; pageInfo: { /** diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 3e2dee7ae0..2207802859 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -226,7 +226,7 @@ describe('AuthorizedEntitiesCatalog', () => { ]; fakeCatalog.queryEntities.mockResolvedValue({ - items: entities, + items: { type: 'objects', entities }, pageInfo: { nextCursor: { isPrevious: false, diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index 5358e90825..225c87cae9 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -60,7 +60,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { if (authorizeDecision.result === AuthorizeResult.DENY) { return { - entities: [], + entities: { type: 'objects', entities: [] }, pageInfo: { hasNextPage: false }, }; } @@ -92,7 +92,10 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { if (authorizeDecision.result === AuthorizeResult.DENY) { return { - items: new Array(request.entityRefs.length).fill(null), + items: { + type: 'objects', + entities: new Array(request.entityRefs.length).fill(null), + }, }; } @@ -123,7 +126,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { if (authorizeDecision.result === AuthorizeResult.DENY) { return { - items: [], + items: { type: 'objects', entities: [] }, pageInfo: {}, totalItems: 0, }; @@ -208,7 +211,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { allOf: [permissionFilter, basicEntityFilter({ 'metadata.uid': uid })], }, }); - if (entities.length === 0) { + if (entities.entities.length === 0) { throw new NotAllowedError(); } } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 31ec40b887..fd052bf474 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -115,6 +115,7 @@ import { UrlReaderService, SchedulerService, } from '@backstage/backend-plugin-api'; +import { entitiesResponseToObjects } from './response'; /** * This is a duplicate of the alpha `CatalogPermissionRule` type, for use in the stable API. @@ -485,6 +486,10 @@ export class CatalogBuilder { discovery, }); + const enableRawJson = config.getOptionalBoolean( + 'catalog.enableRawJsonResponses', + ); + const policy = this.buildEntityPolicy(); const processors = this.buildProcessors(); const parser = this.parser || defaultEntityDataParser; @@ -521,6 +526,7 @@ export class CatalogBuilder { database: dbClient, logger, stitcher, + enableRawJson, }); let permissionsService: PermissionsService; @@ -566,7 +572,10 @@ export class CatalogBuilder { }, }); - const entitiesByRef = keyBy(entities, stringifyEntityRef); + const entitiesByRef = keyBy( + entitiesResponseToObjects(entities), + stringifyEntityRef, + ); return resourceRefs.map( resourceRef => @@ -629,6 +638,7 @@ export class CatalogBuilder { auth, httpAuth, permissionsService, + enableRawJson, }); await connectEntityProviders(providerDatabase, entityProviders); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index bb2ba3d587..379c861ba1 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -38,6 +38,7 @@ import { Stitcher } from '../stitching/types'; import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { EntitiesRequest } from '../catalog/types'; import { buildEntitySearch } from '../database/operations/stitcher/buildEntitySearch'; +import { entitiesResponseToObjects } from './response'; jest.setTimeout(60_000); @@ -310,10 +311,11 @@ describe('DefaultEntitiesCatalog', () => { const testFilter = { key: 'spec.test', }; - const { entities } = await catalog.entities({ + const res = await catalog.entities({ filter: testFilter, credentials: mockCredentials.none(), }); + const entities = entitiesResponseToObjects(res.entities); expect(entities.length).toBe(1); expect(entities[0]).toEqual(entity2); @@ -351,10 +353,11 @@ describe('DefaultEntitiesCatalog', () => { key: 'spec.test', }, }; - const { entities } = await catalog.entities({ + const res = await catalog.entities({ filter: testFilter, credentials: mockCredentials.none(), }); + const entities = entitiesResponseToObjects(res.entities); expect(entities.length).toBe(1); expect(entities[0]).toEqual(entity1); @@ -416,7 +419,7 @@ describe('DefaultEntitiesCatalog', () => { values: ['red'], }, }; - const { entities } = await catalog.entities({ + const res = await catalog.entities({ filter: { allOf: [ testFilter1, @@ -427,6 +430,7 @@ describe('DefaultEntitiesCatalog', () => { }, credentials: mockCredentials.none(), }); + const entities = entitiesResponseToObjects(res.entities); expect(entities.length).toBe(2); expect(entities).toContainEqual(entity2); @@ -465,7 +469,7 @@ describe('DefaultEntitiesCatalog', () => { const testFilter2 = { key: 'metadata.desc', }; - const { entities } = await catalog.entities({ + const res = await catalog.entities({ filter: { not: { allOf: [testFilter1, testFilter2], @@ -474,6 +478,7 @@ describe('DefaultEntitiesCatalog', () => { credentials: mockCredentials.none(), }); + const entities = entitiesResponseToObjects(res.entities); expect(entities.length).toBe(1); expect(entities).toContainEqual(entity1); @@ -509,10 +514,11 @@ describe('DefaultEntitiesCatalog', () => { key: 'kind', values: [], }; - const { entities } = await catalog.entities({ + const res = await catalog.entities({ filter: testFilter, credentials: mockCredentials.none(), }); + const entities = entitiesResponseToObjects(res.entities); expect(entities.length).toBe(0); }, @@ -553,10 +559,11 @@ describe('DefaultEntitiesCatalog', () => { stitcher, }); - const { entities } = await catalog.entities(); + const res = await catalog.entities(); + const entities = entitiesResponseToObjects(res.entities); expect( - entities.find(e => e.metadata.name === 'one')!.relations, + entities.find(e => e?.metadata.name === 'one')!.relations, ).toEqual([ { type: 'r', @@ -565,7 +572,7 @@ describe('DefaultEntitiesCatalog', () => { }, ]); expect( - entities.find(e => e.metadata.name === 'two')!.relations, + entities.find(e => e?.metadata.name === 'two')!.relations, ).toEqual([ { type: 'r', @@ -615,7 +622,9 @@ describe('DefaultEntitiesCatalog', () => { return catalog .entities({ ...request, credentials: mockCredentials.none() }) .then(response => - response.entities.map(e => e.metadata.name).toSorted(), + entitiesResponseToObjects(response.entities) + .map(e => e!.metadata.name) + .toSorted(), ); } @@ -678,7 +687,11 @@ describe('DefaultEntitiesCatalog', () => { ): Promise { return catalog .entities({ ...request, credentials: mockCredentials.none() }) - .then(response => response.entities.map(e => e.metadata.name)); + .then(response => + entitiesResponseToObjects(response.entities).map( + e => e!.metadata.name, + ), + ); } await expect( @@ -764,7 +777,7 @@ describe('DefaultEntitiesCatalog', () => { stitcher, }); - const { items } = await catalog.entitiesBatch({ + const res = await catalog.entitiesBatch({ entityRefs: [ 'k:default/two', 'k:default/one', @@ -775,6 +788,7 @@ describe('DefaultEntitiesCatalog', () => { ], credentials: mockCredentials.none(), }); + const items = entitiesResponseToObjects(res.items); expect(items.map(e => e && stringifyEntityRef(e))).toEqual([ 'k:default/two', @@ -819,11 +833,12 @@ describe('DefaultEntitiesCatalog', () => { stitcher, }); - const { items } = await catalog.entitiesBatch({ + const res = await catalog.entitiesBatch({ entityRefs: ['k:default/two', 'k:default/one'], filter: { key: 'spec.owner', values: ['me'] }, credentials: mockCredentials.none(), }); + const items = entitiesResponseToObjects(res.items); expect(items.map(e => e && stringifyEntityRef(e))).toEqual([ 'k:default/two', @@ -1830,7 +1845,9 @@ describe('DefaultEntitiesCatalog', () => { orderFields: [{ field: 'metadata.title', order: 'asc' }], credentials: mockCredentials.none(), }) - .then(r => r.items.map(e => e.metadata.name)), + .then(r => + entitiesResponseToObjects(r.items).map(e => e!.metadata.name), + ), ).resolves.toEqual(['CC', 'BB', 'AA']); // 'AA' has no title, ends up last await expect( @@ -1839,7 +1856,9 @@ describe('DefaultEntitiesCatalog', () => { orderFields: [{ field: 'metadata.title', order: 'desc' }], credentials: mockCredentials.none(), }) - .then(r => r.items.map(e => e.metadata.name)), + .then(r => + entitiesResponseToObjects(r.items).map(e => e!.metadata.name), + ), ).resolves.toEqual(['BB', 'CC', 'AA']); // 'AA' has no title, ends up last }, ); @@ -1869,7 +1888,9 @@ describe('DefaultEntitiesCatalog', () => { limit: 10, credentials: mockCredentials.none(), }) - .then(r => r.items.map(e => e.metadata.name)), + .then(r => + entitiesResponseToObjects(r.items).map(e => e!.metadata.name), + ), ).resolves.toEqual(['AA', 'BB']); // simulate a situation where stitching is not yet complete @@ -1884,7 +1905,9 @@ describe('DefaultEntitiesCatalog', () => { limit: 10, credentials: mockCredentials.none(), }) - .then(r => r.items.map(e => e.metadata.name)), + .then(r => + entitiesResponseToObjects(r.items).map(e => e!.metadata.name), + ), ).resolves.toEqual(['BB']); }, ); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index cddc95f496..ee9b46ce55 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -45,13 +45,14 @@ import { import { Stitcher } from '../stitching/types'; import { - expandLegacyCompoundRelationRefsInResponse, + expandLegacyCompoundRelationsInEntity, isQueryEntitiesCursorRequest, isQueryEntitiesInitialRequest, } from './util'; import { EntityFilter } from '@backstage/plugin-catalog-node'; import { LoggerService } from '@backstage/backend-plugin-api'; import { applyEntityFilterToQuery } from './request/applyEntityFilterToQuery'; +import { processRawEntitiesResult } from './response'; const DEFAULT_LIMIT = 200; @@ -104,15 +105,18 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { private readonly database: Knex; private readonly logger: LoggerService; private readonly stitcher: Stitcher; + private readonly enableRawJson: boolean; constructor(options: { database: Knex; logger: LoggerService; stitcher: Stitcher; + enableRawJson?: boolean; }) { this.database = options.database; this.logger = options.logger; this.stitcher = options.stitcher; + this.enableRawJson = Boolean(options.enableRawJson); } async entities(request?: EntitiesRequest): Promise { @@ -190,16 +194,19 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } - let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!)); - - if (request?.fields) { - entities = entities.map(e => request.fields!(e)); - } - - expandLegacyCompoundRelationRefsInResponse(entities); - return { - entities, + entities: processRawEntitiesResult( + rows.map(r => r.final_entity!), + this.enableRawJson + ? request?.fields + : e => { + expandLegacyCompoundRelationsInEntity(e); + if (request?.fields) { + return request.fields(e); + } + return e; + }, + ), pageInfo, }; } @@ -207,7 +214,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { async entitiesBatch( request: EntitiesBatchRequest, ): Promise { - const lookup = new Map(); + const lookup = new Map(); for (const chunk of lodashChunk(request.entityRefs, 200)) { let query = this.database('final_entities') @@ -227,17 +234,13 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } for (const row of await query) { - lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); + lookup.set(row.entityRef, row.entity ? row.entity : null); } } - let items = request.entityRefs.map(ref => lookup.get(ref) ?? null); + const items = request.entityRefs.map(ref => lookup.get(ref) ?? null); - if (request.fields) { - items = items.map(e => e && request.fields!(e)); - } - - return { items }; + return { items: processRawEntitiesResult(items, request.fields) }; } async queryEntities( @@ -505,12 +508,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } : undefined; - const items = rows - .map(e => JSON.parse(e.final_entity!)) - .map(e => (request.fields ? request.fields(e) : e)); - return { - items, + items: processRawEntitiesResult( + rows.map(r => r.final_entity!), + request.fields, + ), pageInfo: { ...(!!prevCursor && { prevCursor }), ...(!!nextCursor && { nextCursor }), diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index e728017f0b..8efb6543a3 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -136,7 +136,7 @@ describe('createRouter readonly disabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items: [entities[0]], + items: { type: 'objects', entities: [entities[0]] }, pageInfo: {}, totalItems: 1, }); @@ -149,7 +149,7 @@ describe('createRouter readonly disabled', () => { it('parses single and multiple request parameters and passes them down', async () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items: [], + items: { type: 'objects', entities: [] }, pageInfo: {}, totalItems: 0, }); @@ -185,7 +185,7 @@ describe('createRouter readonly disabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items, + items: { type: 'objects', entities: items }, totalItems: 100, pageInfo: { nextCursor: mockCursor() }, }); @@ -203,7 +203,7 @@ describe('createRouter readonly disabled', () => { it('parses initial request', async () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items: [], + items: { type: 'objects', entities: [] }, pageInfo: {}, totalItems: 0, }); @@ -239,7 +239,7 @@ describe('createRouter readonly disabled', () => { it('parses encoded params request', async () => { entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items: [], + items: { type: 'objects', entities: [] }, pageInfo: {}, totalItems: 0, }); @@ -283,7 +283,7 @@ describe('createRouter readonly disabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items, + items: { type: 'objects', entities: items }, totalItems: 100, pageInfo: { nextCursor: mockCursor() }, }); @@ -312,7 +312,7 @@ describe('createRouter readonly disabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items, + items: { type: 'objects', entities: items }, totalItems: 100, pageInfo: { nextCursor: mockCursor({ fullTextFilter: { term: 'mySearch' } }), @@ -354,7 +354,7 @@ describe('createRouter readonly disabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items, + items: { type: 'objects', entities: items }, totalItems: 100, pageInfo: { nextCursor: mockCursor() }, }); @@ -379,7 +379,7 @@ describe('createRouter readonly disabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items, + items: { type: 'objects', entities: items }, totalItems: 100, pageInfo: { nextCursor: mockCursor() }, }); @@ -402,7 +402,7 @@ describe('createRouter readonly disabled', () => { }, }; entitiesCatalog.entities.mockResolvedValue({ - entities: [entity], + entities: { type: 'objects', entities: [entity] }, pageInfo: { hasNextPage: false }, }); @@ -419,7 +419,7 @@ describe('createRouter readonly disabled', () => { it('responds with a 404 for missing entities', async () => { entitiesCatalog.entities.mockResolvedValue({ - entities: [], + entities: { type: 'objects', entities: [] }, pageInfo: { hasNextPage: false }, }); @@ -446,7 +446,7 @@ describe('createRouter readonly disabled', () => { }, }; entitiesCatalog.entitiesBatch.mockResolvedValue({ - items: [entity], + items: { type: 'objects', entities: [entity] }, }); const response = await request(app).get('/entities/by-name/k/ns/n'); @@ -462,7 +462,7 @@ describe('createRouter readonly disabled', () => { it('responds with a 404 for missing entities', async () => { entitiesCatalog.entitiesBatch.mockResolvedValue({ - items: [null], + items: { type: 'objects', entities: [null] }, }); const response = await request(app).get('/entities/by-name/b/d/c'); @@ -533,7 +533,9 @@ describe('createRouter readonly disabled', () => { }, }; const entityRef = stringifyEntityRef(entity); - entitiesCatalog.entitiesBatch.mockResolvedValue({ items: [entity] }); + entitiesCatalog.entitiesBatch.mockResolvedValue({ + items: { type: 'objects', entities: [entity] }, + }); const response = await request(app) .post('/entities/by-refs?filter=kind=Component') .set('Content-Type', 'application/json') @@ -908,7 +910,7 @@ describe('createRouter readonly enabled', () => { ]; entitiesCatalog.queryEntities.mockResolvedValueOnce({ - items: [entities[0]], + items: { type: 'objects', entities: [entities[0]] }, pageInfo: {}, totalItems: 1, }); @@ -1128,7 +1130,7 @@ describe('NextRouter permissioning', () => { }, }; entitiesCatalog.entities.mockResolvedValueOnce({ - entities: [spideySense], + entities: { type: 'objects', entities: [spideySense] }, pageInfo: { hasNextPage: false }, }); diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index d70419c565..b670253089 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -44,7 +44,7 @@ import { createEntityArrayJsonStream, disallowReadonlyMode, encodeCursor, - expandLegacyCompoundRelationRefsInResponse, + expandLegacyCompoundRelationsInEntity, locationInput, validateRequestBody, } from './util'; @@ -60,6 +60,11 @@ import { import { LocationAnalyzer } from '@backstage/plugin-catalog-node'; import { AuthorizedValidationService } from './AuthorizedValidationService'; import { DeferredPromise, createDeferred } from '@backstage/types'; +import { + processEntitiesResponseItems, + writeEntitiesResponse, + writeSingleEntityResponse, +} from './response'; /** * Options used by {@link createRouter}. @@ -80,6 +85,7 @@ export interface RouterOptions { auth: AuthService; httpAuth: HttpAuthService; permissionsService: PermissionsService; + enableRawJson?: boolean; } /** @@ -107,6 +113,7 @@ export async function createRouter( permissionsService, auth, httpAuth, + enableRawJson = false, } = options; const readonlyEnabled = @@ -164,7 +171,7 @@ export async function createRouter( res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`); } - res.json(entities); + writeEntitiesResponse(res, entities); return; } @@ -203,12 +210,17 @@ export async function createRouter( : { credentials, fields, limit, cursor }, ); - if (result.items.length) { + if (result.items.entities.length) { await locks?.writeLock; signal.throwIfAborted(); - expandLegacyCompoundRelationRefsInResponse(result.items); + if (!enableRawJson) { + processEntitiesResponseItems( + result.items, + expandLegacyCompoundRelationsInEntity, + ); + } if (!responseStream.send(result.items)) { // The kernel buffer is full. Create the lock but do not await it // yet - we can better spend our time going to the next round of @@ -241,8 +253,8 @@ export async function createRouter( credentials: await httpAuth.credentials(req), }); - res.json({ - items, + writeEntitiesResponse(res, items, entities => ({ + items: entities, totalItems, pageInfo: { ...(pageInfo.nextCursor && { @@ -252,7 +264,7 @@ export async function createRouter( prevCursor: encodeCursor(pageInfo.prevCursor), }), }, - }); + })); }) .get('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; @@ -260,10 +272,9 @@ export async function createRouter( filter: basicEntityFilter({ 'metadata.uid': uid }), credentials: await httpAuth.credentials(req), }); - if (!entities.length) { + if (!writeSingleEntityResponse(res, entities)) { throw new NotFoundError(`No entity with uid ${uid}`); } - res.status(200).json(entities[0]); }) .delete('/entities/by-uid/:uid', async (req, res) => { const { uid } = req.params; @@ -278,12 +289,11 @@ export async function createRouter( entityRefs: [stringifyEntityRef({ kind, namespace, name })], credentials: await httpAuth.credentials(req), }); - if (!items[0]) { + if (!writeSingleEntityResponse(res, items)) { throw new NotFoundError( `No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`, ); } - res.status(200).json(items[0]); }) .get( '/entities/by-name/:kind/:namespace/:name/ancestry', @@ -298,13 +308,13 @@ export async function createRouter( ) .post('/entities/by-refs', async (req, res) => { const request = entitiesBatchRequest(req); - const response = await entitiesCatalog.entitiesBatch({ + const { items } = await entitiesCatalog.entitiesBatch({ entityRefs: request.entityRefs, filter: parseEntityFilterParams(req.query), fields: parseEntityTransformParams(req.query, request.fields), credentials: await httpAuth.credentials(req), }); - res.status(200).json(response); + writeEntitiesResponse(res, items, entities => ({ items: entities })); }) .get('/entity-facets', async (req, res) => { const response = await entitiesCatalog.facets({ diff --git a/plugins/catalog-backend/src/service/response/index.ts b/plugins/catalog-backend/src/service/response/index.ts new file mode 100644 index 0000000000..73c82c2aee --- /dev/null +++ b/plugins/catalog-backend/src/service/response/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + processRawEntitiesResult, + processEntitiesResponseItems, + entitiesResponseToObjects, +} from './process'; +export { writeSingleEntityResponse, writeEntitiesResponse } from './write'; diff --git a/plugins/catalog-backend/src/service/response/process.ts b/plugins/catalog-backend/src/service/response/process.ts new file mode 100644 index 0000000000..26ab544a4d --- /dev/null +++ b/plugins/catalog-backend/src/service/response/process.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { EntitiesResponseItems } from '../../catalog/types'; + +export function processRawEntitiesResult( + serializedEntities: (string | null)[], + transform?: (entity: Entity) => Entity, +): EntitiesResponseItems { + if (transform) { + return { + type: 'objects', + entities: serializedEntities.map(e => + e !== null ? transform(JSON.parse(e)) : e, + ), + }; + } + + return { + type: 'raw', + entities: serializedEntities, + }; +} + +export function processEntitiesResponseItems( + response: EntitiesResponseItems, + transform?: (entity: Entity) => Entity, +) { + if (!transform) { + return response; + } + if (response.type === 'raw') { + return processRawEntitiesResult(response.entities, transform); + } + return { + type: 'objects', + entities: response.entities.map(e => (e !== null ? transform(e) : e)), + }; +} + +export function entitiesResponseToObjects( + response: EntitiesResponseItems, +): (Entity | null)[] { + if (response.type === 'objects') { + return response.entities; + } + return response.entities.map(e => (e !== null ? JSON.parse(e) : e)); +} diff --git a/plugins/catalog-backend/src/service/response/write.ts b/plugins/catalog-backend/src/service/response/write.ts new file mode 100644 index 0000000000..1f9392e3a9 --- /dev/null +++ b/plugins/catalog-backend/src/service/response/write.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Response } from 'express'; +import { EntitiesResponseItems } from '../../catalog/types'; +import { JsonValue } from '@backstage/types'; + +const JSON_CONTENT_TYPE = 'application/json; charset=utf-8'; + +export function writeSingleEntityResponse( + res: Response, + response: EntitiesResponseItems, +): boolean { + const entity = response.entities[0]; + if (!entity) { + return false; + } + + if (typeof entity === 'string') { + res.setHeader('Content-Type', JSON_CONTENT_TYPE); + res.status(200); + res.write(entity); + } else { + res.json(entity); + } + + return true; +} + +export function writeEntitiesResponse( + res: Response, + response: EntitiesResponseItems, + responseWrapper?: (entities: JsonValue) => JsonValue, +) { + if (response.type === 'objects') { + res.json( + responseWrapper + ? responseWrapper?.(response.entities) + : response.entities, + ); + return; + } + + res.setHeader('Content-Type', JSON_CONTENT_TYPE); + res.status(200); + + // responseWrapper allows the caller to render the entities within an object + let trailing = ''; + if (responseWrapper) { + const marker = `__MARKER_${Math.random().toString(36).slice(2, 10)}__`; + const wrapped = JSON.stringify(responseWrapper(marker)); + const parts = wrapped.split(marker); + if (parts.length !== 2) { + throw new Error( + `Entity items response was incorrectly wrapped into ${parts.length} different parts`, + ); + } + res.write(parts[0], 'utf8'); + trailing = parts[1]; + } + + let first = true; + for (const entity of response.entities) { + if (first) { + res.write('[', 'utf8'); + first = false; + } else { + res.write(',', 'utf8'); + } + res.write(entity, 'utf8'); + } + res.end(']'); + if (trailing) { + res.write(trailing, 'utf8'); + } +} diff --git a/plugins/catalog-backend/src/service/util.ts b/plugins/catalog-backend/src/service/util.ts index c05fb14558..2126cf25b3 100644 --- a/plugins/catalog-backend/src/service/util.ts +++ b/plugins/catalog-backend/src/service/util.ts @@ -20,6 +20,7 @@ import lodash from 'lodash'; import { z } from 'zod'; import { Cursor, + EntitiesResponseItems, QueryEntitiesCursorRequest, QueryEntitiesInitialRequest, QueryEntitiesRequest, @@ -148,35 +149,32 @@ export function decodeCursor(encodedCursor: string) { // sure that all adopters have re-stitched their entities so that the new // targetRef field is present on them, and that they have stopped consuming // the now-removed old field -// TODO(jhaals): Remove this in April 2022 -export function expandLegacyCompoundRelationRefsInResponse( - entities: Entity[], -): void { - for (const entity of entities) { - if (entity.relations) { - for (const relation of entity.relations as any) { - if (!relation.targetRef && relation.target) { - // This is the case where an old-form entity, not yet stitched with - // the updated code, was in the database - relation.targetRef = stringifyEntityRef(relation.target); - } else if (!relation.target && relation.targetRef) { - // This is the case where a new-form entity, stitched with the - // updated code, was in the database but we still want to produce - // the old data shape as well for compatibility reasons - relation.target = parseEntityRef(relation.targetRef); - } +// TODO(patriko): Remove this in catalog 2.0 +export function expandLegacyCompoundRelationsInEntity(entity: Entity): Entity { + if (entity.relations) { + for (const relation of entity.relations as any) { + if (!relation.targetRef && relation.target) { + // This is the case where an old-form entity, not yet stitched with + // the updated code, was in the database + relation.targetRef = stringifyEntityRef(relation.target); + } else if (!relation.target && relation.targetRef) { + // This is the case where a new-form entity, stitched with the + // updated code, was in the database but we still want to produce + // the old data shape as well for compatibility reasons + relation.target = parseEntityRef(relation.targetRef); } } } + return entity; } export interface EntityArrayJsonStream { - send(entities: Entity[]): boolean; + send(entities: EntitiesResponseItems): boolean; complete(): void; close(): void; } -// Helps stream Entity[] as a JSON response stream to avoid performance issues +// Helps stream EntitiesResponseItems[] as a JSON response stream to avoid performance issues export function createEntityArrayJsonStream( res: Response, ): EntityArrayJsonStream { @@ -186,19 +184,33 @@ export function createEntityArrayJsonStream( let completed = false; return { - send(entities) { + send(response) { if (firstSend) { res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.status(200); res.flushHeaders(); } + if (response.type === 'raw') { + let result = true; + for (const item of response.entities) { + if (firstSend) { + result ||= res.write('['); + firstSend = false; + } else { + result ||= res.write(','); + } + result ||= res.write(item); + } + return result; + } + let data: string; if (prettyPrint) { - data = JSON.stringify(entities, null, 2); + data = JSON.stringify(response.entities, null, 2); data = firstSend ? data.slice(0, -2) : `,\n${data.slice(2, -2)}`; } else { - data = JSON.stringify(entities); + data = JSON.stringify(response.entities); data = firstSend ? data.slice(0, -1) : `,${data.slice(1, -1)}`; } diff --git a/plugins/catalog-backend/src/tests/integration.test.ts b/plugins/catalog-backend/src/tests/integration.test.ts index 737bb107ca..fdafbd5edf 100644 --- a/plugins/catalog-backend/src/tests/integration.test.ts +++ b/plugins/catalog-backend/src/tests/integration.test.ts @@ -55,6 +55,7 @@ import { DefaultStitcher } from '../stitching/DefaultStitcher'; import { mockServices } from '@backstage/backend-test-utils'; import { LoggerService } from '@backstage/backend-plugin-api'; import { DatabaseManager } from '@backstage/backend-common'; +import { entitiesResponseToObjects } from '../service/response'; const voidLogger = mockServices.logger.mock(); @@ -365,7 +366,12 @@ class TestHarness { async getOutputEntities(): Promise> { const { entities } = await this.#catalog.entities(); - return Object.fromEntries(entities.map(e => [stringifyEntityRef(e), e])); + return Object.fromEntries( + entitiesResponseToObjects(entities).map(e => [ + stringifyEntityRef(e!), + e!, + ]), + ); } async refresh(options: RefreshOptions) {