paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

catalog-backend: avoid JSON parsing when possible

Browse Source 
Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
2024-12-12 12:00:33 +01:00
parent 9eb2b5e86d
commit 02bd2cb19c
15 changed files with 361 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/catalog/types.ts
+17 -3
View File
@@ -52,8 +52,22 @@ export type EntitiesRequest = {
credentials: BackstageCredentials;
};
/**
* Encapsulates either a deserialized or serialized entities to be sent in a response.
* @internal
*/
export type EntitiesResponseItems =
| {
type: 'objects';
entities: (Entity | null)[];
}
| {
type: 'raw';
entities: (string | null)[];
};
export type EntitiesResponse = {
entities: Entity[];
entities: EntitiesResponseItems;
pageInfo: PageInfo;
};
@@ -86,7 +100,7 @@ export interface EntitiesBatchResponse {
* The list of entities, in the same order as the refs in the request. Entries
* that are null signify that no entity existed with that ref.
*/
items: Array<Entity | null>;
items: EntitiesResponseItems;
}
export type EntityAncestryResponse = {
@@ -224,7 +238,7 @@ export interface QueryEntitiesResponse {
/**
* The entities for the current pagination request
*/
items: Entity[];
items: EntitiesResponseItems;
pageInfo: {
/**
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts
+1 -1
View File
@@ -226,7 +226,7 @@ describe('AuthorizedEntitiesCatalog', () => {
];
fakeCatalog.queryEntities.mockResolvedValue({
items: entities,
items: { type: 'objects', entities },
pageInfo: {
nextCursor: {
isPrevious: false,
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts
+7 -4
View File
@@ -60,7 +60,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
entities: [],
entities: { type: 'objects', entities: [] },
pageInfo: { hasNextPage: false },
};
}
@@ -92,7 +92,10 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
items: new Array(request.entityRefs.length).fill(null),
items: {
type: 'objects',
entities: new Array(request.entityRefs.length).fill(null),
},
};
}
@@ -123,7 +126,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
items: [],
items: { type: 'objects', entities: [] },
pageInfo: {},
totalItems: 0,
};
@@ -208,7 +211,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
allOf: [permissionFilter, basicEntityFilter({ 'metadata.uid': uid })],
},
});
if (entities.length === 0) {
if (entities.entities.length === 0) {
throw new NotAllowedError();
}
}
plugins/catalog-backend/src/service/CatalogBuilder.ts
+11 -1
View File
@@ -115,6 +115,7 @@ import {
UrlReaderService,
SchedulerService,
} from '@backstage/backend-plugin-api';
import { entitiesResponseToObjects } from './response';
/**
* This is a duplicate of the alpha `CatalogPermissionRule` type, for use in the stable API.
@@ -485,6 +486,10 @@ export class CatalogBuilder {
discovery,
});
const enableRawJson = config.getOptionalBoolean(
'catalog.enableRawJsonResponses',
);
const policy = this.buildEntityPolicy();
const processors = this.buildProcessors();
const parser = this.parser || defaultEntityDataParser;
@@ -521,6 +526,7 @@ export class CatalogBuilder {
database: dbClient,
logger,
stitcher,
enableRawJson,
});
let permissionsService: PermissionsService;
@@ -566,7 +572,10 @@ export class CatalogBuilder {
},
});
const entitiesByRef = keyBy(entities, stringifyEntityRef);
const entitiesByRef = keyBy(
entitiesResponseToObjects(entities),
stringifyEntityRef,
);
return resourceRefs.map(
resourceRef =>
@@ -629,6 +638,7 @@ export class CatalogBuilder {
auth,
httpAuth,
permissionsService,
enableRawJson,
});
await connectEntityProviders(providerDatabase, entityProviders);
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+39 -16
View File
@@ -38,6 +38,7 @@ import { Stitcher } from '../stitching/types';
import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';
import { EntitiesRequest } from '../catalog/types';
import { buildEntitySearch } from '../database/operations/stitcher/buildEntitySearch';
import { entitiesResponseToObjects } from './response';
jest.setTimeout(60_000);
@@ -310,10 +311,11 @@ describe('DefaultEntitiesCatalog', () => {
const testFilter = {
key: 'spec.test',
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: testFilter,
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity2);
@@ -351,10 +353,11 @@ describe('DefaultEntitiesCatalog', () => {
key: 'spec.test',
},
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: testFilter,
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity1);
@@ -416,7 +419,7 @@ describe('DefaultEntitiesCatalog', () => {
values: ['red'],
},
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: {
allOf: [
testFilter1,
@@ -427,6 +430,7 @@ describe('DefaultEntitiesCatalog', () => {
},
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(2);
expect(entities).toContainEqual(entity2);
@@ -465,7 +469,7 @@ describe('DefaultEntitiesCatalog', () => {
const testFilter2 = {
key: 'metadata.desc',
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: {
not: {
allOf: [testFilter1, testFilter2],
@@ -474,6 +478,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(1);
expect(entities).toContainEqual(entity1);
@@ -509,10 +514,11 @@ describe('DefaultEntitiesCatalog', () => {
key: 'kind',
values: [],
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: testFilter,
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(0);
},
@@ -553,10 +559,11 @@ describe('DefaultEntitiesCatalog', () => {
stitcher,
});
const { entities } = await catalog.entities();
const res = await catalog.entities();
const entities = entitiesResponseToObjects(res.entities);
expect(
entities.find(e => e.metadata.name === 'one')!.relations,
entities.find(e => e?.metadata.name === 'one')!.relations,
).toEqual([
{
type: 'r',
@@ -565,7 +572,7 @@ describe('DefaultEntitiesCatalog', () => {
},
]);
expect(
entities.find(e => e.metadata.name === 'two')!.relations,
entities.find(e => e?.metadata.name === 'two')!.relations,
).toEqual([
{
type: 'r',
@@ -615,7 +622,9 @@ describe('DefaultEntitiesCatalog', () => {
return catalog
.entities({ ...request, credentials: mockCredentials.none() })
.then(response =>
response.entities.map(e => e.metadata.name).toSorted(),
entitiesResponseToObjects(response.entities)
.map(e => e!.metadata.name)
.toSorted(),
);
}
@@ -678,7 +687,11 @@ describe('DefaultEntitiesCatalog', () => {
): Promise<string[]> {
return catalog
.entities({ ...request, credentials: mockCredentials.none() })
.then(response => response.entities.map(e => e.metadata.name));
.then(response =>
entitiesResponseToObjects(response.entities).map(
e => e!.metadata.name,
),
);
}
await expect(
@@ -764,7 +777,7 @@ describe('DefaultEntitiesCatalog', () => {
stitcher,
});
const { items } = await catalog.entitiesBatch({
const res = await catalog.entitiesBatch({
entityRefs: [
'k:default/two',
'k:default/one',
@@ -775,6 +788,7 @@ describe('DefaultEntitiesCatalog', () => {
],
credentials: mockCredentials.none(),
});
const items = entitiesResponseToObjects(res.items);
expect(items.map(e => e && stringifyEntityRef(e))).toEqual([
'k:default/two',
@@ -819,11 +833,12 @@ describe('DefaultEntitiesCatalog', () => {
stitcher,
});
const { items } = await catalog.entitiesBatch({
const res = await catalog.entitiesBatch({
entityRefs: ['k:default/two', 'k:default/one'],
filter: { key: 'spec.owner', values: ['me'] },
credentials: mockCredentials.none(),
});
const items = entitiesResponseToObjects(res.items);
expect(items.map(e => e && stringifyEntityRef(e))).toEqual([
'k:default/two',
@@ -1830,7 +1845,9 @@ describe('DefaultEntitiesCatalog', () => {
orderFields: [{ field: 'metadata.title', order: 'asc' }],
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['CC', 'BB', 'AA']); // 'AA' has no title, ends up last
await expect(
@@ -1839,7 +1856,9 @@ describe('DefaultEntitiesCatalog', () => {
orderFields: [{ field: 'metadata.title', order: 'desc' }],
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['BB', 'CC', 'AA']); // 'AA' has no title, ends up last
},
);
@@ -1869,7 +1888,9 @@ describe('DefaultEntitiesCatalog', () => {
limit: 10,
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['AA', 'BB']);
// simulate a situation where stitching is not yet complete
@@ -1884,7 +1905,9 @@ describe('DefaultEntitiesCatalog', () => {
limit: 10,
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['BB']);
},
);
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts
+25 -23
View File
@@ -45,13 +45,14 @@ import {
import { Stitcher } from '../stitching/types';
import {
expandLegacyCompoundRelationRefsInResponse,
expandLegacyCompoundRelationsInEntity,
isQueryEntitiesCursorRequest,
isQueryEntitiesInitialRequest,
} from './util';
import { EntityFilter } from '@backstage/plugin-catalog-node';
import { LoggerService } from '@backstage/backend-plugin-api';
import { applyEntityFilterToQuery } from './request/applyEntityFilterToQuery';
import { processRawEntitiesResult } from './response';
const DEFAULT_LIMIT = 200;
@@ -104,15 +105,18 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
private readonly database: Knex;
private readonly logger: LoggerService;
private readonly stitcher: Stitcher;
private readonly enableRawJson: boolean;
constructor(options: {
database: Knex;
logger: LoggerService;
stitcher: Stitcher;
enableRawJson?: boolean;
}) {
this.database = options.database;
this.logger = options.logger;
this.stitcher = options.stitcher;
this.enableRawJson = Boolean(options.enableRawJson);
}
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
@@ -190,16 +194,19 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
};
}
let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!));
if (request?.fields) {
entities = entities.map(e => request.fields!(e));
}
expandLegacyCompoundRelationRefsInResponse(entities);
return {
entities,
entities: processRawEntitiesResult(
rows.map(r => r.final_entity!),
this.enableRawJson
? request?.fields
: e => {
expandLegacyCompoundRelationsInEntity(e);
if (request?.fields) {
return request.fields(e);
}
return e;
},
),
pageInfo,
};
}
@@ -207,7 +214,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
async entitiesBatch(
request: EntitiesBatchRequest,
): Promise<EntitiesBatchResponse> {
const lookup = new Map<string, Entity>();
const lookup = new Map<string, string>();
for (const chunk of lodashChunk(request.entityRefs, 200)) {
let query = this.database<DbFinalEntitiesRow>('final_entities')
@@ -227,17 +234,13 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
}
for (const row of await query) {
lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null);
lookup.set(row.entityRef, row.entity ? row.entity : null);
}
}
let items = request.entityRefs.map(ref => lookup.get(ref) ?? null);
const items = request.entityRefs.map(ref => lookup.get(ref) ?? null);
if (request.fields) {
items = items.map(e => e && request.fields!(e));
}
return { items };
return { items: processRawEntitiesResult(items, request.fields) };
}
async queryEntities(
@@ -505,12 +508,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
}
: undefined;
const items = rows
.map(e => JSON.parse(e.final_entity!))
.map(e => (request.fields ? request.fields(e) : e));
return {
items,
items: processRawEntitiesResult(
rows.map(r => r.final_entity!),
request.fields,
),
pageInfo: {
...(!!prevCursor && { prevCursor }),
...(!!nextCursor && { nextCursor }),
plugins/catalog-backend/src/service/createRouter.test.ts
+18 -16
View File
@@ -136,7 +136,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [entities[0]],
items: { type: 'objects', entities: [entities[0]] },
pageInfo: {},
totalItems: 1,
});
@@ -149,7 +149,7 @@ describe('createRouter readonly disabled', () => {
it('parses single and multiple request parameters and passes them down', async () => {
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [],
items: { type: 'objects', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -185,7 +185,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'objects', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -203,7 +203,7 @@ describe('createRouter readonly disabled', () => {
it('parses initial request', async () => {
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [],
items: { type: 'objects', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -239,7 +239,7 @@ describe('createRouter readonly disabled', () => {
it('parses encoded params request', async () => {
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [],
items: { type: 'objects', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -283,7 +283,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'objects', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -312,7 +312,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'objects', entities: items },
totalItems: 100,
pageInfo: {
nextCursor: mockCursor({ fullTextFilter: { term: 'mySearch' } }),
@@ -354,7 +354,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'objects', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -379,7 +379,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'objects', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -402,7 +402,7 @@ describe('createRouter readonly disabled', () => {
},
};
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
entities: { type: 'objects', entities: [entity] },
pageInfo: { hasNextPage: false },
});
@@ -419,7 +419,7 @@ describe('createRouter readonly disabled', () => {
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue({
entities: [],
entities: { type: 'objects', entities: [] },
pageInfo: { hasNextPage: false },
});
@@ -446,7 +446,7 @@ describe('createRouter readonly disabled', () => {
},
};
entitiesCatalog.entitiesBatch.mockResolvedValue({
items: [entity],
items: { type: 'objects', entities: [entity] },
});
const response = await request(app).get('/entities/by-name/k/ns/n');
@@ -462,7 +462,7 @@ describe('createRouter readonly disabled', () => {
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entitiesBatch.mockResolvedValue({
items: [null],
items: { type: 'objects', entities: [null] },
});
const response = await request(app).get('/entities/by-name/b/d/c');
@@ -533,7 +533,9 @@ describe('createRouter readonly disabled', () => {
},
};
const entityRef = stringifyEntityRef(entity);
entitiesCatalog.entitiesBatch.mockResolvedValue({ items: [entity] });
entitiesCatalog.entitiesBatch.mockResolvedValue({
items: { type: 'objects', entities: [entity] },
});
const response = await request(app)
.post('/entities/by-refs?filter=kind=Component')
.set('Content-Type', 'application/json')
@@ -908,7 +910,7 @@ describe('createRouter readonly enabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [entities[0]],
items: { type: 'objects', entities: [entities[0]] },
pageInfo: {},
totalItems: 1,
});
@@ -1128,7 +1130,7 @@ describe('NextRouter permissioning', () => {
},
};
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [spideySense],
entities: { type: 'objects', entities: [spideySense] },
pageInfo: { hasNextPage: false },
});
plugins/catalog-backend/src/service/createRouter.ts
+23 -13
View File
@@ -44,7 +44,7 @@ import {
createEntityArrayJsonStream,
disallowReadonlyMode,
encodeCursor,
expandLegacyCompoundRelationRefsInResponse,
expandLegacyCompoundRelationsInEntity,
locationInput,
validateRequestBody,
} from './util';
@@ -60,6 +60,11 @@ import {
import { LocationAnalyzer } from '@backstage/plugin-catalog-node';
import { AuthorizedValidationService } from './AuthorizedValidationService';
import { DeferredPromise, createDeferred } from '@backstage/types';
import {
processEntitiesResponseItems,
writeEntitiesResponse,
writeSingleEntityResponse,
} from './response';
/**
* Options used by {@link createRouter}.
@@ -80,6 +85,7 @@ export interface RouterOptions {
auth: AuthService;
httpAuth: HttpAuthService;
permissionsService: PermissionsService;
enableRawJson?: boolean;
}
/**
@@ -107,6 +113,7 @@ export async function createRouter(
permissionsService,
auth,
httpAuth,
enableRawJson = false,
} = options;
const readonlyEnabled =
@@ -164,7 +171,7 @@ export async function createRouter(
res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`);
}
res.json(entities);
writeEntitiesResponse(res, entities);
return;
}
@@ -203,12 +210,17 @@ export async function createRouter(
: { credentials, fields, limit, cursor },
);
if (result.items.length) {
if (result.items.entities.length) {
await locks?.writeLock;
signal.throwIfAborted();
expandLegacyCompoundRelationRefsInResponse(result.items);
if (!enableRawJson) {
processEntitiesResponseItems(
result.items,
expandLegacyCompoundRelationsInEntity,
);
}
if (!responseStream.send(result.items)) {
// The kernel buffer is full. Create the lock but do not await it
// yet - we can better spend our time going to the next round of
@@ -241,8 +253,8 @@ export async function createRouter(
credentials: await httpAuth.credentials(req),
});
res.json({
items,
writeEntitiesResponse(res, items, entities => ({
items: entities,
totalItems,
pageInfo: {
...(pageInfo.nextCursor && {
@@ -252,7 +264,7 @@ export async function createRouter(
prevCursor: encodeCursor(pageInfo.prevCursor),
}),
},
});
}));
})
.get('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
@@ -260,10 +272,9 @@ export async function createRouter(
filter: basicEntityFilter({ 'metadata.uid': uid }),
credentials: await httpAuth.credentials(req),
});
if (!entities.length) {
if (!writeSingleEntityResponse(res, entities)) {
throw new NotFoundError(`No entity with uid ${uid}`);
}
res.status(200).json(entities[0]);
})
.delete('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
@@ -278,12 +289,11 @@ export async function createRouter(
entityRefs: [stringifyEntityRef({ kind, namespace, name })],
credentials: await httpAuth.credentials(req),
});
if (!items[0]) {
if (!writeSingleEntityResponse(res, items)) {
throw new NotFoundError(
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,
);
}
res.status(200).json(items[0]);
})
.get(
'/entities/by-name/:kind/:namespace/:name/ancestry',
@@ -298,13 +308,13 @@ export async function createRouter(
)
.post('/entities/by-refs', async (req, res) => {
const request = entitiesBatchRequest(req);
const response = await entitiesCatalog.entitiesBatch({
const { items } = await entitiesCatalog.entitiesBatch({
entityRefs: request.entityRefs,
filter: parseEntityFilterParams(req.query),
fields: parseEntityTransformParams(req.query, request.fields),
credentials: await httpAuth.credentials(req),
});
res.status(200).json(response);
writeEntitiesResponse(res, items, entities => ({ items: entities }));
})
.get('/entity-facets', async (req, res) => {
const response = await entitiesCatalog.facets({
plugins/catalog-backend/src/service/response/index.ts
+22
View File
@@ -0,0 +1,22 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {
processRawEntitiesResult,
processEntitiesResponseItems,
entitiesResponseToObjects,
} from './process';
export { writeSingleEntityResponse, writeEntitiesResponse } from './write';
plugins/catalog-backend/src/service/response/process.ts
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { EntitiesResponseItems } from '../../catalog/types';
export function processRawEntitiesResult(
serializedEntities: (string | null)[],
transform?: (entity: Entity) => Entity,
): EntitiesResponseItems {
if (transform) {
return {
type: 'objects',
entities: serializedEntities.map(e =>
e !== null ? transform(JSON.parse(e)) : e,
),
};
}
return {
type: 'raw',
entities: serializedEntities,
};
}
export function processEntitiesResponseItems(
response: EntitiesResponseItems,
transform?: (entity: Entity) => Entity,
) {
if (!transform) {
return response;
}
if (response.type === 'raw') {
return processRawEntitiesResult(response.entities, transform);
}
return {
type: 'objects',
entities: response.entities.map(e => (e !== null ? transform(e) : e)),
};
}
export function entitiesResponseToObjects(
response: EntitiesResponseItems,
): (Entity | null)[] {
if (response.type === 'objects') {
return response.entities;
}
return response.entities.map(e => (e !== null ? JSON.parse(e) : e));
}
plugins/catalog-backend/src/service/response/write.ts
+89
View File
@@ -0,0 +1,89 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Response } from 'express';
import { EntitiesResponseItems } from '../../catalog/types';
import { JsonValue } from '@backstage/types';
const JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
export function writeSingleEntityResponse(
res: Response,
response: EntitiesResponseItems,
): boolean {
const entity = response.entities[0];
if (!entity) {
return false;
}
if (typeof entity === 'string') {
res.setHeader('Content-Type', JSON_CONTENT_TYPE);
res.status(200);
res.write(entity);
} else {
res.json(entity);
}
return true;
}
export function writeEntitiesResponse(
res: Response,
response: EntitiesResponseItems,
responseWrapper?: (entities: JsonValue) => JsonValue,
) {
if (response.type === 'objects') {
res.json(
responseWrapper
? responseWrapper?.(response.entities)
: response.entities,
);
return;
}
res.setHeader('Content-Type', JSON_CONTENT_TYPE);
res.status(200);
// responseWrapper allows the caller to render the entities within an object
let trailing = '';
if (responseWrapper) {
const marker = `__MARKER_${Math.random().toString(36).slice(2, 10)}__`;
const wrapped = JSON.stringify(responseWrapper(marker));
const parts = wrapped.split(marker);
if (parts.length !== 2) {
throw new Error(
`Entity items response was incorrectly wrapped into ${parts.length} different parts`,
);
}
res.write(parts[0], 'utf8');
trailing = parts[1];
}
let first = true;
for (const entity of response.entities) {
if (first) {
res.write('[', 'utf8');
first = false;
} else {
res.write(',', 'utf8');
}
res.write(entity, 'utf8');
}
res.end(']');
if (trailing) {
res.write(trailing, 'utf8');
}
}
plugins/catalog-backend/src/service/util.ts
+34 -22
View File
@@ -20,6 +20,7 @@ import lodash from 'lodash';
import { z } from 'zod';
import {
Cursor,
EntitiesResponseItems,
QueryEntitiesCursorRequest,
QueryEntitiesInitialRequest,
QueryEntitiesRequest,
@@ -148,35 +149,32 @@ export function decodeCursor(encodedCursor: string) {
// sure that all adopters have re-stitched their entities so that the new
// targetRef field is present on them, and that they have stopped consuming
// the now-removed old field
// TODO(jhaals): Remove this in April 2022
export function expandLegacyCompoundRelationRefsInResponse(
entities: Entity[],
): void {
for (const entity of entities) {
if (entity.relations) {
for (const relation of entity.relations as any) {
if (!relation.targetRef && relation.target) {
// This is the case where an old-form entity, not yet stitched with
// the updated code, was in the database
relation.targetRef = stringifyEntityRef(relation.target);
} else if (!relation.target && relation.targetRef) {
// This is the case where a new-form entity, stitched with the
// updated code, was in the database but we still want to produce
// the old data shape as well for compatibility reasons
relation.target = parseEntityRef(relation.targetRef);
}
// TODO(patriko): Remove this in catalog 2.0
export function expandLegacyCompoundRelationsInEntity(entity: Entity): Entity {
if (entity.relations) {
for (const relation of entity.relations as any) {
if (!relation.targetRef && relation.target) {
// This is the case where an old-form entity, not yet stitched with
// the updated code, was in the database
relation.targetRef = stringifyEntityRef(relation.target);
} else if (!relation.target && relation.targetRef) {
// This is the case where a new-form entity, stitched with the
// updated code, was in the database but we still want to produce
// the old data shape as well for compatibility reasons
relation.target = parseEntityRef(relation.targetRef);
}
}
}
return entity;
}
export interface EntityArrayJsonStream {
send(entities: Entity[]): boolean;
send(entities: EntitiesResponseItems): boolean;
complete(): void;
close(): void;
}
// Helps stream Entity[] as a JSON response stream to avoid performance issues
// Helps stream EntitiesResponseItems[] as a JSON response stream to avoid performance issues
export function createEntityArrayJsonStream(
res: Response,
): EntityArrayJsonStream {
@@ -186,19 +184,33 @@ export function createEntityArrayJsonStream(
let completed = false;
return {
send(entities) {
send(response) {
if (firstSend) {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.status(200);
res.flushHeaders();
}
if (response.type === 'raw') {
let result = true;
for (const item of response.entities) {
if (firstSend) {
result ||= res.write('[');
firstSend = false;
} else {
result ||= res.write(',');
}
result ||= res.write(item);
}
return result;
}
let data: string;
if (prettyPrint) {
data = JSON.stringify(entities, null, 2);
data = JSON.stringify(response.entities, null, 2);
data = firstSend ? data.slice(0, -2) : `,\n${data.slice(2, -2)}`;
} else {
data = JSON.stringify(entities);
data = JSON.stringify(response.entities);
data = firstSend ? data.slice(0, -1) : `,${data.slice(1, -1)}`;
}
plugins/catalog-backend/src/tests/integration.test.ts
+7 -1
View File
@@ -55,6 +55,7 @@ import { DefaultStitcher } from '../stitching/DefaultStitcher';
import { mockServices } from '@backstage/backend-test-utils';
import { LoggerService } from '@backstage/backend-plugin-api';
import { DatabaseManager } from '@backstage/backend-common';
import { entitiesResponseToObjects } from '../service/response';
const voidLogger = mockServices.logger.mock();
@@ -365,7 +366,12 @@ class TestHarness {
async getOutputEntities(): Promise<Record<string, Entity>> {
const { entities } = await this.#catalog.entities();
return Object.fromEntries(entities.map(e => [stringifyEntityRef(e), e]));
return Object.fromEntries(
entitiesResponseToObjects(entities).map(e => [
stringifyEntityRef(e!),
e!,
]),
);
}
async refresh(options: RefreshOptions) {