paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 41 Packages Projects Releases Wiki Activity

Merge pull request #28121 from backstage/rugvip/noserial

Browse Source 
catalog-backend: avoid JSON parsing when possible
This commit is contained in:
Patrik Oldsberg
2024-12-13 13:54:46 +01:00
committed by GitHub
parent 9eb2b5e86d 5dcb0f3b7d
commit 1d1d7ad658
19 changed files with 1068 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/curly-teachers-marry.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': minor
---
Added a new `catalog.disableRelationsCompatibility` configuration option that avoids JSON deserialization and serialization if possible when reading entities. This significantly reduces the memory usage of the catalog, and slightly increases performance, but it removes the backwards compatibility processing that ensures that both `entity.relation[].target` and `entity.relation[].targetRef` are present in returned entities.
.github/vale/config/vocabularies/Backstage/accept.txt
+1
View File
@@ -97,6 +97,7 @@ deliverables
denormalized
dependabot
deps
deserialization
destructured
destructuring
Deutsche
plugins/catalog-backend/config.d.ts
Vendored
+10
View File
@@ -138,6 +138,16 @@ export interface Config {
}>;
}>;
/**
* Disables the compatibility layer for relations in returned entities that
* ensures that all relations objects have both `target` and `targetRef`.
*
* Enabling this option significantly reduces the memory usage of the
* catalog, and slightly increases performance, but may break consumers that
* rely on the existence of `target` in the relations objects.
*/
disableRelationsCompatibility?: boolean;
/**
* The strategy to use for entities that are orphaned, i.e. no longer have
* any other entities or providers referencing them. The default value is
plugins/catalog-backend/src/catalog/types.ts
+17 -3
View File
@@ -52,8 +52,22 @@ export type EntitiesRequest = {
credentials: BackstageCredentials;
};
/**
* Encapsulates either a deserialized or serialized entities to be sent in a response.
* @internal
*/
export type EntitiesResponseItems =
| {
type: 'object';
entities: (Entity | null)[];
}
| {
type: 'raw';
entities: (string | null)[];
};
export type EntitiesResponse = {
entities: Entity[];
entities: EntitiesResponseItems;
pageInfo: PageInfo;
};
@@ -86,7 +100,7 @@ export interface EntitiesBatchResponse {
* The list of entities, in the same order as the refs in the request. Entries
* that are null signify that no entity existed with that ref.
*/
items: Array<Entity | null>;
items: EntitiesResponseItems;
}
export type EntityAncestryResponse = {
@@ -224,7 +238,7 @@ export interface QueryEntitiesResponse {
/**
* The entities for the current pagination request
*/
items: Entity[];
items: EntitiesResponseItems;
pageInfo: {
/**
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts
+13 -8
View File
@@ -64,7 +64,7 @@ describe('AuthorizedEntitiesCatalog', () => {
credentials: mockCredentials.none(),
}),
).toEqual({
entities: [],
entities: { type: 'object', entities: [] },
pageInfo: { hasNextPage: false },
});
});
@@ -113,7 +113,7 @@ describe('AuthorizedEntitiesCatalog', () => {
credentials: mockCredentials.none(),
}),
).resolves.toEqual({
items: [null],
items: { type: 'object', entities: [null] },
});
expect(fakeCatalog.entitiesBatch).not.toHaveBeenCalled();
@@ -174,7 +174,7 @@ describe('AuthorizedEntitiesCatalog', () => {
filter: { key: 'kind', values: ['b'] },
}),
).resolves.toEqual({
items: [],
items: { type: 'object', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -226,7 +226,7 @@ describe('AuthorizedEntitiesCatalog', () => {
];
fakeCatalog.queryEntities.mockResolvedValue({
items: entities,
items: { type: 'object', entities },
pageInfo: {
nextCursor: {
isPrevious: false,
@@ -254,7 +254,7 @@ describe('AuthorizedEntitiesCatalog', () => {
});
expect(response).toEqual({
items: entities,
items: { type: 'object', entities: entities },
totalItems: 4,
pageInfo: {
nextCursor: {
@@ -290,7 +290,7 @@ describe('AuthorizedEntitiesCatalog', () => {
});
expect(response).toEqual({
items: entities,
items: { type: 'object', entities: entities },
totalItems: 4,
pageInfo: {
nextCursor: {
@@ -338,7 +338,9 @@ describe('AuthorizedEntitiesCatalog', () => {
conditions: { rule: 'IS_ENTITY_KIND', params: { kinds: ['b'] } },
},
]);
fakeCatalog.entities.mockResolvedValue({ entities: [] });
fakeCatalog.entities.mockResolvedValue({
entities: { type: 'object', entities: [] },
});
const catalog = new AuthorizedEntitiesCatalog(
fakeCatalog,
fakePermissionApi,
@@ -360,7 +362,10 @@ describe('AuthorizedEntitiesCatalog', () => {
},
]);
fakeCatalog.entities.mockResolvedValue({
entities: [{ kind: 'b', namespace: 'default', name: 'my-component' }],
entities: {
type: 'object',
entities: [{ kind: 'b', namespace: 'default', name: 'my-component' }],
},
});
const catalog = new AuthorizedEntitiesCatalog(
fakeCatalog,
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts
+7 -4
View File
@@ -60,7 +60,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
entities: [],
entities: { type: 'object', entities: [] },
pageInfo: { hasNextPage: false },
};
}
@@ -92,7 +92,10 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
items: new Array(request.entityRefs.length).fill(null),
items: {
type: 'object',
entities: new Array(request.entityRefs.length).fill(null),
},
};
}
@@ -123,7 +126,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
items: [],
items: { type: 'object', entities: [] },
pageInfo: {},
totalItems: 0,
};
@@ -208,7 +211,7 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
allOf: [permissionFilter, basicEntityFilter({ 'metadata.uid': uid })],
},
});
if (entities.length === 0) {
if (entities.entities.length === 0) {
throw new NotAllowedError();
}
}
plugins/catalog-backend/src/service/CatalogBuilder.ts
+11 -1
View File
@@ -115,6 +115,7 @@ import {
UrlReaderService,
SchedulerService,
} from '@backstage/backend-plugin-api';
import { entitiesResponseToObjects } from './response';
/**
* This is a duplicate of the alpha `CatalogPermissionRule` type, for use in the stable API.
@@ -485,6 +486,10 @@ export class CatalogBuilder {
discovery,
});
const disableRelationsCompatibility = config.getOptionalBoolean(
'catalog.disableRelationsCompatibility',
);
const policy = this.buildEntityPolicy();
const processors = this.buildProcessors();
const parser = this.parser || defaultEntityDataParser;
@@ -521,6 +526,7 @@ export class CatalogBuilder {
database: dbClient,
logger,
stitcher,
disableRelationsCompatibility,
});
let permissionsService: PermissionsService;
@@ -566,7 +572,10 @@ export class CatalogBuilder {
},
});
const entitiesByRef = keyBy(entities, stringifyEntityRef);
const entitiesByRef = keyBy(
entitiesResponseToObjects(entities),
stringifyEntityRef,
);
return resourceRefs.map(
resourceRef =>
@@ -629,6 +638,7 @@ export class CatalogBuilder {
auth,
httpAuth,
permissionsService,
disableRelationsCompatibility,
});
await connectEntityProviders(providerDatabase, entityProviders);
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+139 -57
View File
@@ -38,6 +38,7 @@ import { Stitcher } from '../stitching/types';
import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog';
import { EntitiesRequest } from '../catalog/types';
import { buildEntitySearch } from '../database/operations/stitcher/buildEntitySearch';
import { entitiesResponseToObjects } from './response';
jest.setTimeout(60_000);
@@ -310,10 +311,11 @@ describe('DefaultEntitiesCatalog', () => {
const testFilter = {
key: 'spec.test',
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: testFilter,
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity2);
@@ -351,10 +353,11 @@ describe('DefaultEntitiesCatalog', () => {
key: 'spec.test',
},
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: testFilter,
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(1);
expect(entities[0]).toEqual(entity1);
@@ -416,7 +419,7 @@ describe('DefaultEntitiesCatalog', () => {
values: ['red'],
},
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: {
allOf: [
testFilter1,
@@ -427,6 +430,7 @@ describe('DefaultEntitiesCatalog', () => {
},
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(2);
expect(entities).toContainEqual(entity2);
@@ -465,7 +469,7 @@ describe('DefaultEntitiesCatalog', () => {
const testFilter2 = {
key: 'metadata.desc',
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: {
not: {
allOf: [testFilter1, testFilter2],
@@ -474,6 +478,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(1);
expect(entities).toContainEqual(entity1);
@@ -509,10 +514,11 @@ describe('DefaultEntitiesCatalog', () => {
key: 'kind',
values: [],
};
const { entities } = await catalog.entities({
const res = await catalog.entities({
filter: testFilter,
credentials: mockCredentials.none(),
});
const entities = entitiesResponseToObjects(res.entities);
expect(entities.length).toBe(0);
},
@@ -553,10 +559,11 @@ describe('DefaultEntitiesCatalog', () => {
stitcher,
});
const { entities } = await catalog.entities();
const res = await catalog.entities();
const entities = entitiesResponseToObjects(res.entities);
expect(
entities.find(e => e.metadata.name === 'one')!.relations,
entities.find(e => e?.metadata.name === 'one')!.relations,
).toEqual([
{
type: 'r',
@@ -565,7 +572,7 @@ describe('DefaultEntitiesCatalog', () => {
},
]);
expect(
entities.find(e => e.metadata.name === 'two')!.relations,
entities.find(e => e?.metadata.name === 'two')!.relations,
).toEqual([
{
type: 'r',
@@ -615,7 +622,9 @@ describe('DefaultEntitiesCatalog', () => {
return catalog
.entities({ ...request, credentials: mockCredentials.none() })
.then(response =>
response.entities.map(e => e.metadata.name).toSorted(),
entitiesResponseToObjects(response.entities)
.map(e => e!.metadata.name)
.toSorted(),
);
}
@@ -678,7 +687,11 @@ describe('DefaultEntitiesCatalog', () => {
): Promise<string[]> {
return catalog
.entities({ ...request, credentials: mockCredentials.none() })
.then(response => response.entities.map(e => e.metadata.name));
.then(response =>
entitiesResponseToObjects(response.entities).map(
e => e!.metadata.name,
),
);
}
await expect(
@@ -764,7 +777,7 @@ describe('DefaultEntitiesCatalog', () => {
stitcher,
});
const { items } = await catalog.entitiesBatch({
const res = await catalog.entitiesBatch({
entityRefs: [
'k:default/two',
'k:default/one',
@@ -775,6 +788,7 @@ describe('DefaultEntitiesCatalog', () => {
],
credentials: mockCredentials.none(),
});
const items = entitiesResponseToObjects(res.items);
expect(items.map(e => e && stringifyEntityRef(e))).toEqual([
'k:default/two',
@@ -819,11 +833,12 @@ describe('DefaultEntitiesCatalog', () => {
stitcher,
});
const { items } = await catalog.entitiesBatch({
const res = await catalog.entitiesBatch({
entityRefs: ['k:default/two', 'k:default/one'],
filter: { key: 'spec.owner', values: ['me'] },
credentials: mockCredentials.none(),
});
const items = entitiesResponseToObjects(res.items);
expect(items.map(e => e && stringifyEntityRef(e))).toEqual([
'k:default/two',
@@ -881,7 +896,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response1 = await catalog.queryEntities(request1);
expect(response1.items).toEqual([entityFrom('A'), entityFrom('B')]);
expect(entitiesResponseToObjects(response1.items)).toEqual([
entityFrom('A'),
entityFrom('B'),
]);
expect(response1.pageInfo.nextCursor).toBeDefined();
expect(response1.pageInfo.prevCursor).toBeUndefined();
expect(response1.totalItems).toBe(names.length);
@@ -893,7 +911,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response2 = await catalog.queryEntities(request2);
expect(response2.items).toEqual([entityFrom('C'), entityFrom('D')]);
expect(entitiesResponseToObjects(response2.items)).toEqual([
entityFrom('C'),
entityFrom('D'),
]);
expect(response2.pageInfo.nextCursor).toBeDefined();
expect(response2.pageInfo.prevCursor).toBeDefined();
expect(response2.totalItems).toBe(names.length);
@@ -905,7 +926,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response3 = await catalog.queryEntities(request3);
expect(response3.items).toEqual([entityFrom('E'), entityFrom('F')]);
expect(entitiesResponseToObjects(response3.items)).toEqual([
entityFrom('E'),
entityFrom('F'),
]);
expect(response3.pageInfo.nextCursor).toBeDefined();
expect(response3.pageInfo.prevCursor).toBeDefined();
expect(response3.totalItems).toBe(names.length);
@@ -917,7 +941,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response4 = await catalog.queryEntities(request4);
expect(response4.items).toEqual([entityFrom('C'), entityFrom('D')]);
expect(entitiesResponseToObjects(response4.items)).toEqual([
entityFrom('C'),
entityFrom('D'),
]);
expect(response4.pageInfo.nextCursor).toBeDefined();
expect(response4.pageInfo.prevCursor).toBeDefined();
expect(response4.totalItems).toBe(names.length);
@@ -929,7 +956,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response5 = await catalog.queryEntities(request5);
expect(response5.items).toEqual([entityFrom('A'), entityFrom('B')]);
expect(entitiesResponseToObjects(response5.items)).toEqual([
entityFrom('A'),
entityFrom('B'),
]);
expect(response5.pageInfo.nextCursor).toBeDefined();
expect(response5.pageInfo.prevCursor).toBeUndefined();
expect(response5.totalItems).toBe(names.length);
@@ -941,7 +971,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response6 = await catalog.queryEntities(request6);
expect(response6.items).toEqual([entityFrom('C'), entityFrom('D')]);
expect(entitiesResponseToObjects(response6.items)).toEqual([
entityFrom('C'),
entityFrom('D'),
]);
expect(response6.pageInfo.nextCursor).toBeDefined();
expect(response6.pageInfo.prevCursor).toBeDefined();
expect(response6.totalItems).toBe(names.length);
@@ -953,7 +986,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response7 = await catalog.queryEntities(request7);
expect(response7.items).toEqual([entityFrom('E'), entityFrom('F')]);
expect(entitiesResponseToObjects(response7.items)).toEqual([
entityFrom('E'),
entityFrom('F'),
]);
expect(response7.pageInfo.nextCursor).toBeDefined();
expect(response7.pageInfo.prevCursor).toBeDefined();
expect(response7.totalItems).toBe(names.length);
@@ -965,7 +1001,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response7bis = await catalog.queryEntities(request7bis);
expect(response7bis.items).toEqual([
expect(entitiesResponseToObjects(response7bis.items)).toEqual([
entityFrom('E'),
entityFrom('F'),
entityFrom('G'),
@@ -981,7 +1017,9 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response8 = await catalog.queryEntities(request8);
expect(response8.items).toEqual([entityFrom('G')]);
expect(entitiesResponseToObjects(response8.items)).toEqual([
entityFrom('G'),
]);
expect(response8.pageInfo.nextCursor).toBeUndefined();
expect(response8.pageInfo.prevCursor).toBeDefined();
expect(response8.totalItems).toBe(names.length);
@@ -1035,7 +1073,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response1 = await catalog.queryEntities(request1);
expect(response1.items).toEqual([entityFrom('G'), entityFrom('F')]);
expect(entitiesResponseToObjects(response1.items)).toEqual([
entityFrom('G'),
entityFrom('F'),
]);
expect(response1.pageInfo.nextCursor).toBeDefined();
expect(response1.pageInfo.prevCursor).toBeUndefined();
expect(response1.totalItems).toBe(names.length);
@@ -1047,7 +1088,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response2 = await catalog.queryEntities(request2);
expect(response2.items).toEqual([entityFrom('E'), entityFrom('D')]);
expect(entitiesResponseToObjects(response2.items)).toEqual([
entityFrom('E'),
entityFrom('D'),
]);
expect(response2.pageInfo.nextCursor).toBeDefined();
expect(response2.pageInfo.prevCursor).toBeDefined();
expect(response2.totalItems).toBe(names.length);
@@ -1059,7 +1103,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response3 = await catalog.queryEntities(request3);
expect(response3.items).toEqual([entityFrom('C'), entityFrom('B')]);
expect(entitiesResponseToObjects(response3.items)).toEqual([
entityFrom('C'),
entityFrom('B'),
]);
expect(response3.pageInfo.nextCursor).toBeDefined();
expect(response3.pageInfo.prevCursor).toBeDefined();
expect(response3.totalItems).toBe(names.length);
@@ -1072,7 +1119,10 @@ describe('DefaultEntitiesCatalog', () => {
};
const response4 = await catalog.queryEntities(request4);
expect(response4.items).toEqual([entityFrom('E'), entityFrom('D')]);
expect(entitiesResponseToObjects(response4.items)).toEqual([
entityFrom('E'),
entityFrom('D'),
]);
expect(response4.pageInfo.nextCursor).toBeDefined();
expect(response4.pageInfo.prevCursor).toBeDefined();
expect(response4.totalItems).toBe(names.length);
@@ -1084,7 +1134,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response5 = await catalog.queryEntities(request5);
expect(response5.items).toEqual([entityFrom('G'), entityFrom('F')]);
expect(entitiesResponseToObjects(response5.items)).toEqual([
entityFrom('G'),
entityFrom('F'),
]);
expect(response5.pageInfo.nextCursor).toBeDefined();
expect(response5.pageInfo.prevCursor).toBeUndefined();
expect(response5.totalItems).toBe(names.length);
@@ -1096,7 +1149,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response6 = await catalog.queryEntities(request6);
expect(response6.items).toEqual([entityFrom('E'), entityFrom('D')]);
expect(entitiesResponseToObjects(response6.items)).toEqual([
entityFrom('E'),
entityFrom('D'),
]);
expect(response6.pageInfo.nextCursor).toBeDefined();
expect(response6.pageInfo.prevCursor).toBeDefined();
expect(response6.totalItems).toBe(names.length);
@@ -1108,7 +1164,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response7 = await catalog.queryEntities(request7);
expect(response7.items).toEqual([entityFrom('C'), entityFrom('B')]);
expect(entitiesResponseToObjects(response7.items)).toEqual([
entityFrom('C'),
entityFrom('B'),
]);
expect(response7.pageInfo.nextCursor).toBeDefined();
expect(response7.pageInfo.prevCursor).toBeDefined();
expect(response7.totalItems).toBe(names.length);
@@ -1120,7 +1179,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response7bis = await catalog.queryEntities(request7bis);
expect(response7bis.items).toEqual([
expect(entitiesResponseToObjects(response7bis.items)).toEqual([
entityFrom('C'),
entityFrom('B'),
entityFrom('A'),
@@ -1136,7 +1195,9 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response8 = await catalog.queryEntities(request8);
expect(response8.items).toEqual([entityFrom('A')]);
expect(entitiesResponseToObjects(response8.items)).toEqual([
entityFrom('A'),
]);
expect(response8.pageInfo.nextCursor).toBeUndefined();
expect(response8.pageInfo.prevCursor).toBeDefined();
expect(response8.totalItems).toBe(names.length);
@@ -1188,7 +1249,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response = await catalog.queryEntities(request);
expect(response.items).toEqual([
expect(entitiesResponseToObjects(response.items)).toEqual([
entityFrom('atcatss'),
entityFrom('cat'),
entityFrom('dogcat'),
@@ -1246,7 +1307,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response = await catalog.queryEntities(request);
expect(response.items).toEqual(entities);
expect(entitiesResponseToObjects(response.items)).toEqual(entities);
expect(response.pageInfo.nextCursor).toBeUndefined();
expect(response.pageInfo.prevCursor).toBeUndefined();
expect(response.totalItems).toBe(1);
@@ -1301,7 +1362,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response = await catalog.queryEntities(request);
expect(response.items).toEqual([
expect(entitiesResponseToObjects(response.items)).toEqual([
entityFrom('1', { uid: 'id1', title: 'cat' }),
entityFrom('2', { uid: 'id2', title: 'atcatss' }),
entityFrom('4', { uid: 'id4', title: 'dogcat' }),
@@ -1314,7 +1375,7 @@ describe('DefaultEntitiesCatalog', () => {
...request,
limit: 2,
});
expect(paginatedResponse.items).toEqual([
expect(entitiesResponseToObjects(paginatedResponse.items)).toEqual([
entityFrom('1', { uid: 'id1', title: 'cat' }),
entityFrom('2', { uid: 'id2', title: 'atcatss' }),
]);
@@ -1326,7 +1387,7 @@ describe('DefaultEntitiesCatalog', () => {
cursor: paginatedResponse.pageInfo.nextCursor!,
credentials: mockCredentials.none(),
});
expect(paginatedResponseNext.items).toEqual([
expect(entitiesResponseToObjects(paginatedResponseNext.items)).toEqual([
entityFrom('4', { uid: 'id4', title: 'dogcat' }),
]);
expect(paginatedResponseNext.pageInfo.nextCursor).toBeUndefined();
@@ -1404,7 +1465,7 @@ describe('DefaultEntitiesCatalog', () => {
};
const response = await catalog.queryEntities(request);
expect(response.items).toEqual([
expect(entitiesResponseToObjects(response.items)).toEqual([
entityFrom('KingOfTheJungle', { uid: 'id0', title: 'lion' }),
entityFrom('NotKingOfTheJungle', { uid: 'id1', title: 'cat' }),
entityFrom('NotACatKing', { uid: 'id2', title: 'atcatss' }),
@@ -1418,7 +1479,7 @@ describe('DefaultEntitiesCatalog', () => {
...request,
limit: 2,
});
expect(paginatedResponse.items).toEqual([
expect(entitiesResponseToObjects(paginatedResponse.items)).toEqual([
entityFrom('KingOfTheJungle', { uid: 'id0', title: 'lion' }),
entityFrom('NotKingOfTheJungle', { uid: 'id1', title: 'cat' }),
]);
@@ -1430,7 +1491,7 @@ describe('DefaultEntitiesCatalog', () => {
cursor: paginatedResponse.pageInfo.nextCursor!,
credentials: mockCredentials.none(),
});
expect(paginatedResponseNext.items).toEqual([
expect(entitiesResponseToObjects(paginatedResponseNext.items)).toEqual([
entityFrom('NotACatKing', { uid: 'id2', title: 'atcatss' }),
entityFrom('123', { uid: 'id3', title: 'king' }),
]);
@@ -1474,7 +1535,11 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response = await catalog.queryEntities(request);
expect(response).toEqual({ totalItems: 20, items: [], pageInfo: {} });
expect(response).toEqual({
totalItems: 20,
items: { type: 'raw', entities: [] },
pageInfo: {},
});
},
);
@@ -1509,7 +1574,10 @@ describe('DefaultEntitiesCatalog', () => {
let response = await catalog.queryEntities(request);
expect(response).toEqual({
totalItems: 0,
items: expect.objectContaining({ length: 10 }),
items: {
type: 'raw',
entities: expect.objectContaining({ length: 10 }),
},
pageInfo: { nextCursor: expect.anything() },
});
response = await catalog.queryEntities({
@@ -1518,7 +1586,10 @@ describe('DefaultEntitiesCatalog', () => {
});
expect(response).toEqual({
totalItems: 0,
items: expect.objectContaining({ length: 5 }),
items: {
type: 'raw',
entities: expect.objectContaining({ length: 5 }),
},
pageInfo: { prevCursor: expect.anything() },
});
},
@@ -1553,7 +1624,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response1 = await catalog.queryEntities(request1);
expect(response1.items).toMatchObject([
expect(entitiesResponseToObjects(response1.items)).toMatchObject([
entityFrom('AA'),
entityFrom('AA'),
]);
@@ -1568,7 +1639,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response2 = await catalog.queryEntities(request2);
expect(response2.items).toMatchObject([
expect(entitiesResponseToObjects(response2.items)).toMatchObject([
entityFrom('AA'),
entityFrom('AA'),
]);
@@ -1583,7 +1654,10 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response3 = await catalog.queryEntities(request3);
expect(response3.items).toEqual([entityFrom('CC'), entityFrom('DD')]);
expect(entitiesResponseToObjects(response3.items)).toEqual([
entityFrom('CC'),
entityFrom('DD'),
]);
expect(response3.pageInfo.nextCursor).toBeUndefined();
expect(response3.pageInfo.prevCursor).toBeDefined();
expect(response3.totalItems).toBe(6);
@@ -1595,7 +1669,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response4 = await catalog.queryEntities(request4);
expect(response4.items).toMatchObject([
expect(entitiesResponseToObjects(response4.items)).toMatchObject([
entityFrom('AA'),
entityFrom('AA'),
]);
@@ -1610,7 +1684,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response5 = await catalog.queryEntities(request5);
expect(response5.items).toMatchObject([
expect(entitiesResponseToObjects(response5.items)).toMatchObject([
entityFrom('AA'),
entityFrom('AA'),
]);
@@ -1678,7 +1752,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response1 = await catalog.queryEntities(request1);
expect(response1.items).toMatchObject([
expect(entitiesResponseToObjects(response1.items)).toMatchObject([
entityFrom('AA', { uid: '1', kind: 'included' }),
entityFrom('AA', { uid: '2', kind: 'included' }),
]);
@@ -1693,7 +1767,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response2 = await catalog.queryEntities(request2);
expect(response2.items).toMatchObject([
expect(entitiesResponseToObjects(response2.items)).toMatchObject([
entityFrom('AA', { uid: '4', kind: 'included' }),
entityFrom('AA', { uid: '5', kind: 'included' }),
]);
@@ -1737,7 +1811,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response1 = await catalog.queryEntities(request1);
expect(response1.items).toMatchObject([
expect(entitiesResponseToObjects(response1.items)).toMatchObject([
entityFrom('AA'),
entityFrom('CC'),
]);
@@ -1752,7 +1826,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response2 = await catalog.queryEntities(request2);
expect(response2.items).toMatchObject([
expect(entitiesResponseToObjects(response2.items)).toMatchObject([
entityFrom('DD'),
entityFrom('AA', { namespace: 'namespace2' }),
]);
@@ -1767,7 +1841,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response3 = await catalog.queryEntities(request3);
expect(response3.items).toMatchObject([
expect(entitiesResponseToObjects(response3.items)).toMatchObject([
entityFrom('AA', { namespace: 'namespace3' }),
entityFrom('AA', { namespace: 'namespace4' }),
]);
@@ -1782,7 +1856,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response4 = await catalog.queryEntities(request4);
expect(response4.items).toMatchObject([
expect(entitiesResponseToObjects(response4.items)).toMatchObject([
entityFrom('DD'),
entityFrom('AA', { namespace: 'namespace2' }),
]);
@@ -1797,7 +1871,7 @@ describe('DefaultEntitiesCatalog', () => {
credentials: mockCredentials.none(),
};
const response5 = await catalog.queryEntities(request5);
expect(response5.items).toMatchObject([
expect(entitiesResponseToObjects(response5.items)).toMatchObject([
entityFrom('AA'),
entityFrom('CC'),
]);
@@ -1830,7 +1904,9 @@ describe('DefaultEntitiesCatalog', () => {
orderFields: [{ field: 'metadata.title', order: 'asc' }],
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['CC', 'BB', 'AA']); // 'AA' has no title, ends up last
await expect(
@@ -1839,7 +1915,9 @@ describe('DefaultEntitiesCatalog', () => {
orderFields: [{ field: 'metadata.title', order: 'desc' }],
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['BB', 'CC', 'AA']); // 'AA' has no title, ends up last
},
);
@@ -1869,7 +1947,9 @@ describe('DefaultEntitiesCatalog', () => {
limit: 10,
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['AA', 'BB']);
// simulate a situation where stitching is not yet complete
@@ -1884,7 +1964,9 @@ describe('DefaultEntitiesCatalog', () => {
limit: 10,
credentials: mockCredentials.none(),
})
.then(r => r.items.map(e => e.metadata.name)),
.then(r =>
entitiesResponseToObjects(r.items).map(e => e!.metadata.name),
),
).resolves.toEqual(['BB']);
},
);
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts
+27 -23
View File
@@ -45,13 +45,14 @@ import {
import { Stitcher } from '../stitching/types';
import {
expandLegacyCompoundRelationRefsInResponse,
expandLegacyCompoundRelationsInEntity,
isQueryEntitiesCursorRequest,
isQueryEntitiesInitialRequest,
} from './util';
import { EntityFilter } from '@backstage/plugin-catalog-node';
import { LoggerService } from '@backstage/backend-plugin-api';
import { applyEntityFilterToQuery } from './request/applyEntityFilterToQuery';
import { processRawEntitiesResult } from './response';
const DEFAULT_LIMIT = 200;
@@ -104,15 +105,20 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
private readonly database: Knex;
private readonly logger: LoggerService;
private readonly stitcher: Stitcher;
private readonly disableRelationsCompatibility: boolean;
constructor(options: {
database: Knex;
logger: LoggerService;
stitcher: Stitcher;
disableRelationsCompatibility?: boolean;
}) {
this.database = options.database;
this.logger = options.logger;
this.stitcher = options.stitcher;
this.disableRelationsCompatibility = Boolean(
options.disableRelationsCompatibility,
);
}
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
@@ -190,16 +196,19 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
};
}
let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!));
if (request?.fields) {
entities = entities.map(e => request.fields!(e));
}
expandLegacyCompoundRelationRefsInResponse(entities);
return {
entities,
entities: processRawEntitiesResult(
rows.map(r => r.final_entity!),
this.disableRelationsCompatibility
? request?.fields
: e => {
expandLegacyCompoundRelationsInEntity(e);
if (request?.fields) {
return request.fields(e);
}
return e;
},
),
pageInfo,
};
}
@@ -207,7 +216,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
async entitiesBatch(
request: EntitiesBatchRequest,
): Promise<EntitiesBatchResponse> {
const lookup = new Map<string, Entity>();
const lookup = new Map<string, string>();
for (const chunk of lodashChunk(request.entityRefs, 200)) {
let query = this.database<DbFinalEntitiesRow>('final_entities')
@@ -227,17 +236,13 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
}
for (const row of await query) {
lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null);
lookup.set(row.entityRef, row.entity ? row.entity : null);
}
}
let items = request.entityRefs.map(ref => lookup.get(ref) ?? null);
const items = request.entityRefs.map(ref => lookup.get(ref) ?? null);
if (request.fields) {
items = items.map(e => e && request.fields!(e));
}
return { items };
return { items: processRawEntitiesResult(items, request.fields) };
}
async queryEntities(
@@ -505,12 +510,11 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
}
: undefined;
const items = rows
.map(e => JSON.parse(e.final_entity!))
.map(e => (request.fields ? request.fields(e) : e));
return {
items,
items: processRawEntitiesResult(
rows.map(r => r.final_entity!),
request.fields,
),
pageInfo: {
...(!!prevCursor && { prevCursor }),
...(!!nextCursor && { nextCursor }),
plugins/catalog-backend/src/service/createRouter.test.ts
+20 -17
View File
@@ -136,7 +136,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [entities[0]],
items: { type: 'object', entities: [entities[0]] },
pageInfo: {},
totalItems: 1,
});
@@ -149,7 +149,7 @@ describe('createRouter readonly disabled', () => {
it('parses single and multiple request parameters and passes them down', async () => {
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [],
items: { type: 'object', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -185,7 +185,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'object', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -203,7 +203,7 @@ describe('createRouter readonly disabled', () => {
it('parses initial request', async () => {
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [],
items: { type: 'object', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -239,7 +239,7 @@ describe('createRouter readonly disabled', () => {
it('parses encoded params request', async () => {
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [],
items: { type: 'object', entities: [] },
pageInfo: {},
totalItems: 0,
});
@@ -283,7 +283,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'object', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -312,7 +312,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'object', entities: items },
totalItems: 100,
pageInfo: {
nextCursor: mockCursor({ fullTextFilter: { term: 'mySearch' } }),
@@ -354,7 +354,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'object', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -379,7 +379,7 @@ describe('createRouter readonly disabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items,
items: { type: 'object', entities: items },
totalItems: 100,
pageInfo: { nextCursor: mockCursor() },
});
@@ -402,7 +402,7 @@ describe('createRouter readonly disabled', () => {
},
};
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
entities: { type: 'object', entities: [entity] },
pageInfo: { hasNextPage: false },
});
@@ -419,7 +419,7 @@ describe('createRouter readonly disabled', () => {
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue({
entities: [],
entities: { type: 'object', entities: [] },
pageInfo: { hasNextPage: false },
});
@@ -446,7 +446,7 @@ describe('createRouter readonly disabled', () => {
},
};
entitiesCatalog.entitiesBatch.mockResolvedValue({
items: [entity],
items: { type: 'object', entities: [entity] },
});
const response = await request(app).get('/entities/by-name/k/ns/n');
@@ -462,7 +462,7 @@ describe('createRouter readonly disabled', () => {
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entitiesBatch.mockResolvedValue({
items: [null],
items: { type: 'object', entities: [null] },
});
const response = await request(app).get('/entities/by-name/b/d/c');
@@ -533,7 +533,9 @@ describe('createRouter readonly disabled', () => {
},
};
const entityRef = stringifyEntityRef(entity);
entitiesCatalog.entitiesBatch.mockResolvedValue({ items: [entity] });
entitiesCatalog.entitiesBatch.mockResolvedValue({
items: { type: 'object', entities: [entity] },
});
const response = await request(app)
.post('/entities/by-refs?filter=kind=Component')
.set('Content-Type', 'application/json')
@@ -858,7 +860,7 @@ describe('createRouter readonly disabled', () => {
});
});
describe('createRouter readonly enabled', () => {
describe('createRouter readonly and raw json enabled', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let app: express.Express;
let locationService: jest.Mocked<LocationService>;
@@ -881,6 +883,7 @@ describe('createRouter readonly enabled', () => {
getLocationByEntity: jest.fn(),
};
const router = await createRouter({
disableRelationsCompatibility: true,
entitiesCatalog,
locationService,
logger: mockServices.logger.mock(),
@@ -908,7 +911,7 @@ describe('createRouter readonly enabled', () => {
];
entitiesCatalog.queryEntities.mockResolvedValueOnce({
items: [entities[0]],
items: { type: 'raw', entities: [JSON.stringify(entities[0])] },
pageInfo: {},
totalItems: 1,
});
@@ -1128,7 +1131,7 @@ describe('NextRouter permissioning', () => {
},
};
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [spideySense],
entities: { type: 'object', entities: [spideySense] },
pageInfo: { hasNextPage: false },
});
plugins/catalog-backend/src/service/createRouter.ts
+31 -21
View File
@@ -23,7 +23,7 @@ import {
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { InputError, NotFoundError, serializeError } from '@backstage/errors';
import { InputError, serializeError } from '@backstage/errors';
import express from 'express';
import yn from 'yn';
import { z } from 'zod';
@@ -41,10 +41,9 @@ import { parseEntityFacetParams } from './request/parseEntityFacetParams';
import { parseEntityOrderParams } from './request/parseEntityOrderParams';
import { LocationService, RefreshService } from './types';
import {
createEntityArrayJsonStream,
disallowReadonlyMode,
encodeCursor,
expandLegacyCompoundRelationRefsInResponse,
expandLegacyCompoundRelationsInEntity,
locationInput,
validateRequestBody,
} from './util';
@@ -60,6 +59,12 @@ import {
import { LocationAnalyzer } from '@backstage/plugin-catalog-node';
import { AuthorizedValidationService } from './AuthorizedValidationService';
import { DeferredPromise, createDeferred } from '@backstage/types';
import {
createEntityArrayJsonStream,
processEntitiesResponseItems,
writeEntitiesResponse,
writeSingleEntityResponse,
} from './response';
/**
* Options used by {@link createRouter}.
@@ -80,6 +85,7 @@ export interface RouterOptions {
auth: AuthService;
httpAuth: HttpAuthService;
permissionsService: PermissionsService;
disableRelationsCompatibility?: boolean;
}
/**
@@ -107,6 +113,7 @@ export async function createRouter(
permissionsService,
auth,
httpAuth,
disableRelationsCompatibility = false,
} = options;
const readonlyEnabled =
@@ -164,7 +171,7 @@ export async function createRouter(
res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`);
}
res.json(entities);
await writeEntitiesResponse(res, entities);
return;
}
@@ -203,12 +210,17 @@ export async function createRouter(
: { credentials, fields, limit, cursor },
);
if (result.items.length) {
if (result.items.entities.length) {
await locks?.writeLock;
signal.throwIfAborted();
expandLegacyCompoundRelationRefsInResponse(result.items);
if (!disableRelationsCompatibility) {
result.items = processEntitiesResponseItems(
result.items,
expandLegacyCompoundRelationsInEntity,
);
}
if (!responseStream.send(result.items)) {
// The kernel buffer is full. Create the lock but do not await it
// yet - we can better spend our time going to the next round of
@@ -241,8 +253,8 @@ export async function createRouter(
credentials: await httpAuth.credentials(req),
});
res.json({
items,
await writeEntitiesResponse(res, items, entities => ({
items: entities,
totalItems,
pageInfo: {
...(pageInfo.nextCursor && {
@@ -252,7 +264,7 @@ export async function createRouter(
prevCursor: encodeCursor(pageInfo.prevCursor),
}),
},
});
}));
})
.get('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
@@ -260,10 +272,7 @@ export async function createRouter(
filter: basicEntityFilter({ 'metadata.uid': uid }),
credentials: await httpAuth.credentials(req),
});
if (!entities.length) {
throw new NotFoundError(`No entity with uid ${uid}`);
}
res.status(200).json(entities[0]);
writeSingleEntityResponse(res, entities, `No entity with uid ${uid}`);
})
.delete('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
@@ -278,12 +287,11 @@ export async function createRouter(
entityRefs: [stringifyEntityRef({ kind, namespace, name })],
credentials: await httpAuth.credentials(req),
});
if (!items[0]) {
throw new NotFoundError(
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,
);
}
res.status(200).json(items[0]);
writeSingleEntityResponse(
res,
items,
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,
);
})
.get(
'/entities/by-name/:kind/:namespace/:name/ancestry',
@@ -298,13 +306,15 @@ export async function createRouter(
)
.post('/entities/by-refs', async (req, res) => {
const request = entitiesBatchRequest(req);
const response = await entitiesCatalog.entitiesBatch({
const { items } = await entitiesCatalog.entitiesBatch({
entityRefs: request.entityRefs,
filter: parseEntityFilterParams(req.query),
fields: parseEntityTransformParams(req.query, request.fields),
credentials: await httpAuth.credentials(req),
});
res.status(200).json(response);
await writeEntitiesResponse(res, items, entities => ({
items: entities,
}));
})
.get('/entity-facets', async (req, res) => {
const response = await entitiesCatalog.facets({
plugins/catalog-backend/src/service/response/createEntityArrayJsonStream.ts
+79
View File
@@ -0,0 +1,79 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntitiesResponseItems } from '../../catalog/types';
import { Response } from 'express';
export interface EntityArrayJsonStream {
send(entities: EntitiesResponseItems): boolean;
complete(): void;
close(): void;
}
// Helps stream EntitiesResponseItems[] as a JSON response stream to avoid performance issues
export function createEntityArrayJsonStream(
res: Response,
): EntityArrayJsonStream {
// Imitate the httpRouter behavior of pretty-printing in development
const prettyPrint = process.env.NODE_ENV === 'development';
let firstSend = true;
let completed = false;
return {
send(response) {
if (firstSend) {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.status(200);
res.flushHeaders();
}
if (response.type === 'raw') {
let needsDrain = false;
for (const item of response.entities) {
const prefix = firstSend ? '[' : ',';
firstSend = false;
needsDrain ||= !res.write(prefix + item, 'utf8');
}
return !needsDrain;
}
let data: string;
if (prettyPrint) {
data = JSON.stringify(response.entities, null, 2);
data = firstSend ? data.slice(0, -2) : `,\n${data.slice(2, -2)}`;
} else {
data = JSON.stringify(response.entities);
data = firstSend ? data.slice(0, -1) : `,${data.slice(1, -1)}`;
}
firstSend = false;
return res.write(data, 'utf8');
},
complete() {
if (firstSend) {
res.json([]);
} else {
res.end(prettyPrint ? '\n]' : ']', 'utf8');
}
completed = true;
},
close() {
if (!completed) {
res.end();
}
},
};
}
plugins/catalog-backend/src/service/response/index.ts
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {
processRawEntitiesResult,
processEntitiesResponseItems,
entitiesResponseToObjects,
} from './process';
export { writeSingleEntityResponse, writeEntitiesResponse } from './write';
export { createEntityArrayJsonStream } from './createEntityArrayJsonStream';
plugins/catalog-backend/src/service/response/process.test.ts
+115
View File
@@ -0,0 +1,115 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import {
entitiesResponseToObjects,
processEntitiesResponseItems,
processRawEntitiesResult,
} from './process';
const mockTransform = (entity: Entity): Entity => ({
...entity,
kind: `transformed-${entity.kind}`,
});
describe('processRawEntitiesResult', () => {
it('should prefer keeping results in raw form', () => {
expect(processRawEntitiesResult(['{"kind":"test"}', null])).toEqual({
type: 'raw',
entities: ['{"kind":"test"}', null],
});
expect(
processRawEntitiesResult(['{"kind":"test"}', null], mockTransform),
).toEqual({
type: 'object',
entities: [{ kind: 'transformed-test' }, null],
});
});
});
describe('processEntitiesResponseItems', () => {
it('should transform entities in object form', () => {
expect(
processEntitiesResponseItems({
type: 'object',
entities: [{ kind: 'test' } as Entity, null],
}),
).toEqual({
type: 'object',
entities: [{ kind: 'test' }, null],
});
expect(
processEntitiesResponseItems(
{
type: 'object',
entities: [{ kind: 'test' } as Entity, null],
},
mockTransform,
),
).toEqual({
type: 'object',
entities: [{ kind: 'transformed-test' }, null],
});
});
it('should transform entities in raw form', () => {
expect(
processEntitiesResponseItems({
type: 'raw',
entities: ['{"kind":"test"}', null],
}),
).toEqual({
type: 'raw',
entities: ['{"kind":"test"}', null],
});
expect(
processEntitiesResponseItems(
{
type: 'raw',
entities: ['{"kind":"test"}', null],
},
mockTransform,
),
).toEqual({
type: 'object',
entities: [{ kind: 'transformed-test' }, null],
});
});
});
describe('entitiesResponseToObjects', () => {
it('should convert entities in object form', () => {
expect(
entitiesResponseToObjects({
type: 'object',
entities: [null, { kind: 'test' } as Entity],
}),
).toEqual([null, { kind: 'test' }]);
});
it('should convert entities in raw form', () => {
expect(
entitiesResponseToObjects({
type: 'raw',
entities: [null, '{"kind":"test"}'],
}),
).toEqual([null, { kind: 'test' }]);
});
});
plugins/catalog-backend/src/service/response/process.ts
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { EntitiesResponseItems } from '../../catalog/types';
export function processRawEntitiesResult(
serializedEntities: (string | null)[],
transform?: (entity: Entity) => Entity,
): EntitiesResponseItems {
if (transform) {
return {
type: 'object',
entities: serializedEntities.map(e =>
e !== null ? transform(JSON.parse(e)) : e,
),
};
}
return {
type: 'raw',
entities: serializedEntities,
};
}
export function processEntitiesResponseItems(
response: EntitiesResponseItems,
transform?: (entity: Entity) => Entity,
): EntitiesResponseItems {
if (!transform) {
return response;
}
if (response.type === 'raw') {
return processRawEntitiesResult(response.entities, transform);
}
return {
type: 'object',
entities: response.entities.map(e => (e !== null ? transform(e) : e)),
};
}
export function entitiesResponseToObjects(
response: EntitiesResponseItems,
): (Entity | null)[] {
if (response.type === 'object') {
return response.entities;
}
return response.entities.map(e => (e !== null ? JSON.parse(e) : e));
}
plugins/catalog-backend/src/service/response/write.test.ts
+317
View File
@@ -0,0 +1,317 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import express from 'express';
import { mockErrorHandler } from '@backstage/backend-test-utils';
import request from 'supertest';
import { writeEntitiesResponse, writeSingleEntityResponse } from './write';
describe('writeSingleEntityResponse', () => {
const app = express();
app.use(express.json());
app.get('/echo', (req, res) => {
writeSingleEntityResponse(res, req.body, 'not found');
});
app.use(mockErrorHandler());
describe('in object form', () => {
it('should write a single entity', async () => {
const res = await request(app)
.get('/echo')
.send({
type: 'object',
entities: [{ kind: 'Component' }, { kind: 'User' }],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({ kind: 'Component' });
});
it('should write a missing entity', async () => {
const res = await request(app)
.get('/echo')
.send({ type: 'object', entities: [null] });
expect(res.status).toBe(404);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toMatchObject({
error: { name: 'NotFoundError', message: 'not found' },
});
});
it('should write no entities', async () => {
const res = await request(app)
.get('/echo')
.send({ type: 'object', entities: [] });
expect(res.status).toBe(404);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toMatchObject({
error: { name: 'NotFoundError', message: 'not found' },
});
});
});
describe('in raw form', () => {
it('should write a single entity', async () => {
const res = await request(app)
.get('/echo')
.send({
type: 'raw',
entities: ['{"kind":"Component"}', '{"kind":"User"}'],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({ kind: 'Component' });
});
it('should write a missing entity', async () => {
const res = await request(app)
.get('/echo')
.send({ type: 'raw', entities: [null] });
expect(res.status).toBe(404);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toMatchObject({
error: { name: 'NotFoundError', message: 'not found' },
});
});
it('should write no entities', async () => {
const res = await request(app)
.get('/echo')
.send({ type: 'raw', entities: [] });
expect(res.status).toBe(404);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toMatchObject({
error: { name: 'NotFoundError', message: 'not found' },
});
});
});
});
describe('writeEntitiesResponse', () => {
const app = express();
app.use(express.json());
app.get('/echo', (req, res) => {
writeEntitiesResponse(res, req.body);
});
app.get('/wrapped', (req, res) => {
writeEntitiesResponse(res, req.body, entities => ({
page: 1,
items: entities,
totalItems: 1337,
}));
});
app.use(mockErrorHandler());
describe('in object form', () => {
it('should return empty list', async () => {
const res = await request(app).get('/echo').send({
type: 'object',
entities: [],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual([]);
});
it('should return mixed objects', async () => {
const res = await request(app)
.get('/echo')
.send({
type: 'object',
entities: [{ kind: 'Component' }, null, { kind: 'User' }, null],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual([
{ kind: 'Component' },
null,
{ kind: 'User' },
null,
]);
});
it('should wrap response of empty list', async () => {
const res = await request(app)
.get('/wrapped')
.send({ type: 'object', entities: [] });
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({ page: 1, items: [], totalItems: 1337 });
});
it('should wrap response of mixed list', async () => {
const res = await request(app)
.get('/wrapped')
.send({
type: 'object',
entities: [{ kind: 'Component' }, null, { kind: 'User' }, null],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({
page: 1,
items: [{ kind: 'Component' }, null, { kind: 'User' }, null],
totalItems: 1337,
});
});
});
describe('in raw form', () => {
it('should return empty list', async () => {
const res = await request(app).get('/echo').send({
type: 'raw',
entities: [],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual([]);
});
it('should return mixed objects', async () => {
const res = await request(app)
.get('/echo')
.send({
type: 'raw',
entities: ['{"kind":"Component"}', null, '{"kind":"User"}', null],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual([
{ kind: 'Component' },
null,
{ kind: 'User' },
null,
]);
});
it('should wrap response of empty list', async () => {
const res = await request(app)
.get('/wrapped')
.send({ type: 'raw', entities: [] });
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({ page: 1, items: [], totalItems: 1337 });
});
it('should wrap response of mixed list', async () => {
const res = await request(app)
.get('/wrapped')
.send({
type: 'raw',
entities: ['{"kind":"Component"}', null, '{"kind":"User"}', null],
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({
page: 1,
items: [{ kind: 'Component' }, null, { kind: 'User' }, null],
totalItems: 1337,
});
});
it('should write a large wrapped response', async () => {
const entityMock = JSON.stringify({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'my-component',
namespace: 'default',
annotations: {
'backstage.io/managed-by-location': 'url:https://example.com',
},
},
spec: {
type: 'service',
owner: 'me',
lifecycle: 'production',
},
});
const res = await request(app)
.get('/wrapped')
.send({
type: 'raw',
entities: Array(300).fill(entityMock),
});
expect(res.status).toBe(200);
expect(res.type).toBe('application/json');
expect(res.header['content-type']).toBe(
'application/json; charset=utf-8',
);
expect(res.body).toEqual({
page: 1,
items: expect.objectContaining({ length: 300 }),
totalItems: 1337,
});
});
});
});
plugins/catalog-backend/src/service/response/write.ts
+103
View File
@@ -0,0 +1,103 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Response } from 'express';
import { EntitiesResponseItems } from '../../catalog/types';
import { JsonValue } from '@backstage/types';
import { NotFoundError } from '@backstage/errors';
const JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
export function writeSingleEntityResponse(
res: Response,
response: EntitiesResponseItems,
notFoundMessage: string,
) {
if (response.type === 'object') {
if (!response.entities[0]) {
throw new NotFoundError(notFoundMessage);
}
res.json(response.entities[0]);
} else {
if (!response.entities[0]) {
throw new NotFoundError(notFoundMessage);
}
res.setHeader('Content-Type', JSON_CONTENT_TYPE);
res.end(response.entities[0]);
}
}
export async function writeEntitiesResponse(
res: Response,
response: EntitiesResponseItems,
responseWrapper?: (entities: JsonValue) => JsonValue,
) {
if (response.type === 'object') {
res.json(
responseWrapper
? responseWrapper?.(response.entities)
: response.entities,
);
return;
}
res.setHeader('Content-Type', JSON_CONTENT_TYPE);
// responseWrapper allows the caller to render the entities within an object
let trailing = '';
if (responseWrapper) {
const marker = `__MARKER_${Math.random().toString(36).slice(2, 10)}__`;
const wrapped = JSON.stringify(responseWrapper(marker));
const parts = wrapped.split(`"${marker}"`);
if (parts.length !== 2) {
throw new Error(
`Entity items response was incorrectly wrapped into ${parts.length} different parts`,
);
}
res.write(parts[0], 'utf8');
trailing = parts[1];
}
let first = true;
for (const entity of response.entities) {
const prefix = first ? '[' : ',';
first = false;
const needsDrain = !res.write(prefix + entity, 'utf8');
if (needsDrain) {
const closed = await new Promise<boolean>(resolve => {
function onContinue() {
res.off('drain', onContinue);
res.off('close', onClose);
resolve(false);
}
function onClose() {
res.off('drain', onContinue);
res.off('close', onClose);
resolve(true);
}
res.on('drain', onContinue);
res.on('close', onClose);
});
if (closed) {
return;
}
}
}
res.end(`${first ? '[' : ''}]${trailing}`);
}
plugins/catalog-backend/src/service/util.ts
+15 -69
View File
@@ -15,7 +15,7 @@
*/
import { InputError, NotAllowedError } from '@backstage/errors';
import { Request, Response } from 'express';
import { Request } from 'express';
import lodash from 'lodash';
import { z } from 'zod';
import {
@@ -148,75 +148,21 @@ export function decodeCursor(encodedCursor: string) {
// sure that all adopters have re-stitched their entities so that the new
// targetRef field is present on them, and that they have stopped consuming
// the now-removed old field
// TODO(jhaals): Remove this in April 2022
export function expandLegacyCompoundRelationRefsInResponse(
entities: Entity[],
): void {
for (const entity of entities) {
if (entity.relations) {
for (const relation of entity.relations as any) {
if (!relation.targetRef && relation.target) {
// This is the case where an old-form entity, not yet stitched with
// the updated code, was in the database
relation.targetRef = stringifyEntityRef(relation.target);
} else if (!relation.target && relation.targetRef) {
// This is the case where a new-form entity, stitched with the
// updated code, was in the database but we still want to produce
// the old data shape as well for compatibility reasons
relation.target = parseEntityRef(relation.targetRef);
}
// TODO(patriko): Remove this in catalog 2.0
export function expandLegacyCompoundRelationsInEntity(entity: Entity): Entity {
if (entity.relations) {
for (const relation of entity.relations as any) {
if (!relation.targetRef && relation.target) {
// This is the case where an old-form entity, not yet stitched with
// the updated code, was in the database
relation.targetRef = stringifyEntityRef(relation.target);
} else if (!relation.target && relation.targetRef) {
// This is the case where a new-form entity, stitched with the
// updated code, was in the database but we still want to produce
// the old data shape as well for compatibility reasons
relation.target = parseEntityRef(relation.targetRef);
}
}
}
}
export interface EntityArrayJsonStream {
send(entities: Entity[]): boolean;
complete(): void;
close(): void;
}
// Helps stream Entity[] as a JSON response stream to avoid performance issues
export function createEntityArrayJsonStream(
res: Response,
): EntityArrayJsonStream {
// Imitate the httpRouter behavior of pretty-printing in development
const prettyPrint = process.env.NODE_ENV === 'development';
let firstSend = true;
let completed = false;
return {
send(entities) {
if (firstSend) {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.status(200);
res.flushHeaders();
}
let data: string;
if (prettyPrint) {
data = JSON.stringify(entities, null, 2);
data = firstSend ? data.slice(0, -2) : `,\n${data.slice(2, -2)}`;
} else {
data = JSON.stringify(entities);
data = firstSend ? data.slice(0, -1) : `,${data.slice(1, -1)}`;
}
firstSend = false;
return res.write(data);
},
complete() {
if (firstSend) {
res.json([]);
} else {
res.end(prettyPrint ? '\n]' : ']');
}
completed = true;
},
close() {
if (!completed) {
res.end();
}
},
};
return entity;
}
plugins/catalog-backend/src/tests/integration.test.ts
+73 -16
View File
@@ -30,7 +30,6 @@ import {
processingResult,
} from '@backstage/plugin-catalog-node';
import { PermissionEvaluator } from '@backstage/plugin-permission-common';
import { JsonObject } from '@backstage/types';
import { createHash } from 'crypto';
import { Knex } from 'knex';
import { EntitiesCatalog } from '../catalog/types';
@@ -55,6 +54,7 @@ import { DefaultStitcher } from '../stitching/DefaultStitcher';
import { mockServices } from '@backstage/backend-test-utils';
import { LoggerService } from '@backstage/backend-plugin-api';
import { DatabaseManager } from '@backstage/backend-common';
import { entitiesResponseToObjects } from '../service/response';
const voidLogger = mockServices.logger.mock();
@@ -198,7 +198,7 @@ class TestHarness {
readonly #proxyProgressTracker: ProxyProgressTracker;
static async create(options?: {
config?: JsonObject;
disableRelationsCompatibility?: boolean;
logger?: LoggerService;
db?: Knex;
permissions?: PermissionEvaluator;
@@ -208,21 +208,19 @@ class TestHarness {
emit: CatalogProcessorEmit,
): Promise<Entity>;
}) {
const config = new ConfigReader(
options?.config ?? {
backend: {
database: {
client: 'better-sqlite3',
connection: ':memory:',
},
},
catalog: {
stitchingStrategy: {
mode: 'immediate',
},
const config = new ConfigReader({
backend: {
database: {
client: 'better-sqlite3',
connection: ':memory:',
},
},
);
catalog: {
stitchingStrategy: {
mode: 'immediate',
},
},
});
const logger = options?.logger ?? mockServices.logger.mock();
const db =
options?.db ??
@@ -279,6 +277,7 @@ class TestHarness {
database: db,
logger,
stitcher,
disableRelationsCompatibility: options?.disableRelationsCompatibility,
});
const proxyProgressTracker = new ProxyProgressTracker(
new NoopProgressTracker(),
@@ -365,7 +364,12 @@ class TestHarness {
async getOutputEntities(): Promise<Record<string, Entity>> {
const { entities } = await this.#catalog.entities();
return Object.fromEntries(entities.map(e => [stringifyEntityRef(e), e]));
return Object.fromEntries(
entitiesResponseToObjects(entities).map(e => [
stringifyEntityRef(e!),
e!,
]),
);
}
async refresh(options: RefreshOptions) {
@@ -781,4 +785,57 @@ describe('Catalog Backend Integration', () => {
],
});
});
it('should return valid responses in raw JSON mode', async () => {
const harness = await TestHarness.create({
disableRelationsCompatibility: true,
});
const entityA = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'a',
annotations: {
'backstage.io/managed-by-location': 'url:.',
'backstage.io/managed-by-origin-location': 'url:.',
},
},
};
const entityB = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'b',
annotations: {
'backstage.io/managed-by-location': 'url:.',
'backstage.io/managed-by-origin-location': 'url:.',
},
},
};
await harness.setInputEntities([entityA, entityB]);
await expect(harness.process()).resolves.toEqual({});
await expect(harness.getOutputEntities()).resolves.toEqual({
'component:default/a': {
...entityA,
metadata: {
...entityA.metadata,
etag: expect.any(String),
uid: expect.any(String),
},
relations: [],
},
'component:default/b': {
...entityB,
metadata: {
...entityB.metadata,
etag: expect.any(String),
uid: expect.any(String),
},
relations: [],
},
});
});
});