paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

Changed endpoint name, GET->POST, request/response signature, and perform batch queries against db

Browse Source 
Signed-off-by: Gustaf Räntilä <g.rantila@gmail.com>
This commit is contained in:
Gustaf Räntilä
2025-09-29 14:42:57 +02:00
parent f79e0d9d77
commit 233bb3086c
13 changed files with 343 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/user-settings-backend/package.json
+2 -1
View File
@@ -59,7 +59,8 @@
"express": "^4.22.0",
"express-promise-router": "^4.1.0",
"knex": "^3.0.0",
"p-limit": "^3.1.0"
"p-limit": "^3.1.0",
"zod": "^3.22.4"
},
"devDependencies": {
"@backstage/backend-defaults": "workspace:^",
plugins/user-settings-backend/src/database/DatabaseUserSettingsStore.test.ts
+75
View File
@@ -107,6 +107,81 @@ describe.each(databases.eachSupportedId())(
});
});
describe('multiget', () => {
it('should return empty', async () => {
await expect(
storage.multiget({
userEntityRef: 'user-1',
items: [
{
bucket: 'bucket-c',
key: 'key-c',
},
],
}),
).resolves.toEqual([null]);
});
it('should handle large inputs and return existing values', async () => {
// Create 30 buckets with 300 keys in each.
const BUCKETS = 30;
const KEYS_PER_BUCKET = 300;
const buckets = Array.from(Array(BUCKETS)).map(
(_, i) => `mget-bucket-${i}`,
);
const items = buckets.flatMap(bucket => {
const keys = Array.from(Array(KEYS_PER_BUCKET)).map(
(_, i) => `mget-key-${i}`,
);
return keys.map(key => ({ bucket, key }));
});
const chunkify = <T>(keys: Array<T>, size: number): T[][] => {
const chunks: T[][] = [];
for (let i = 0; i < keys.length; i += size) {
chunks.push(keys.slice(i, i + size));
}
return chunks;
};
const valueOfKey = (bucket: string, key: string) => ({
theValue: `Value of ${bucket} / ${key}`,
});
const chunkedItems = chunkify(items, 100);
for (const chunk of chunkedItems) {
await insert(
chunk.map(({ bucket, key }) => ({
user_entity_ref: 'user-1',
bucket,
key,
value: JSON.stringify(valueOfKey(bucket, key)),
})),
);
}
const result = await storage.multiget({
userEntityRef: 'user-1',
// Include a missing key which shouldn't exist in the result
items: [...items, { bucket: 'missing', key: 'missing' }],
});
expect(result).toEqual([
...items.map(({ bucket, key }) => ({
value: valueOfKey(bucket, key),
})),
null, // The missing key
]);
const rows = await knex<RawDbUserSettingsRow>('user_settings')
.where('user_entity_ref', 'user-1')
.count<{ count: string }[]>('* as count');
const savedRows = Number(rows[0].count);
expect(savedRows).toEqual(BUCKETS * KEYS_PER_BUCKET);
});
});
describe('set', () => {
it('should insert a new setting', async () => {
await storage.set({
plugins/user-settings-backend/src/database/DatabaseUserSettingsStore.ts
+75
View File
@@ -21,6 +21,7 @@ import {
import { NotFoundError } from '@backstage/errors';
import { JsonValue } from '@backstage/types';
import { Knex } from 'knex';
import pLimit from 'p-limit';
import { UserSettingsStore, type UserSetting } from './UserSettingsStore';
const migrationsDir = resolvePackagePath(
@@ -91,6 +92,80 @@ export class DatabaseUserSettingsStore implements UserSettingsStore {
};
}
async multiget(options: {
userEntityRef: string;
items: Array<{ bucket: string; key: string }>;
}): Promise<({ value: JsonValue } | null)[]> {
if (options.items.length === 0) {
return [];
}
// Split the items into a map of bucket -> keys
const bucketMap = new Map<string, Set<string>>();
for (const item of options.items) {
let keys = bucketMap.get(item.bucket);
if (!keys) {
keys = new Set();
bucketMap.set(item.bucket, keys);
}
keys.add(item.key);
}
const dbLimit = pLimit(10);
// Chunks the keys per bucket to avoid hitting SQL parameter limits
const chunkKeys = (keys: Array<string>, size: number): string[][] => {
const chunks = [];
for (let i = 0; i < keys.length; i += size) {
chunks.push(keys.slice(i, i + size));
}
return chunks;
};
// Store the database content into a map of bucket -> {key -> value}
const resultsMap = new Map<string, Map<string, JsonValue>>();
await Promise.all(
bucketMap.keys().map(bucket =>
dbLimit(async (): Promise<void> => {
const keyMap = new Map<string, JsonValue>();
resultsMap.set(bucket, keyMap);
const keyChunks = chunkKeys(
Array.from(bucketMap.get(bucket) || []),
100,
);
for (const keys of keyChunks) {
const rows = await this.db<RawDbUserSettingsRow>('user_settings')
.where({
user_entity_ref: options.userEntityRef,
bucket,
})
.whereIn('key', keys)
.select(['bucket', 'key', 'value']);
for (const row of rows) {
keyMap.set(row.key, JSON.parse(row.value));
}
}
}),
),
);
// For each exact bucket/key requested, return either the value or null if
// not found
return options.items.map(({ bucket, key }) => {
const value = resultsMap.get(bucket)?.get(key);
if (typeof value === 'undefined') {
return null;
}
return { value };
});
}
async set(options: {
userEntityRef: string;
bucket: string;
plugins/user-settings-backend/src/database/UserSettingsStore.ts
+10
View File
@@ -35,6 +35,16 @@ export interface UserSettingsStore {
key: string;
}): Promise<UserSetting>;
/**
* Get multiple user settings at once. The result is an array corresponding
* index by index to the elements of the input, where the elements are either
* `null` (not found), or an object `{ value: JsonValue }`.
*/
multiget(options: {
userEntityRef: string;
items: Array<{ bucket: string; key: string }>;
}): Promise<({ value: JsonValue } | null)[]>;
set(options: {
userEntityRef: string;
bucket: string;
plugins/user-settings-backend/src/service/router.test.ts
+72 -95
View File
@@ -18,17 +18,17 @@ import express from 'express';
import request from 'supertest';
import { UserSettingsStore } from '../database/UserSettingsStore';
import { createRouter } from './router';
import { NotFoundError } from '@backstage/errors';
import { SignalsService } from '@backstage/plugin-signals-node';
import {
mockCredentials,
mockServices,
mockErrorHandler,
} from '@backstage/backend-test-utils';
import { stringifyDataLoaderKey } from '@backstage/plugin-user-settings-common';
import { JsonValue } from '@backstage/types';
describe('createRouter', () => {
const userSettingsStore: jest.Mocked<UserSettingsStore> = {
multiget: jest.fn(),
get: jest.fn(),
set: jest.fn(),
delete: jest.fn(),
@@ -111,78 +111,72 @@ describe('createRouter', () => {
});
describe('GET /multi', () => {
const mockMultiget = (store: Record<string, Record<string, JsonValue>>) => {
userSettingsStore.multiget.mockImplementation(async ({ items }) => {
return items.map(({ bucket, key }) => {
const value = store[bucket]?.[key];
if (typeof value !== 'undefined') {
return { value };
}
return null;
});
});
};
it('returns single value', async () => {
const values = { 'key-1': 'a' };
userSettingsStore.get.mockImplementation(async ({ bucket, key }) => {
if (key in values) {
return {
mockMultiget({ 'my-bucket': values });
const responses = await request(app)
.post('/multiget')
.send({
items: Object.keys(values).map(key => ({
bucket: 'my-bucket',
key,
value: values[key as keyof typeof values],
};
}
throw new NotFoundError(
`Unable to find '${key}' in bucket '${bucket}'`,
);
});
const url = new URLSearchParams();
for (const key of Object.keys(values)) {
url.append('items', stringifyDataLoaderKey('my-bucket', key));
}
const responses = await request(app).get(`/multi?${url.toString()}`);
})),
});
expect(responses.status).toEqual(200);
expect(responses.body).toEqual([
{ bucket: 'my-bucket', key: 'key-1', value: 'a' },
]);
expect(responses.body).toEqual({ items: [{ value: 'a' }] });
expect(userSettingsStore.get).toHaveBeenCalledTimes(1);
expect(userSettingsStore.get).toHaveBeenCalledWith({
expect(userSettingsStore.multiget).toHaveBeenCalledTimes(1);
expect(userSettingsStore.multiget).toHaveBeenCalledWith({
userEntityRef: mockUserRef,
bucket: 'my-bucket',
key: 'key-1',
items: [
{
bucket: 'my-bucket',
key: 'key-1',
},
],
});
});
it('returns single missing', async () => {
const values = { 'key-1': 'a' };
userSettingsStore.get.mockImplementation(async ({ bucket, key }) => {
if (key in values) {
return {
bucket: 'my-bucket',
key,
value: values[key as keyof typeof values],
};
}
throw new NotFoundError(
`Unable to find '${key}' in bucket '${bucket}'`,
);
});
mockMultiget({ 'my-bucket': values });
const url = new URLSearchParams();
url.append('items', stringifyDataLoaderKey('my-bucket', 'missing-key'));
const responses = await request(app).get(`/multi?${url.toString()}`);
const responses = await request(app)
.post('/multiget')
.send({
items: [{ bucket: 'my-bucket', key: 'missing-key' }],
});
expect(responses.status).toEqual(200);
expect(responses.body).toEqual([
{
bucket: 'my-bucket',
key: 'missing-key',
error: {
name: 'NotFoundError',
message: expect.stringContaining('missing-key'),
},
},
]);
expect(responses.body).toEqual({
items: [null],
});
expect(userSettingsStore.get).toHaveBeenCalledTimes(1);
expect(userSettingsStore.get).toHaveBeenCalledWith({
expect(userSettingsStore.multiget).toHaveBeenCalledTimes(1);
expect(userSettingsStore.multiget).toHaveBeenCalledWith({
userEntityRef: mockUserRef,
bucket: 'my-bucket',
key: 'missing-key',
items: [
{
bucket: 'my-bucket',
key: 'missing-key',
},
],
});
});
@@ -192,52 +186,35 @@ describe('createRouter', () => {
'key-2': 'b',
};
userSettingsStore.get.mockImplementation(async ({ bucket, key }) => {
if (key in values) {
return {
bucket: 'my-bucket',
key,
value: values[key as keyof typeof values],
};
}
throw new NotFoundError(
`Unable to find '${key}' in bucket '${bucket}'`,
);
});
mockMultiget({ 'my-bucket': values });
const url = new URLSearchParams();
for (const key of Object.keys(values)) {
url.append('items', stringifyDataLoaderKey('my-bucket', key));
}
url.append('items', stringifyDataLoaderKey('my-bucket', 'missing-key'));
const responses = await request(app).get(`/multi?${url.toString()}`);
const responses = await request(app)
.post('/multiget')
.send({
items: [
...Object.keys(values).map(key => ({ bucket: 'my-bucket', key })),
{ bucket: 'my-bucket', key: 'missing-key' },
],
});
expect(responses.status).toEqual(200);
expect(responses.body).toEqual([
{ bucket: 'my-bucket', key: 'key-1', value: 'a' },
{ bucket: 'my-bucket', key: 'key-2', value: 'b' },
{
bucket: 'my-bucket',
key: 'missing-key',
error: {
name: 'NotFoundError',
message: expect.stringContaining('missing-key'),
},
},
]);
expect(responses.body).toEqual({
items: [{ value: 'a' }, { value: 'b' }, null],
});
expect(userSettingsStore.get).toHaveBeenCalledTimes(3);
for (const key of Object.keys(values)) {
expect(userSettingsStore.get).toHaveBeenCalledWith({
userEntityRef: mockUserRef,
bucket: 'my-bucket',
key,
});
}
expect(userSettingsStore.get).toHaveBeenCalledWith({
expect(userSettingsStore.multiget).toHaveBeenCalledTimes(1);
expect(userSettingsStore.multiget).toHaveBeenCalledWith({
userEntityRef: mockUserRef,
bucket: 'my-bucket',
key: 'missing-key',
items: [
...Object.keys(values).map(key => ({
bucket: 'my-bucket',
key,
})),
{
bucket: 'my-bucket',
key: 'missing-key',
},
],
});
});
plugins/user-settings-backend/src/service/router.ts
+14 -52
View File
@@ -14,16 +14,15 @@
* limitations under the License.
*/
import { InputError, serializeError } from '@backstage/errors';
import { InputError } from '@backstage/errors';
import express, { Request } from 'express';
import Router from 'express-promise-router';
import pLimit from 'p-limit';
import { z } from 'zod';
import { UserSettingsStore } from '../database/UserSettingsStore';
import { SignalsService } from '@backstage/plugin-signals-node';
import {
MultiUserSetting,
MultiGetResponse,
UserSettingsSignal,
parseDataLoaderKey,
} from '@backstage/plugin-user-settings-common';
import { HttpAuthService } from '@backstage/backend-plugin-api';
@@ -35,6 +34,10 @@ export async function createRouter(options: {
const router = Router();
router.use(express.json());
const multiGetRequestSchema = z.object({
items: z.array(z.object({ bucket: z.string(), key: z.string() })),
});
/**
* Helper method to extract the userEntityRef from the request.
*/
@@ -46,58 +49,17 @@ export async function createRouter(options: {
};
// get multiple values
router.get('/multi', async (req, res) => {
router.post('/multiget', async (req, res) => {
const userEntityRef = await getUserEntityRef(req);
const bucketsAndKeys: ReturnType<typeof parseDataLoaderKey>[] = [];
const bucketsAndKeys = multiGetRequestSchema.parse(req.body).items;
const items = req.query.items;
if (typeof items === 'string') {
bucketsAndKeys.push(parseDataLoaderKey(items));
} else if (Array.isArray(items)) {
bucketsAndKeys.push(
...items.map(item => {
if (typeof item !== 'string') {
throw new InputError(
'Expected query param "items" to be an array of strings',
);
}
return parseDataLoaderKey(item);
}),
);
} else {
throw new InputError('Expected query param "items" to be an array');
}
const items = await options.userSettingsStore.multiget({
userEntityRef,
items: bucketsAndKeys,
});
const limit = pLimit(10);
const userSettings = await Promise.all(
bucketsAndKeys.map(({ bucket, key }) =>
limit(async (): Promise<MultiUserSetting> => {
try {
const setting = await options.userSettingsStore.get({
userEntityRef,
bucket,
key,
});
return setting;
} catch (e) {
if (e instanceof Error) {
const serialized = serializeError(e);
return { bucket, key, error: serialized };
}
return {
bucket,
key,
error: { name: 'Error', message: 'Unknown error' },
};
}
}),
),
);
res.json(userSettings);
res.json({ items } satisfies MultiGetResponse);
});
// get a single value