feat: allow scaffolder tasks pagination in the backend

Still requires the UI to take use of this. Relates to #25856

Signed-off-by: Heikki Hellgren <heikki.hellgren@op.fi>
This commit is contained in:
Heikki Hellgren
2024-08-30 12:39:31 +03:00
parent f094dfd54a
commit 3ec4e6d891
13 changed files with 486 additions and 53 deletions
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/plugin-scaffolder-backend': minor
'@backstage/plugin-scaffolder-node': minor
---
Added pagination support for listing of tasks and the ability to filter on several users and task statuses.
+52 -2
View File
@@ -428,8 +428,24 @@ export class DatabaseTaskStore implements TaskStore {
// (undocumented)
heartbeatTask(taskId: string): Promise<void>;
// (undocumented)
list(options: { createdBy?: string; status?: TaskStatus_2 }): Promise<{
list(options: {
createdBy?: string;
status?: TaskStatus_2;
filters?: {
createdBy?: string | string[];
status?: TaskStatus_2 | TaskStatus_2[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: {
order: 'asc' | 'desc';
field: string;
}[];
}): Promise<{
tasks: SerializedTask_2[];
totalTasks?: number;
}>;
// (undocumented)
listEvents(options: TaskStoreListEventsOptions): Promise<{
@@ -647,8 +663,42 @@ export interface TaskStore {
// (undocumented)
heartbeatTask(taskId: string): Promise<void>;
// (undocumented)
list?(options: { createdBy?: string; status?: TaskStatus }): Promise<{
list?(options: {
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: {
order: 'asc' | 'desc';
field: string;
}[];
}): Promise<{
tasks: SerializedTask[];
totalTasks?: number;
}>;
// @deprecated (undocumented)
list?(options: {
createdBy?: string;
status?: TaskStatus;
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: {
order: 'asc' | 'desc';
field: string;
}[];
}): Promise<{
tasks: SerializedTask[];
totalTasks?: number;
}>;
// (undocumented)
listEvents(options: TaskStoreListEventsOptions): Promise<{
@@ -75,6 +75,62 @@ describe('DatabaseTaskStore', () => {
expect(tasks[0].id).toBeDefined();
});
it('should allow paginating tasks', async () => {
const { store } = await createStore();
await store.createTask({
spec: {} as TaskSpec,
createdBy: 'me',
});
await store.createTask({
spec: {} as TaskSpec,
createdBy: 'him',
});
const { tasks } = await store.list({ pagination: { limit: 1, offset: 0 } });
expect(tasks.length).toBe(1);
expect(tasks[0].createdBy).toBe('me');
expect(tasks[0].status).toBe('open');
expect(tasks[0].id).toBeDefined();
const { tasks: tasks2 } = await store.list({
pagination: { limit: 1, offset: 1 },
});
expect(tasks2.length).toBe(1);
expect(tasks2[0].createdBy).toBe('him');
expect(tasks2[0].status).toBe('open');
expect(tasks2[0].id).toBeDefined();
});
it('should allow ordering tasks', async () => {
const { store } = await createStore();
await store.createTask({
spec: {} as TaskSpec,
createdBy: 'a',
});
await store.createTask({
spec: {} as TaskSpec,
createdBy: 'b',
});
const { tasks } = await store.list({
order: [{ field: 'created_by', order: 'asc' }],
});
expect(tasks.length).toBe(2);
expect(tasks[0].createdBy).toBe('a');
expect(tasks[0].status).toBe('open');
expect(tasks[0].id).toBeDefined();
const { tasks: tasks2 } = await store.list({
order: [{ field: 'created_by', order: 'desc' }],
});
expect(tasks2.length).toBe(2);
expect(tasks2[0].createdBy).toBe('b');
expect(tasks2[0].status).toBe('open');
expect(tasks2[0].id).toBeDefined();
});
it('should list filtered created tasks by createdBy', async () => {
const { store } = await createStore();
@@ -93,6 +149,14 @@ describe('DatabaseTaskStore', () => {
expect(tasks[0].createdBy).toBe('him');
expect(tasks[0].status).toBe('open');
expect(tasks[0].id).toBeDefined();
const { tasks: tasks2 } = await store.list({
filters: { createdBy: 'him' },
});
expect(tasks2.length).toBe(1);
expect(tasks2[0].createdBy).toBe('him');
expect(tasks2[0].status).toBe('open');
expect(tasks2[0].id).toBeDefined();
});
it('should list filtered created tasks by status', async () => {
@@ -115,8 +179,43 @@ describe('DatabaseTaskStore', () => {
eventBody: { message },
});
const { tasks } = await store.list({ status: 'open' });
const { tasks, totalTasks } = await store.list({
status: 'open',
});
expect(tasks.length).toBe(1);
expect(totalTasks).toBe(1);
expect(tasks[0].createdBy).toBe('him');
expect(tasks[0].status).toBe('open');
expect(tasks[0].id).toBeDefined();
const { tasks: tasks2, totalTasks: totalTasks2 } = await store.list({
filters: { status: ['open'] },
});
expect(tasks2.length).toBe(1);
expect(totalTasks2).toBe(1);
expect(tasks2[0].createdBy).toBe('him');
expect(tasks2[0].status).toBe('open');
expect(tasks2[0].id).toBeDefined();
});
it('should limit and offset based on parameters', async () => {
const { store } = await createStore();
await store.createTask({
spec: {} as TaskSpec,
createdBy: 'me',
});
await store.createTask({
spec: {} as TaskSpec,
createdBy: 'him',
});
const { tasks, totalTasks } = await store.list({
pagination: { limit: 1, offset: 1 },
});
expect(tasks.length).toBe(1);
expect(totalTasks).toBe(2);
expect(tasks[0].createdBy).toBe('him');
expect(tasks[0].status).toBe('open');
expect(tasks[0].id).toBeDefined();
@@ -44,6 +44,7 @@ import {
restoreWorkspace,
serializeWorkspace,
} from '@backstage/plugin-scaffolder-node/alpha';
import { flattenParams } from '../../service/helpers';
const migrationsDir = resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
@@ -183,20 +184,58 @@ export class DatabaseTaskStore implements TaskStore {
async list(options: {
createdBy?: string;
status?: TaskStatus;
}): Promise<{ tasks: SerializedTask[] }> {
const queryBuilder = this.db<RawDbTaskRow>('tasks');
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: { order: 'asc' | 'desc'; field: string }[];
}): Promise<{ tasks: SerializedTask[]; totalTasks?: number }> {
const { createdBy, status, pagination, order, filters } = options ?? {};
const queryBuilder = this.db<RawDbTaskRow & { count: number }>('tasks');
if (options.createdBy) {
queryBuilder.where({
created_by: options.createdBy,
if (createdBy || filters?.createdBy) {
const arr: string[] = flattenParams<string>(
createdBy,
filters?.createdBy,
);
queryBuilder.whereIn('created_by', [...new Set(arr)]);
}
if (status || filters?.status) {
const arr: TaskStatus[] = flattenParams<TaskStatus>(
status,
filters?.status,
);
queryBuilder.whereIn('status', [...new Set(arr)]);
}
if (order) {
order.forEach(f => {
queryBuilder.orderBy(f.field, f.order);
});
} else {
queryBuilder.orderBy('created_at', 'desc');
}
if (options.status) {
queryBuilder.where({ status: options.status });
const countQuery = queryBuilder.clone();
countQuery.count('tasks.id', { as: 'count' });
if (pagination?.limit !== undefined) {
queryBuilder.limit(pagination.limit);
}
const results = await queryBuilder.orderBy('created_at', 'desc').select();
if (pagination?.offset !== undefined) {
queryBuilder.offset(pagination.offset);
}
const [results, [{ count }]] = await Promise.all([
queryBuilder.select(),
countQuery,
]);
const tasks = results.map(result => ({
id: result.id,
@@ -207,7 +246,7 @@ export class DatabaseTaskStore implements TaskStore {
createdAt: parseSqlDateToIsoString(result.created_at),
}));
return { tasks };
return { tasks, totalTasks: count };
}
async getTask(taskId: string): Promise<SerializedTask> {
@@ -21,8 +21,8 @@ import {
import { ConfigReader } from '@backstage/config';
import { TaskSpec } from '@backstage/plugin-scaffolder-common';
import {
TaskSecrets,
SerializedTaskEvent,
TaskSecrets,
} from '@backstage/plugin-scaffolder-node';
import { DatabaseTaskStore } from './DatabaseTaskStore';
import { StorageTaskBroker, TaskManager } from './StorageTaskBroker';
@@ -231,6 +231,7 @@ describe('StorageTaskBroker', () => {
id: taskId,
}),
]),
totalTasks: 13,
});
});
@@ -243,8 +244,30 @@ describe('StorageTaskBroker', () => {
const task = await storage.getTask(taskId);
const promise = broker.list({ createdBy: 'user:default/foo' });
await expect(promise).resolves.toEqual({ tasks: [task] });
const promise = broker.list({
filters: { createdBy: ['user:default/foo'] },
});
await expect(promise).resolves.toEqual({ tasks: [task], totalTasks: 1 });
});
it('should list only tasks with specific status', async () => {
const broker = new StorageTaskBroker(storage, logger);
const { taskId } = await broker.dispatch({
spec: { steps: [] } as unknown as TaskSpec,
createdBy: 'user:default/foo',
});
const promise = broker.list({
filters: { status: ['open'] },
});
await expect(promise).resolves.toEqual({
tasks: expect.arrayContaining([
expect.objectContaining({
id: taskId,
}),
]),
totalTasks: 3,
});
});
it('should handle checkpoints in task state', async () => {
@@ -281,16 +281,22 @@ export class StorageTaskBroker implements TaskBroker {
async list(options?: {
createdBy?: string;
status?: TaskStatus;
}): Promise<{ tasks: SerializedTask[] }> {
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: { order: 'asc' | 'desc'; field: string }[];
}): Promise<{ tasks: SerializedTask[]; totalTasks?: number }> {
if (!this.storage.list) {
throw new Error(
'TaskStore does not implement the list method. Please implement the list method to be able to list tasks',
);
}
return await this.storage.list({
createdBy: options?.createdBy,
status: options?.status,
});
return await this.storage.list(options ?? {});
}
private deferredDispatch = defer();
@@ -190,10 +190,34 @@ export interface TaskStore {
tasks: { taskId: string }[];
}>;
list?(options: {
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: { order: 'asc' | 'desc'; field: string }[];
}): Promise<{ tasks: SerializedTask[]; totalTasks?: number }>;
/**
* @deprecated Make sure to pass `createdBy` and `status` in the `filters` parameter instead
*/
list?(options: {
createdBy?: string;
status?: TaskStatus;
}): Promise<{ tasks: SerializedTask[] }>;
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: { order: 'asc' | 'desc'; field: string }[];
}): Promise<{ tasks: SerializedTask[]; totalTasks?: number }>;
emitLogEvent(options: TaskStoreEmitOptions): Promise<void>;
@@ -0,0 +1,74 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { flattenParams, parseNumberParam, parseStringsParam } from './helpers';
describe('parseStringsParam', () => {
it('should parse a single string', () => {
expect(parseStringsParam('hello', 'param')).toEqual(['hello']);
});
it('should parse an array of strings', () => {
expect(parseStringsParam(['hello', 'world'], 'param')).toEqual([
'hello',
'world',
]);
});
it('should return undefined for undefined', () => {
expect(parseStringsParam(undefined, 'param')).toBeUndefined();
});
it('should throw an error for non-string values', () => {
expect(() => parseStringsParam(5, 'param')).toThrow(
'Invalid param, not a string or array of strings',
);
});
});
describe('parseNumberParam', () => {
it('should parse a number', () => {
expect(parseNumberParam('5', 'param')).toEqual([5]);
});
it('should throw an error for a non-number', () => {
expect(() => parseNumberParam('hello', 'param')).toThrow(
'Invalid param parameter "hello", expected a number or array of numbers',
);
});
it('should return undefined for undefined', () => {
expect(parseNumberParam(undefined, 'param')).toBeUndefined();
});
});
describe('flattenParams', () => {
it('should flatten a single string', () => {
expect(flattenParams<string>('hello')).toEqual(['hello']);
});
it('should flatten an array of strings', () => {
expect(flattenParams<string>(['hello', 'world'])).toEqual([
'hello',
'world',
]);
});
it('should flatten mixed parameters', () => {
expect(flattenParams<string>(['hello', 'world'], 'foo', undefined)).toEqual(
['hello', 'world', 'foo'],
);
});
});
@@ -16,11 +16,11 @@
import { CatalogApi } from '@backstage/catalog-client';
import {
Entity,
ANNOTATION_LOCATION,
parseLocationRef,
ANNOTATION_SOURCE_LOCATION,
CompoundEntityRef,
Entity,
parseLocationRef,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
@@ -107,3 +107,44 @@ export async function findTemplate(options: {
return template as TemplateEntityV1beta3;
}
/**
* Takes a single unknown parameter and makes sure that it's a single string or
* an array of strings, and returns as an array.
*/
export function parseStringsParam(
param: unknown,
paramName: string,
): string[] | undefined {
if (param === undefined) {
return undefined;
}
const array = [param].flat();
if (array.some(p => typeof p !== 'string')) {
throw new InputError(
`Invalid ${paramName}, not a string or array of strings`,
);
}
return array as string[];
}
export function parseNumberParam(
param: unknown,
paramName: string,
): number[] | undefined {
return parseStringsParam(param, paramName)?.map(val => {
const ret = Number.parseInt(val, 10);
if (isNaN(ret)) {
throw new InputError(
`Invalid ${paramName} parameter "${val}", expected a number or array of numbers`,
);
}
return ret;
});
}
export function flattenParams<T>(...params: (undefined | T | T[])[]): T[] {
return [...params].flat().filter(Boolean) as T[];
}
@@ -399,11 +399,13 @@ describe('createRouter', () => {
createdBy: '',
},
],
totalTasks: 1,
});
const response = await request(app).get(`/v2/tasks`);
expect(taskBroker.list).toHaveBeenCalledWith({
createdBy: undefined,
filters: {},
pagination: {},
});
expect(response.status).toEqual(200);
expect(response.body).toStrictEqual({
@@ -416,6 +418,7 @@ describe('createRouter', () => {
createdBy: '',
},
],
totalTasks: 1,
});
});
@@ -432,15 +435,12 @@ describe('createRouter', () => {
createdBy: 'user:default/foo',
},
],
totalTasks: 1,
});
const response = await request(app).get(
`/v2/tasks?createdBy=user:default/foo&status=completed`,
`/v2/tasks?createdBy=user:default/foo&createdBy=user:default/bar&status=completed&status=open&limit=1&offset=0&order=desc:created_at`,
);
expect(taskBroker.list).toHaveBeenCalledWith({
createdBy: 'user:default/foo',
status: 'completed',
});
expect(response.status).toEqual(200);
expect(response.body).toStrictEqual({
@@ -453,6 +453,18 @@ describe('createRouter', () => {
createdBy: 'user:default/foo',
},
],
totalTasks: 1,
});
expect(taskBroker.list).toHaveBeenCalledWith({
filters: {
createdBy: ['user:default/foo', 'user:default/bar'],
status: ['completed', 'open'],
},
pagination: {
limit: 1,
offset: 0,
},
order: [{ order: 'desc', field: 'created_at' }],
});
});
});
@@ -1198,11 +1210,13 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{
createdBy: '',
},
],
totalTasks: 1,
});
const response = await request(app).get(`/v2/tasks`);
expect(taskBroker.list).toHaveBeenCalledWith({
createdBy: undefined,
pagination: {},
filters: {},
});
expect(response.status).toEqual(200);
expect(response.body).toStrictEqual({
@@ -1215,6 +1229,7 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{
createdBy: '',
},
],
totalTasks: 1,
});
});
@@ -1231,13 +1246,17 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{
createdBy: 'user:default/foo',
},
],
totalTasks: 1,
});
const response = await request(app).get(
`/v2/tasks?createdBy=user:default/foo`,
);
expect(taskBroker.list).toHaveBeenCalledWith({
createdBy: 'user:default/foo',
filters: {
createdBy: ['user:default/foo'],
},
pagination: {},
});
expect(response.status).toEqual(200);
@@ -1251,6 +1270,7 @@ data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{
createdBy: 'user:default/foo',
},
],
totalTasks: 1,
});
});
});
@@ -69,7 +69,13 @@ import {
} from '../scaffolder';
import { createDryRunner } from '../scaffolder/dryrun';
import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker';
import { findTemplate, getEntityBaseUrl, getWorkingDirectory } from './helpers';
import {
findTemplate,
getEntityBaseUrl,
getWorkingDirectory,
parseNumberParam,
parseStringsParam,
} from './helpers';
import { PermissionRuleParams } from '@backstage/plugin-permission-common';
import {
createConditionAuthorizer,
@@ -81,13 +87,13 @@ import { Duration } from 'luxon';
import {
AuthService,
BackstageCredentials,
DatabaseService,
DiscoveryService,
HttpAuthService,
LifecycleService,
PermissionsService,
UrlReaderService,
SchedulerService,
DatabaseService,
UrlReaderService,
} from '@backstage/backend-plugin-api';
import {
IdentityApi,
@@ -576,31 +582,42 @@ export async function createRouter(
permissionService: permissions,
});
const [userEntityRef] = [req.query.createdBy].flat();
if (
typeof userEntityRef !== 'string' &&
typeof userEntityRef !== 'undefined'
) {
throw new InputError('createdBy query parameter must be a string');
}
if (!taskBroker.list) {
throw new Error(
'TaskBroker does not support listing tasks, please implement the list method on the TaskBroker.',
);
}
const [statusQuery] = [req.query.status].flat();
if (
typeof statusQuery !== 'string' &&
typeof statusQuery !== 'undefined'
) {
throw new InputError('status query parameter must be a string');
}
const createdBy = parseStringsParam(req.query.createdBy, 'createdBy');
const status = parseStringsParam(req.query.status, 'status');
const order = parseStringsParam(req.query.order, 'order')?.map(item => {
const match = item.match(/^(asc|desc):(.+)$/);
if (!match) {
throw new InputError(
`Invalid order parameter "${item}", expected "<asc or desc>:<field name>"`,
);
}
return {
order: match[1] as 'asc' | 'desc',
field: match[2],
};
});
const limit = parseNumberParam(req.query.limit, 'limit');
const offset = parseNumberParam(req.query.offset, 'offset');
const tasks = await taskBroker.list({
createdBy: userEntityRef,
status: statusQuery ? (statusQuery as TaskStatus) : undefined,
filters: {
createdBy,
status: status ? (status as TaskStatus[]) : undefined,
},
order,
pagination: {
limit: limit ? limit[0] : undefined,
offset: offset ? offset[0] : undefined,
},
});
res.status(200).json(tasks);
+20 -1
View File
@@ -315,8 +315,27 @@ export interface TaskBroker {
// (undocumented)
get(taskId: string): Promise<SerializedTask>;
// (undocumented)
list?(options?: { createdBy?: string; status?: TaskStatus }): Promise<{
list?(options?: {
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: {
order: 'asc' | 'desc';
field: string;
}[];
}): Promise<{
tasks: SerializedTask[];
totalTasks?: number;
}>;
// @deprecated (undocumented)
list?(options: { createdBy?: string; status?: TaskStatus }): Promise<{
tasks: SerializedTask[];
totalTasks?: number;
}>;
// (undocumented)
recoverTasks?(): Promise<void>;
+16 -1
View File
@@ -181,7 +181,22 @@ export interface TaskBroker {
get(taskId: string): Promise<SerializedTask>;
list?(options?: {
filters?: {
createdBy?: string | string[];
status?: TaskStatus | TaskStatus[];
};
pagination?: {
limit?: number;
offset?: number;
};
order?: { order: 'asc' | 'desc'; field: string }[];
}): Promise<{ tasks: SerializedTask[]; totalTasks?: number }>;
/**
* @deprecated Make sure to pass `createdBy` and `status` in the `filters` parameter instead
*/
list?(options: {
createdBy?: string;
status?: TaskStatus;
}): Promise<{ tasks: SerializedTask[] }>;
}): Promise<{ tasks: SerializedTask[]; totalTasks?: number }>;
}