paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

Merge pull request #4993 from backstage/freben/catalog-paging

Browse Source 
Introduce paging in the catalog /entities endpoint
This commit is contained in:
Fredrik Adelöw
2021-03-19 10:09:32 +01:00
committed by GitHub
parent 9570256146 13871f4fee
commit 65cc75ae3e
23 changed files with 844 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts
+68 -41
View File
@@ -17,7 +17,7 @@
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, LOCATION_ANNOTATION } from '@backstage/catalog-model';
import { Database, DatabaseManager, Transaction } from '../database';
import { EntityFilters } from '../service/EntityFilters';
import { basicEntityFilter } from '../service/request';
import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
import { EntityUpsertRequest } from './types';
@@ -63,7 +63,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([]);
db.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
@@ -74,12 +77,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
{ entity: expect.anything(), relations: [] },
@@ -96,7 +100,10 @@ describe('DatabaseEntitiesCatalog', () => {
namespace: 'd',
},
};
db.entities.mockResolvedValue([]);
db.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
@@ -108,12 +115,13 @@ describe('DatabaseEntitiesCatalog', () => {
);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.addEntities).toHaveBeenCalledTimes(1);
expect(db.addEntities).toHaveBeenCalledWith(expect.anything(), [
{ entity: expect.anything(), relations: [] },
@@ -147,7 +155,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
},
};
db.entities.mockResolvedValue([{ entity: dbEntity }]);
db.entities.mockResolvedValue({
entities: [{ entity: dbEntity }],
pageInfo: { hasNext: false },
});
db.addEntities.mockResolvedValue([
{ entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
]);
@@ -198,7 +209,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([existing]);
db.entities.mockResolvedValue({
entities: [existing],
pageInfo: { hasNext: false },
});
db.entityByUid.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue({ entity });
@@ -208,12 +222,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).toHaveBeenCalledTimes(1);
expect(db.entityByUid).toHaveBeenCalledWith(transaction, 'u');
@@ -272,7 +287,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([existing]);
db.entities.mockResolvedValue({
entities: [existing],
pageInfo: { hasNext: false },
});
db.entityByName.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue(existing);
@@ -282,12 +300,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).toHaveBeenCalledTimes(1);
expect(db.entityByName).toHaveBeenCalledWith(transaction, {
kind: 'b',
@@ -334,7 +353,10 @@ describe('DatabaseEntitiesCatalog', () => {
},
};
db.entities.mockResolvedValue([{ entity }]);
db.entities.mockResolvedValue({
entities: [{ entity }],
pageInfo: { hasNext: false },
});
db.entityByUid.mockResolvedValue({ entity });
db.updateEntity.mockResolvedValue({ entity });
@@ -344,12 +366,13 @@ describe('DatabaseEntitiesCatalog', () => {
]);
expect(db.entities).toHaveBeenCalledTimes(1);
expect(db.entities).toHaveBeenCalledWith(
expect.anything(),
EntityFilters.ofFilterString(
'kind=b,metadata.namespace=d,metadata.name=c',
),
);
expect(db.entities).toHaveBeenCalledWith(expect.anything(), {
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
});
expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).not.toHaveBeenCalled();
expect(db.updateEntity).not.toHaveBeenCalled();
@@ -377,7 +400,7 @@ describe('DatabaseEntitiesCatalog', () => {
await catalog.batchAddOrUpdateEntities(entities);
const afterFirst = await catalog.entities();
expect(afterFirst.length).toBe(300);
expect(afterFirst.entities.length).toBe(300);
entities[40].entity.metadata.op = 'changed';
entities.push({
@@ -391,9 +414,13 @@ describe('DatabaseEntitiesCatalog', () => {
await catalog.batchAddOrUpdateEntities(entities);
const afterSecond = await catalog.entities();
expect(afterSecond.length).toBe(301);
expect(afterSecond.find(e => e.metadata.op === 'changed')).toBeDefined();
expect(afterSecond.find(e => e.metadata.op === 'added')).toBeDefined();
expect(afterSecond.entities.length).toBe(301);
expect(
afterSecond.entities.find(e => e.metadata.op === 'changed'),
).toBeDefined();
expect(
afterSecond.entities.find(e => e.metadata.op === 'added'),
).toBeDefined();
}, 10000);
});
});
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+42 -28
View File
@@ -14,7 +14,6 @@
* limitations under the License.
*/
import { ConflictError, NotFoundError } from '@backstage/errors';
import {
Entity,
entityHasChanges,
@@ -23,19 +22,18 @@ import {
LOCATION_ANNOTATION,
serializeEntityRef,
} from '@backstage/catalog-model';
import { ConflictError, NotFoundError } from '@backstage/errors';
import { chunk, groupBy } from 'lodash';
import limiterFactory from 'p-limit';
import { Logger } from 'winston';
import type {
Database,
DbEntityResponse,
EntityFilter,
Transaction,
} from '../database';
import { EntityFilters } from '../service/EntityFilters';
import type { Database, DbEntityResponse, Transaction } from '../database';
import { DbEntitiesRequest } from '../database/types';
import { basicEntityFilter } from '../service/request';
import { durationText } from '../util/timing';
import type {
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
EntityUpsertRequest,
EntityUpsertResponse,
} from './types';
@@ -65,11 +63,24 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
private readonly logger: Logger,
) {}
async entities(filter?: EntityFilter): Promise<Entity[]> {
const items = await this.database.transaction(tx =>
this.database.entities(tx, filter),
async entities(request?: EntitiesRequest): Promise<EntitiesResponse> {
const dbRequest: DbEntitiesRequest = {
filter: request?.filter,
pagination: request?.pagination,
};
const dbResponse = await this.database.transaction(tx =>
this.database.entities(tx, dbRequest),
);
return items.map(i => i.entity);
const entities = dbResponse.entities.map(e =>
request?.fields ? request.fields(e.entity) : e.entity,
);
return {
entities,
pageInfo: dbResponse.pageInfo,
};
}
async removeEntityByUid(uid: string): Promise<void> {
@@ -78,16 +89,20 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
if (!entityResponse) {
throw new NotFoundError(`Entity with ID ${uid} was not found`);
}
const location =
entityResponse.entity.metadata.annotations?.[LOCATION_ANNOTATION];
const colocatedEntities = location
? await this.database.entities(
tx,
EntityFilters.ofMatchers({
[`metadata.annotations.${LOCATION_ANNOTATION}`]: location,
}),
)
? (
await this.database.entities(tx, {
filter: basicEntityFilter({
[`metadata.annotations.${LOCATION_ANNOTATION}`]: location,
}),
})
).entities
: [entityResponse];
for (const dbResponse of colocatedEntities) {
await this.database.removeEntityByUid(
tx,
@@ -98,6 +113,7 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
if (entityResponse.locationId) {
await this.database.removeLocation(tx, entityResponse?.locationId!);
}
return undefined;
});
}
@@ -206,13 +222,12 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
}
if (options?.outputEntities && responses.length > 0) {
const writtenEntities = await this.database.entities(
tx,
EntityFilters.ofMatchers({
const writtenEntities = await this.database.entities(tx, {
filter: basicEntityFilter({
'metadata.uid': responses.map(e => e.entityId),
}),
);
responses = writtenEntities.map(e => ({
});
responses = writtenEntities.entities.map(e => ({
entityId: e.entity.metadata.uid!,
entity: e.entity,
}));
@@ -247,17 +262,16 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
// Here we make use of the fact that all of the entities share kind and
// namespace within a batch
const names = requests.map(({ entity }) => entity.metadata.name);
const oldEntities = await this.database.entities(
tx,
EntityFilters.ofMatchers({
const oldEntitiesResponse = await this.database.entities(tx, {
filter: basicEntityFilter({
kind: kind,
'metadata.namespace': namespace,
'metadata.name': names,
}),
);
});
const oldEntitiesByName = new Map(
oldEntities.map(e => [e.entity.metadata.name, e.entity]),
oldEntitiesResponse.entities.map(e => [e.entity.metadata.name, e.entity]),
);
const toAdd: EntityUpsertRequest[] = [];
plugins/catalog-backend/src/catalog/types.ts
+23 -3
View File
@@ -15,12 +15,32 @@
*/
import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model';
import type { EntityFilter } from '../database';
import { EntityFilter, EntityPagination } from '../database/types';
//
// Entities
//
export type PageInfo =
| {
hasNext: false;
}
| {
hasNext: true;
endCursor: string;
};
export type EntitiesRequest = {
filter?: EntityFilter;
fields?: (entity: Entity) => Entity;
pagination?: EntityPagination;
};
export type EntitiesResponse = {
entities: Entity[];
pageInfo: PageInfo;
};
export type EntityUpsertRequest = {
entity: Entity;
relations: EntityRelationSpec[];
@@ -35,9 +55,9 @@ export type EntitiesCatalog = {
/**
* Fetch entities.
*
* @param filter A filter to apply when reading
* @param request Request options
*/
entities(filter?: EntityFilter): Promise<Entity[]>;
entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
/**
* Removes a single entity.
plugins/catalog-backend/src/database/CommonDatabase.test.ts
+20 -18
View File
@@ -14,9 +14,9 @@
* limitations under the License.
*/
import { ConflictError } from '@backstage/errors';
import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
import { EntityFilters } from '../service/EntityFilters';
import { ConflictError } from '@backstage/errors';
import { basicEntityFilter } from '../service/request';
import { DatabaseManager } from './DatabaseManager';
import type {
DbEntityRequest,
@@ -385,8 +385,8 @@ describe('CommonDatabase', () => {
]);
});
const result = await db.transaction(async tx => db.entities(tx));
expect(result.length).toEqual(2);
expect(result).toEqual(
expect(result.entities.length).toEqual(2);
expect(result.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
@@ -424,11 +424,13 @@ describe('CommonDatabase', () => {
);
});
await expect(
db.transaction(async tx =>
db.entities(tx, EntityFilters.ofFilterString('kind=k2,spec.c=some')),
),
).resolves.toEqual([
const response = await db.transaction(async tx =>
db.entities(tx, {
filter: basicEntityFilter({ kind: 'k2', 'spec.c': 'some' }),
}),
);
expect(response.entities).toEqual([
{
locationId: undefined,
entity: expect.objectContaining({ kind: 'k2' }),
@@ -466,14 +468,13 @@ describe('CommonDatabase', () => {
});
const rows = await db.transaction(async tx =>
db.entities(
tx,
EntityFilters.ofFilterString('ApiVersioN=A,spEc.C=some'),
),
db.entities(tx, {
filter: basicEntityFilter({ ApiVersioN: 'A', 'spEc.C': 'some' }),
}),
);
expect(rows.length).toEqual(3);
expect(rows).toEqual(
expect(rows.entities.length).toEqual(3);
expect(rows.entities).toEqual(
expect.arrayContaining([
{
locationId: undefined,
@@ -531,7 +532,8 @@ describe('CommonDatabase', () => {
{ target: mockRelations[0].target, type: 'child' },
]);
const [returnedEntity3] = await db.transaction(tx => db.entities(tx));
const { entities } = await db.transaction(tx => db.entities(tx));
const [returnedEntity3] = entities;
expect(returnedEntity3?.entity.relations).toEqual([
{ target: mockRelations[0].target, type: 'child' },
]);
@@ -615,7 +617,7 @@ describe('CommonDatabase', () => {
const res = await db.transaction(tx => db.entities(tx));
expect(
res.map(r => ({
res.entities.map(r => ({
name: r.entity.metadata.name,
relations: r.entity.relations,
})),
@@ -660,7 +662,7 @@ describe('CommonDatabase', () => {
const res2 = await db.transaction(tx => db.entities(tx));
expect(
res2.map(r => ({
res2.entities.map(r => ({
name: r.entity.metadata.name,
relations: r.entity.relations,
})),
plugins/catalog-backend/src/database/CommonDatabase.ts
+76 -7
View File
@@ -35,13 +35,16 @@ import {
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
DbEntitiesRelationsRow,
DbEntitiesRequest,
DbEntitiesResponse,
DbEntitiesRow,
DbEntitiesSearchRow,
DbEntityRequest,
DbEntityResponse,
DbLocationsRow,
DbLocationsRowWithStatus,
EntityFilter,
DbPageInfo,
EntityPagination,
Transaction,
} from './types';
@@ -207,13 +210,13 @@ export class CommonDatabase implements Database {
async entities(
txOpaque: Transaction,
filter?: EntityFilter,
): Promise<DbEntityResponse[]> {
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse> {
const tx = txOpaque as Knex.Transaction;
let entitiesQuery = tx<DbEntitiesRow>('entities');
for (const singleFilter of filter?.anyOf ?? []) {
for (const singleFilter of request?.filter?.anyOf ?? []) {
entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
for (const { key, matchValueIn } of singleFilter.allOf) {
// NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
@@ -235,17 +238,43 @@ export class CommonDatabase implements Database {
}
}
});
this.andWhere('id', 'in', matchQuery);
}
});
}
const rows = await entitiesQuery
entitiesQuery = entitiesQuery
.select('entities.*')
.orderBy('full_name', 'asc');
return this.toEntityResponses(tx, rows);
const { limit, offset } = parsePagination(request?.pagination);
if (limit !== undefined) {
entitiesQuery = entitiesQuery.limit(limit + 1);
}
if (offset !== undefined) {
entitiesQuery = entitiesQuery.offset(offset);
}
let rows = await entitiesQuery;
let pageInfo: DbPageInfo;
if (limit === undefined || rows.length <= limit) {
pageInfo = { hasNext: false };
} else {
rows = rows.slice(0, -1);
pageInfo = {
hasNext: true,
endCursor: stringifyPagination({
limit,
offset: (offset ?? 0) + limit,
}),
};
}
return {
entities: await this.toEntityResponses(tx, rows),
pageInfo,
};
}
async entityByName(
@@ -519,6 +548,46 @@ export class CommonDatabase implements Database {
}
}
function parsePagination(
input?: EntityPagination,
): { limit?: number; offset?: number } {
if (!input) {
return {};
}
let { limit, offset } = input;
if (input.after !== undefined) {
let cursor;
try {
const json = Buffer.from(input.after, 'base64').toString('utf8');
cursor = JSON.parse(json);
} catch {
throw new InputError('Malformed after cursor, could not be parsed');
}
if (cursor.limit !== undefined) {
if (!Number.isInteger(cursor.limit)) {
throw new InputError('Malformed after cursor, limit was not an number');
}
limit = cursor.limit;
}
if (cursor.offset !== undefined) {
if (!Number.isInteger(cursor.offset)) {
throw new InputError('Malformed after cursor, offset was not a number');
}
offset = cursor.offset;
}
}
return { limit, offset };
}
function stringifyPagination(input: { limit: number; offset: number }) {
const json = JSON.stringify({ limit: input.limit, offset: input.offset });
const base64 = Buffer.from(json, 'utf8').toString('base64');
return base64;
}
function deduplicateRelations(
rows: DbEntitiesRelationsRow[],
): DbEntitiesRelationsRow[] {
plugins/catalog-backend/src/database/index.ts
+1
View File
@@ -22,5 +22,6 @@ export type {
DbEntityResponse,
EntitiesSearchFilter,
EntityFilter,
EntityPagination,
Transaction,
} from './types';
plugins/catalog-backend/src/database/types.ts
+32 -1
View File
@@ -36,6 +36,25 @@ export type DbEntityRequest = {
relations: EntityRelationSpec[];
};
export type DbEntitiesRequest = {
filter?: EntityFilter;
pagination?: EntityPagination;
};
export type DbEntitiesResponse = {
entities: DbEntityResponse[];
pageInfo: DbPageInfo;
};
export type DbPageInfo =
| {
hasNext: false;
}
| {
hasNext: true;
endCursor: string;
};
export type DbEntityResponse = {
locationId?: string;
entity: Entity;
@@ -111,6 +130,15 @@ export type EntityFilter = {
anyOf: { allOf: EntitiesSearchFilter[] }[];
};
/**
* A pagination rule for entities.
*/
export type EntityPagination = {
limit?: number;
offset?: number;
after?: string;
};
/**
* An abstraction for transactions of the underlying database technology.
*/
@@ -170,7 +198,10 @@ export type Database = {
matchingGeneration?: number,
): Promise<DbEntityResponse>;
entities(tx: Transaction, filter?: EntityFilter): Promise<DbEntityResponse[]>;
entities(
tx: Transaction,
request?: DbEntitiesRequest,
): Promise<DbEntitiesResponse>;
entityByName(
tx: Transaction,
plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
+4 -1
View File
@@ -360,7 +360,10 @@ describe('HigherOrderOperations', () => {
entities: [{ entity: desc, location, relations: [] }],
errors: [],
});
entitiesCatalog.entities.mockResolvedValue([]);
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]);
await expect(
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+6 -3
View File
@@ -61,7 +61,10 @@ describe('CatalogBuilder', () => {
it('works with no changes', async () => {
const builder = new CatalogBuilder(env);
const built = await builder.build();
await expect(built.entitiesCatalog.entities()).resolves.toEqual([]);
await expect(built.entitiesCatalog.entities()).resolves.toEqual({
entities: [],
pageInfo: { hasNext: false },
});
await expect(built.locationsCatalog.locations()).resolves.toEqual([
expect.objectContaining({
data: expect.objectContaining({ type: 'bootstrap' }),
@@ -166,7 +169,7 @@ describe('CatalogBuilder', () => {
type: 'github',
target: 'https://github.com/a/b/x.yaml',
});
const entities = await entitiesCatalog.entities();
const { entities } = await entitiesCatalog.entities();
expect(entities).toEqual([
expect.objectContaining({
@@ -200,7 +203,7 @@ describe('CatalogBuilder', () => {
type: 'x',
target: 'y',
});
const entities = await entitiesCatalog.entities();
const { entities } = await entitiesCatalog.entities();
expect.assertions(3);
expect(entities).toEqual([
plugins/catalog-backend/src/service/EntityFilters.test.ts
-113
View File
@@ -1,113 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityFilters } from './EntityFilters';
describe('EntityFilters', () => {
describe('ofQuery', () => {
it('translates empty query to empty list', () => {
const result = EntityFilters.ofQuery({});
expect(result).toEqual(undefined);
});
it('supports single-string format', () => {
const result = EntityFilters.ofQuery({ filter: 'a=1' })!;
expect(result).toEqual({
anyOf: [{ allOf: [{ key: 'a', matchValueIn: ['1'] }] }],
});
});
it('supports array-of-strings format', () => {
const result = EntityFilters.ofQuery({ filter: ['a=1', 'b=2'] });
expect(result).toEqual({
anyOf: [
{ allOf: [{ key: 'a', matchValueIn: ['1'] }] },
{ allOf: [{ key: 'b', matchValueIn: ['2'] }] },
],
});
});
it('throws for non-strings', () => {
expect(() => EntityFilters.ofQuery({ filter: [3] })).toThrow(/string/);
});
});
describe('ofFilterString', () => {
it('runs the happy path', () => {
const result = EntityFilters.ofFilterString('a=1,b=2');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1'] },
{ key: 'b', matchValueIn: ['2'] },
],
},
],
});
});
it('ignores empty', () => {
const result = EntityFilters.ofFilterString('a=1,,b=2,');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1'] },
{ key: 'b', matchValueIn: ['2'] },
],
},
],
});
});
it('trims', () => {
const result = EntityFilters.ofFilterString(' a = 1 ,, b=2 ,');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1'] },
{ key: 'b', matchValueIn: ['2'] },
],
},
],
});
});
it('merges multiple of the same key', () => {
const result = EntityFilters.ofFilterString('a=1,a=2,b=3');
expect(result).toEqual({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1', '2'] },
{ key: 'b', matchValueIn: ['3'] },
],
},
],
});
});
it('throws on missing equal sign', () => {
expect(() => EntityFilters.ofFilterString('a,b=2')).toThrow();
});
it('throws on misplaced equal sign', () => {
expect(() => EntityFilters.ofFilterString('a=,b=2')).toThrow();
});
});
});
plugins/catalog-backend/src/service/EntityFilters.ts
-120
View File
@@ -1,120 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { EntitiesSearchFilter, EntityFilter } from '../database';
/**
* A builder that assists in creating entity filter instances.
*/
export class EntityFilters {
// Builds a filter from the value of a filter=a=1,b=2 type filter string
static ofFilterString(filterString: string): EntityFilter | undefined {
const builder = new EntityFilters();
builder.addFilterString(filterString);
return builder.build();
}
// Builds a filter from an entire query params structure
static ofQuery(query: Record<string, any>): EntityFilter | undefined {
const filterStrings = [query.filter || []].flat();
if (filterStrings.some(f => typeof f !== 'string')) {
throw new InputError(
'Only string type filter query parameters are supported',
);
}
const builder = new EntityFilters();
for (const filterString of filterStrings) {
builder.addFilterString(filterString);
}
return builder.build();
}
// Builds a filter from a { key: value } matcher object
static ofMatchers(
matchers: Record<string, string | string[]>,
): EntityFilter | undefined {
const builder = new EntityFilters();
builder.addMatchers(matchers);
return builder.build();
}
private filters: EntitiesSearchFilter[][];
constructor() {
this.filters = [];
}
addFilterString(filterString: string): EntityFilters {
const filtersByKey: Record<string, EntitiesSearchFilter> = {};
const addFilter = (key: string, value: string) => {
const f =
key in filtersByKey
? filtersByKey[key]
: (filtersByKey[key] = { key, matchValueIn: [] });
f.matchValueIn!.push(value);
};
const statements = filterString
.split(',')
.map(s => s.trim())
.filter(Boolean);
for (const statement of statements) {
const equalsIndex = statement.indexOf('=');
if (equalsIndex < 1) {
throw new InputError('Malformed filter query');
} else {
const key = statement.substr(0, equalsIndex).trim();
const value = statement.substr(equalsIndex + 1).trim();
if (!key || !value) {
throw new InputError('Malformed filter query');
}
addFilter(key, value);
}
}
this.filters.push(Object.values(filtersByKey));
return this;
}
addMatchers(matchers: Record<string, string | string[]>): EntityFilters {
const filters: EntitiesSearchFilter[] = [];
for (const [key, value] of Object.entries(matchers)) {
filters.push({ key, matchValueIn: [value].flat() });
}
if (filters.length) {
this.filters.push(filters);
}
return this;
}
build(): EntityFilter | undefined {
if (!this.filters.length) {
return undefined;
}
return { anyOf: this.filters.map(f => ({ allOf: f })) };
}
}
plugins/catalog-backend/src/service/request/basicEntityFilter.ts
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntitiesSearchFilter, EntityFilter } from '../../database';
/**
* Forms a full EntityFilter based on a single key-value(s) object.
*/
export function basicEntityFilter(
items: Record<string, string | string[]>,
): EntityFilter {
const filtersByKey: Record<string, EntitiesSearchFilter> = {};
for (const [key, value] of Object.entries(items)) {
const values = [value].flat();
const f =
key in filtersByKey
? filtersByKey[key]
: (filtersByKey[key] = { key, matchValueIn: [] });
f.matchValueIn!.push(...values);
}
return { anyOf: [{ allOf: Object.values(filtersByKey) }] };
}
plugins/catalog-backend/src/service/request/common.ts
+79
View File
@@ -0,0 +1,79 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
/**
* Takes a single unknown parameter and makes sure that it's a string that can
* be parsed as an integer.
*/
export function parseIntegerParam(
param: unknown,
ctx: string,
): number | undefined {
if (param === undefined) {
return undefined;
}
if (typeof param !== 'string') {
throw new InputError(`Invalid ${ctx}, not an integer on string form`);
}
const parsed = parseInt(param, 10);
if (!Number.isInteger(parsed) || String(parsed) !== param) {
throw new InputError(`Invalid ${ctx}, not an integer`);
}
return parsed;
}
/**
* Takes a single unknown parameter and makes sure that it's a string.
*/
export function parseStringParam(
param: unknown,
ctx: string,
): string | undefined {
if (param === undefined) {
return undefined;
}
if (typeof param !== 'string') {
throw new InputError(`Invalid ${ctx}, not a string`);
}
return param;
}
/**
* Takes a single unknown parameter and makes sure that it's a single string or
* an array of strings, and returns as an array.
*/
export function parseStringsParam(
param: unknown,
ctx: string,
): string[] | undefined {
if (param === undefined) {
return undefined;
}
const array = [param].flat();
if (array.some(p => typeof p !== 'string')) {
throw new InputError(`Invalid ${ctx}, not a string`);
}
return array as string[];
}
plugins/catalog-backend/src/service/request/index.ts
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { basicEntityFilter } from './basicEntityFilter';
export { parseEntityFilterParams } from './parseEntityFilterParams';
export { parseEntityPaginationParams } from './parseEntityPaginationParams';
export { parseEntityTransformParams } from './parseEntityTransformParams';
plugins/catalog-backend/src/service/request/parseEntityFilterParams.test.ts
+99
View File
@@ -0,0 +1,99 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
parseEntityFilterParams,
parseEntityFilterString,
} from './parseEntityFilterParams';
describe('parseEntityFilterParams', () => {
it('translates empty query to empty list', () => {
const result = parseEntityFilterParams({});
expect(result).toEqual(undefined);
});
it('supports single-string format', () => {
const result = parseEntityFilterParams({ filter: 'a=1' })!;
expect(result).toEqual({
anyOf: [{ allOf: [{ key: 'a', matchValueIn: ['1'] }] }],
});
});
it('supports array-of-strings format', () => {
const result = parseEntityFilterParams({
filter: ['a=1', 'b=2'],
});
expect(result).toEqual({
anyOf: [
{ allOf: [{ key: 'a', matchValueIn: ['1'] }] },
{ allOf: [{ key: 'b', matchValueIn: ['2'] }] },
],
});
});
it('merges values within each filter', () => {
const result = parseEntityFilterParams({
filter: ['a=1', 'b=2,b=3,c=4'],
});
expect(result).toEqual({
anyOf: [
{ allOf: [{ key: 'a', matchValueIn: ['1'] }] },
{
allOf: [
{ key: 'b', matchValueIn: ['2', '3'] },
{ key: 'c', matchValueIn: ['4'] },
],
},
],
});
});
it('throws for non-strings', () => {
expect(() => parseEntityFilterParams({ filter: [3] })).toThrow(/string/);
});
});
describe('parseEntityFilterString', () => {
it('works for the happy path', () => {
expect(parseEntityFilterString('')).toBeUndefined();
expect(parseEntityFilterString('a=1,b=2,a=3')).toEqual([
{ key: 'a', matchValueIn: ['1', '3'] },
{ key: 'b', matchValueIn: ['2'] },
]);
});
it('trims values', () => {
expect(parseEntityFilterString(' a = 1 , b = 2 , a = 3 ')).toEqual([
{ key: 'a', matchValueIn: ['1', '3'] },
{ key: 'b', matchValueIn: ['2'] },
]);
});
it('rejects malformed strings', () => {
expect(() => parseEntityFilterString('x=2,a=')).toThrow(
"Invalid filter, 'a=' is not a valid statement (expected a string on the form a=b)",
);
expect(() => parseEntityFilterString('x=2,=a')).toThrow(
"Invalid filter, '=a' is not a valid statement (expected a string on the form a=b)",
);
expect(() => parseEntityFilterString('x=2,=')).toThrow(
"Invalid filter, '=' is not a valid statement (expected a string on the form a=b)",
);
expect(() => parseEntityFilterString('x=2,a')).toThrow(
"Invalid filter, 'a' is not a valid statement (expected a string on the form a=b)",
);
});
});
plugins/catalog-backend/src/service/request/parseEntityFilterParams.ts
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { EntitiesSearchFilter, EntityFilter } from '../../database';
import { parseStringsParam } from './common';
/**
* Parses the filtering part of a query, like
* /entities?filter=metadata.namespace=default,kind=Component
*/
export function parseEntityFilterParams(
params: Record<string, unknown>,
): EntityFilter | undefined {
// Each filter string is on the form a=b,c=d
const filterStrings = parseStringsParam(params.filter, 'filter');
if (!filterStrings) {
return undefined;
}
// Outer array: "any of the inner ones"
// Inner arrays: "all of these must match"
const filters = filterStrings.map(parseEntityFilterString).filter(Boolean);
if (!filters.length) {
return undefined;
}
return { anyOf: filters.map(f => ({ allOf: f! })) };
}
/**
* Parses a single filter string as seen in a filter query, for example
* metadata.namespace=default,kind=Component
*/
export function parseEntityFilterString(
filterString: string,
): EntitiesSearchFilter[] | undefined {
const statements = filterString
.split(',')
.map(s => s.trim())
.filter(Boolean);
if (!statements.length) {
return undefined;
}
const filtersByKey: Record<string, EntitiesSearchFilter> = {};
for (const statement of statements) {
const equalsIndex = statement.indexOf('=');
if (equalsIndex < 1) {
throw new InputError(
`Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`,
);
}
const key = statement.substr(0, equalsIndex).trim();
const value = statement.substr(equalsIndex + 1).trim();
if (!key || !value) {
throw new InputError(
`Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b)`,
);
}
const f =
key in filtersByKey
? filtersByKey[key]
: (filtersByKey[key] = { key, matchValueIn: [] });
f.matchValueIn!.push(value);
}
return Object.values(filtersByKey);
}
plugins/catalog-backend/src/service/request/parseEntityPaginationParams.test.ts
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { parseEntityPaginationParams } from './parseEntityPaginationParams';
describe('parseEntityPaginationParams', () => {
it('works for the happy path', () => {
expect(parseEntityPaginationParams({})).toBeUndefined();
expect(parseEntityPaginationParams({ limit: '1' })).toEqual({ limit: 1 });
expect(parseEntityPaginationParams({ offset: '0' })).toEqual({ offset: 0 });
expect(parseEntityPaginationParams({ offset: '2' })).toEqual({ offset: 2 });
expect(parseEntityPaginationParams({ after: 'x' })).toEqual({ after: 'x' });
expect(
parseEntityPaginationParams({ limit: '1', offset: '2', after: 'x' }),
).toEqual({ limit: 1, offset: 2, after: 'x' });
});
it('rejects bad values', () => {
expect(() => parseEntityPaginationParams({ limit: '' })).toThrow(
'Invalid limit, not an integer',
);
expect(() => parseEntityPaginationParams({ limit: '0' })).toThrow(
'Invalid limit, must be greater than zero',
);
expect(() => parseEntityPaginationParams({ limit: '-1' })).toThrow(
'Invalid limit, must be greater than zero',
);
expect(() => parseEntityPaginationParams({ offset: '' })).toThrow(
'Invalid offset, not an integer',
);
expect(() => parseEntityPaginationParams({ offset: '-1' })).toThrow(
'Invalid offset, must be zero or greater',
);
expect(() => parseEntityPaginationParams({ after: '' })).toThrow(
'Invalid after, must not be empty',
);
});
});
plugins/catalog-backend/src/service/request/parseEntityPaginationParams.ts
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { EntityPagination } from '../../database';
import { parseIntegerParam, parseStringParam } from './common';
/**
* Parses the pagination related parameters out of a query, e.g.
* /entities?offset=100&limit=10
*/
export function parseEntityPaginationParams(
params: Record<string, unknown>,
): EntityPagination | undefined {
const offset = parseIntegerParam(params.offset, 'offset');
const limit = parseIntegerParam(params.limit, 'limit');
const after = parseStringParam(params.after, 'after');
if (offset === undefined && limit === undefined && after === undefined) {
return undefined;
}
if (offset !== undefined && offset < 0) {
throw new InputError(`Invalid offset, must be zero or greater`);
}
if (limit !== undefined && limit <= 0) {
throw new InputError(`Invalid limit, must be greater than zero`);
}
if (after !== undefined && !after) {
throw new InputError(`Invalid after, must not be empty`);
}
return {
...(offset !== undefined ? { offset } : {}),
...(limit !== undefined ? { limit } : {}),
...(after !== undefined ? { after } : {}),
};
}
plugins/catalog-backend/src/service/filterQuery.test.ts → plugins/catalog-backend/src/service/request/parseEntityTransformParams.test.ts
+16 -15
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Spotify AB
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,9 +15,9 @@
*/
import { Entity } from '@backstage/catalog-model';
import { translateQueryToFieldMapper } from './filterQuery';
import { parseEntityTransformParams } from './parseEntityTransformParams';
describe('translateQueryToFieldMapper', () => {
describe('parseEntityTransformParams', () => {
const entity: Entity = {
apiVersion: 'av',
kind: 'k',
@@ -30,37 +30,38 @@ describe('translateQueryToFieldMapper', () => {
},
};
it('passes through when no fields given', () => {
expect(translateQueryToFieldMapper({})(entity)).toBe(entity);
expect(translateQueryToFieldMapper({ fields: [] })(entity)).toBe(entity);
expect(translateQueryToFieldMapper({ fields: [''] })(entity)).toBe(entity);
expect(translateQueryToFieldMapper({ fields: [','] })(entity)).toBe(entity);
it('returns undefined when no fields given', () => {
expect(parseEntityTransformParams({})).toBeUndefined();
expect(parseEntityTransformParams({ fields: '' })).toBeUndefined();
expect(parseEntityTransformParams({ fields: [] })).toBeUndefined();
expect(parseEntityTransformParams({ fields: [''] })).toBeUndefined();
expect(parseEntityTransformParams({ fields: [','] })).toBeUndefined();
});
it('rejects attempts at array filtering', () => {
expect(() =>
translateQueryToFieldMapper({ fields: 'metadata.tags[0]' })(entity),
).toThrow(/array/i);
parseEntityTransformParams({ fields: 'metadata.tags[0]' })!(entity),
).toThrow(/invalid fields, array type fields are not supported/i);
});
it('accepts both strings and arrays of strings as input', () => {
expect(translateQueryToFieldMapper({ fields: 'kind' })(entity)).toEqual({
expect(parseEntityTransformParams({ fields: 'kind' })!(entity)).toEqual({
kind: 'k',
});
expect(translateQueryToFieldMapper({ fields: ['kind'] })(entity)).toEqual({
expect(parseEntityTransformParams({ fields: ['kind'] })!(entity)).toEqual({
kind: 'k',
});
expect(
translateQueryToFieldMapper({ fields: ['kind', 'apiVersion'] })(entity),
parseEntityTransformParams({ fields: ['kind', 'apiVersion'] })!(entity),
).toEqual({ apiVersion: 'av', kind: 'k' });
});
it('supports sub-selection properly', () => {
expect(
translateQueryToFieldMapper({ fields: 'kind,metadata.name' })(entity),
parseEntityTransformParams({ fields: 'kind,metadata.name' })!(entity),
).toEqual({ kind: 'k', metadata: { name: 'n' } });
expect(
translateQueryToFieldMapper({ fields: 'metadata' })(entity),
parseEntityTransformParams({ fields: 'metadata' })!(entity),
).toEqual({ metadata: { name: 'n', tags: ['t1', 't2'] } });
});
});
plugins/catalog-backend/src/service/filterQuery.ts → plugins/catalog-backend/src/service/request/parseEntityTransformParams.ts
+12 -22
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Spotify AB
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,26 +14,18 @@
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { Entity } from '@backstage/catalog-model';
import { InputError } from '@backstage/errors';
import lodash from 'lodash';
import { RecursivePartial } from '../util';
import { RecursivePartial } from '../../util';
import { parseStringsParam } from './common';
type FieldMapper = (entity: Entity) => Entity;
export function translateQueryToFieldMapper(
query: Record<string, any>,
): FieldMapper {
if (!query.fields) {
return x => x;
}
const fieldsStrings = [query.fields].flat() as string[];
if (fieldsStrings.some(s => typeof s !== 'string')) {
throw new InputError(
'Only string type fields query parameters are supported',
);
export function parseEntityTransformParams(
params: Record<string, unknown>,
): ((entity: Entity) => Entity) | undefined {
const fieldsStrings = parseStringsParam(params.fields, 'fields');
if (!fieldsStrings) {
return undefined;
}
const fields = fieldsStrings
@@ -43,13 +35,11 @@ export function translateQueryToFieldMapper(
.filter(Boolean);
if (!fields.length) {
return x => x;
return undefined;
}
if (fields.some(f => f.includes('['))) {
throw new InputError(
'Array type fields query parameters are not supported',
);
throw new InputError('invalid fields, array type fields are not supported');
}
return input => {
plugins/catalog-backend/src/service/router.test.ts
+55 -32
View File
@@ -22,8 +22,8 @@ import request from 'supertest';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { LocationResponse } from '../catalog/types';
import { HigherOrderOperation } from '../ingestion/types';
import { EntityFilters } from './EntityFilters';
import { createRouter } from './router';
import { basicEntityFilter } from './request';
describe('createRouter', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
@@ -69,7 +69,10 @@ describe('createRouter', () => {
{ apiVersion: 'a', kind: 'b', metadata: { name: 'n' } },
];
entitiesCatalog.entities.mockResolvedValueOnce(entities);
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [entities[0]],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities');
@@ -78,7 +81,10 @@ describe('createRouter', () => {
});
it('parses single and multiple request parameters and passes them down', async () => {
entitiesCatalog.entities.mockResolvedValueOnce([]);
entitiesCatalog.entities.mockResolvedValueOnce({
entities: [],
pageInfo: { hasNext: false },
});
const response = await request(app).get(
'/entities?filter=a=1,a=2,b=3&filter=c=4',
);
@@ -86,15 +92,17 @@ describe('createRouter', () => {
expect(response.status).toEqual(200);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1', '2'] },
{ key: 'b', matchValueIn: ['3'] },
],
},
{ allOf: [{ key: 'c', matchValueIn: ['4'] }] },
],
filter: {
anyOf: [
{
allOf: [
{ key: 'a', matchValueIn: ['1', '2'] },
{ key: 'b', matchValueIn: ['3'] },
],
},
{ allOf: [{ key: 'c', matchValueIn: ['4'] }] },
],
},
});
});
});
@@ -108,27 +116,33 @@ describe('createRouter', () => {
name: 'c',
},
};
entitiesCatalog.entities.mockResolvedValue([entity]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({ 'metadata.uid': 'zzz' }),
);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue([]);
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({ 'metadata.uid': 'zzz' }),
);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'zzz' }),
});
expect(response.status).toEqual(404);
expect(response.text).toMatch(/uid/);
});
@@ -144,35 +158,41 @@ describe('createRouter', () => {
namespace: 'ns',
},
};
entitiesCatalog.entities.mockResolvedValue([entity]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-name/k/ns/n');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({
kind: 'k',
'metadata.namespace': 'ns',
'metadata.name': 'n',
}),
);
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entities.mockResolvedValue([]);
entitiesCatalog.entities.mockResolvedValue({
entities: [],
pageInfo: { hasNext: false },
});
const response = await request(app).get('/entities/by-name/b/d/c');
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({
kind: 'b',
'metadata.namespace': 'd',
'metadata.name': 'c',
}),
);
});
expect(response.status).toEqual(404);
expect(response.text).toMatch(/name/);
});
@@ -203,7 +223,10 @@ describe('createRouter', () => {
entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
{ entityId: 'u' },
]);
entitiesCatalog.entities.mockResolvedValue([entity]);
entitiesCatalog.entities.mockResolvedValue({
entities: [entity],
pageInfo: { hasNext: false },
});
const response = await request(app)
.post('/entities')
@@ -215,9 +238,9 @@ describe('createRouter', () => {
{ entity, relations: [] },
]);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith(
EntityFilters.ofMatchers({ 'metadata.uid': 'u' }),
);
expect(entitiesCatalog.entities).toHaveBeenCalledWith({
filter: basicEntityFilter({ 'metadata.uid': 'u' }),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual(entity);
});
plugins/catalog-backend/src/service/router.ts
+32 -16
View File
@@ -27,8 +27,12 @@ import { Logger } from 'winston';
import yn from 'yn';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { HigherOrderOperation, LocationAnalyzer } from '../ingestion/types';
import { EntityFilters } from './EntityFilters';
import { translateQueryToFieldMapper } from './filterQuery';
import {
basicEntityFilter,
parseEntityFilterParams,
parseEntityPaginationParams,
parseEntityTransformParams,
} from './request';
import { requireRequestBody, validateRequestBody } from './util';
export interface RouterOptions {
@@ -55,10 +59,22 @@ export async function createRouter(
if (entitiesCatalog) {
router
.get('/entities', async (req, res) => {
const filter = EntityFilters.ofQuery(req.query);
const fieldMapper = translateQueryToFieldMapper(req.query);
const entities = await entitiesCatalog.entities(filter);
res.status(200).json(entities.map(fieldMapper));
const { entities, pageInfo } = await entitiesCatalog.entities({
filter: parseEntityFilterParams(req.query),
fields: parseEntityTransformParams(req.query),
pagination: parseEntityPaginationParams(req.query),
});
// Add a Link header to the next page
if (pageInfo.hasNext) {
const url = new URL(`http://ignored${req.url}`);
url.searchParams.delete('offset');
url.searchParams.set('after', pageInfo.endCursor);
res.setHeader('link', `<${url.pathname}${url.search}>; rel="next"`);
}
// TODO(freben): encode the pageInfo in the response
res.json(entities);
})
.post('/entities', async (req, res) => {
/*
@@ -75,16 +91,16 @@ export async function createRouter(
const [result] = await entitiesCatalog.batchAddOrUpdateEntities([
{ entity: body as Entity, relations: [] },
]);
const [entity] = await entitiesCatalog.entities(
EntityFilters.ofMatchers({ 'metadata.uid': result.entityId }),
);
res.status(200).json(entity);
const response = await entitiesCatalog.entities({
filter: basicEntityFilter({ 'metadata.uid': result.entityId }),
});
res.status(200).json(response.entities[0]);
})
.get('/entities/by-uid/:uid', async (req, res) => {
const { uid } = req.params;
const entities = await entitiesCatalog.entities(
EntityFilters.ofMatchers({ 'metadata.uid': uid }),
);
const { entities } = await entitiesCatalog.entities({
filter: basicEntityFilter({ 'metadata.uid': uid }),
});
if (!entities.length) {
throw new NotFoundError(`No entity with uid ${uid}`);
}
@@ -97,13 +113,13 @@ export async function createRouter(
})
.get('/entities/by-name/:kind/:namespace/:name', async (req, res) => {
const { kind, namespace, name } = req.params;
const entities = await entitiesCatalog.entities(
EntityFilters.ofMatchers({
const { entities } = await entitiesCatalog.entities({
filter: basicEntityFilter({
kind: kind,
'metadata.namespace': namespace,
'metadata.name': name,
}),
);
});
if (!entities.length) {
throw new NotFoundError(
`No entity named '${name}' found, with kind '${kind}' in namespace '${namespace}'`,