paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 15 Packages Projects Releases Wiki Activity

catalog-backend: add support for querying facets

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2022-02-02 11:04:14 +01:00
parent fd02e6df7d
commit 01e124ea60
11 changed files with 388 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/dry-ties-collect.md
+10
View File
@@ -0,0 +1,10 @@
---
'@backstage/plugin-catalog-backend': patch
---
Added an `/entity-facets` endpoint, which lets you query the distribution of
possible values for fields of entities.
This can be useful for example when populating a dropdown in the user interface,
such as listing all tag values that are actually being used right now in your
catalog instance, along with putting the most common ones at the top.
plugins/catalog-backend/api-report.md
+19
View File
@@ -600,6 +600,7 @@ export type EntitiesCatalog = {
authorizationToken?: string;
},
): Promise<EntityAncestryResponse>;
facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse>;
};
// Warning: (ae-missing-release-tag) "EntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -645,6 +646,24 @@ export type EntityAncestryResponse = {
}>;
};
// @public
export interface EntityFacetsRequest {
authorizationToken?: string;
facets: string[];
filter?: EntityFilter;
}
// @public
export interface EntityFacetsResponse {
facets: Record<
string,
Array<{
value: string;
count: number;
}>
>;
}
// Warning: (ae-missing-release-tag) "EntityFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
plugins/catalog-backend/src/catalog/index.ts
+4 -2
View File
@@ -18,9 +18,11 @@ export type {
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
EntityAncestryResponse,
PageInfo,
EntitiesSearchFilter,
EntityAncestryResponse,
EntityFacetsRequest,
EntityFacetsResponse,
EntityFilter,
EntityPagination,
PageInfo,
} from './types';
plugins/catalog-backend/src/catalog/types.ts
+46
View File
@@ -87,6 +87,44 @@ export type EntityAncestryResponse = {
}>;
};
/**
* The request shape for {@link EntitiesCatalog.facets}.
*
* @public
*/
export interface EntityFacetsRequest {
/**
* A filter to apply on the full list of entities before computing the facets.
*/
filter?: EntityFilter;
/**
* The facets to compute.
*
* @remarks
*
* This is a list of strings corresponding to paths within individual entity
* shapes. For example, to compute the facets for all available tags, you
* would pass in the string 'metadata.tags'.
*/
facets: string[];
/**
* The optional token that authorizes the action.
*/
authorizationToken?: string;
}
/**
* The response shape for {@link EntitiesCatalog.facets}.
*
* @public
*/
export interface EntityFacetsResponse {
/**
* The computed facets, one entry per facet in the request.
*/
facets: Record<string, Array<{ value: string; count: number }>>;
}
/** @public */
export type EntitiesCatalog = {
/**
@@ -115,4 +153,12 @@ export type EntitiesCatalog = {
entityRef: string,
options?: { authorizationToken?: string },
): Promise<EntityAncestryResponse>;
/**
* Computes facets for a set of entities, e.g. for populating filter lists
* or driving insights or similar.
*
* @param request - Request options
*/
facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse>;
};
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts
+51
View File
@@ -30,6 +30,7 @@ describe('AuthorizedEntitiesCatalog', () => {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
};
const fakePermissionApi = {
authorize: jest.fn(),
@@ -254,4 +255,54 @@ describe('AuthorizedEntitiesCatalog', () => {
});
});
});
describe('facets', () => {
it('returns empty response on DENY', async () => {
fakePermissionApi.authorize.mockResolvedValue([
{ result: AuthorizeResult.DENY },
]);
const catalog = createCatalog();
expect(
await catalog.facets({
facets: ['a'],
authorizationToken: 'abcd',
}),
).toEqual({
facets: { a: [] },
});
});
it('calls underlying catalog method with correct filter on CONDITIONAL', async () => {
fakePermissionApi.authorize.mockResolvedValue([
{
result: AuthorizeResult.CONDITIONAL,
conditions: { rule: 'IS_ENTITY_KIND', params: [['b']] },
},
]);
const catalog = createCatalog(isEntityKind);
await catalog.facets({ facets: ['a'], authorizationToken: 'abcd' });
expect(fakeCatalog.facets).toHaveBeenCalledWith({
facets: ['a'],
authorizationToken: 'abcd',
filter: { key: 'kind', values: ['b'] },
});
});
it('calls underlying catalog method on ALLOW', async () => {
fakePermissionApi.authorize.mockResolvedValue([
{ result: AuthorizeResult.ALLOW },
]);
const catalog = createCatalog();
await catalog.facets({ facets: ['a'], authorizationToken: 'abcd' });
expect(fakeCatalog.facets).toHaveBeenCalledWith({
facets: ['a'],
authorizationToken: 'abcd',
});
});
});
});
plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts
+31
View File
@@ -30,6 +30,8 @@ import {
EntitiesRequest,
EntitiesResponse,
EntityAncestryResponse,
EntityFacetsRequest,
EntityFacetsResponse,
EntityFilter,
} from '../catalog/types';
import { basicEntityFilter } from './request/basicEntityFilter';
@@ -151,6 +153,35 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog {
};
}
async facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse> {
const authorizeDecision = (
await this.permissionApi.authorize(
[{ permission: catalogEntityReadPermission }],
{ token: request?.authorizationToken },
)
)[0];
if (authorizeDecision.result === AuthorizeResult.DENY) {
return {
facets: Object.fromEntries(request.facets.map(f => [f, []])),
};
}
if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) {
const permissionFilter: EntityFilter = this.transformConditions(
authorizeDecision.conditions,
);
return this.entitiesCatalog.facets({
...request,
filter: request?.filter
? { allOf: [permissionFilter, request.filter] }
: permissionFilter,
});
}
return this.entitiesCatalog.facets(request);
}
private findParents(
entityRef: string,
allAncestryItems: { entity: Entity; parentEntityRefs: string[] }[],
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts
+125
View File
@@ -535,4 +535,129 @@ describe('DefaultEntitiesCatalog', () => {
},
);
});
describe('facets', () => {
it.each(databases.eachSupportedId())(
'can filter and collect properly',
async databaseId => {
const { knex } = await createDatabase(databaseId);
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'one' },
spec: {},
});
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k',
metadata: { name: 'two' },
spec: {},
});
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k2',
metadata: { name: 'two' },
spec: {},
});
const catalog = new DefaultEntitiesCatalog(knex);
await expect(catalog.facets({ facets: ['kind'] })).resolves.toEqual({
facets: {
kind: [
{ value: 'k', count: 2 },
{ value: 'k2', count: 1 },
],
},
});
},
);
it.each(databases.eachSupportedId())(
'can match on annotations and labels with dots in them',
async databaseId => {
const { knex } = await createDatabase(databaseId);
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'one',
annotations: { 'a.b/c.d': 'annotation1' },
labels: { 'e.f/g.h': 'label1' },
},
spec: {},
});
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'two',
annotations: { 'a.b/c.d': 'annotation2' },
labels: { 'e.f/g.h': 'label2' },
},
spec: {},
});
const catalog = new DefaultEntitiesCatalog(knex);
await expect(
catalog.facets({
facets: ['metadata.annotations.a.b/c.d', 'metadata.labels.e.f/g.h'],
}),
).resolves.toEqual({
facets: {
'metadata.annotations.a.b/c.d': [
{ value: 'annotation1', count: 1 },
{ value: 'annotation2', count: 1 },
],
'metadata.labels.e.f/g.h': [
{ value: 'label1', count: 1 },
{ value: 'label2', count: 1 },
],
},
});
},
);
it.each(databases.eachSupportedId())(
'can match on strings in arrays',
async databaseId => {
const { knex } = await createDatabase(databaseId);
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'one',
tags: ['java', 'rust'],
},
spec: {},
});
await addEntityToSearch(knex, {
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'two',
tags: ['java', 'node'],
},
spec: {},
});
const catalog = new DefaultEntitiesCatalog(knex);
await expect(
catalog.facets({
facets: ['metadata.tags'],
}),
).resolves.toEqual({
facets: {
'metadata.tags': expect.arrayContaining([
{ value: 'java', count: 2 },
{ value: 'rust', count: 1 },
{ value: 'node', count: 1 },
]),
},
});
},
);
});
});
plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts
+52 -4
View File
@@ -17,21 +17,24 @@
import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
import { InputError, NotFoundError } from '@backstage/errors';
import { Knex } from 'knex';
import lodash from 'lodash';
import {
EntitiesCatalog,
EntitiesRequest,
EntitiesResponse,
EntityAncestryResponse,
EntityPagination,
EntityFilter,
EntitiesSearchFilter,
EntityAncestryResponse,
EntityFacetsRequest,
EntityFacetsResponse,
EntityFilter,
EntityPagination,
} from '../catalog/types';
import {
DbFinalEntitiesRow,
DbPageInfo,
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbSearchRow,
DbPageInfo,
} from '../database/tables';
function parsePagination(input?: EntityPagination): {
@@ -295,4 +298,49 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog {
items,
};
}
async facets(request: EntityFacetsRequest): Promise<EntityFacetsResponse> {
const { entities } = await this.entities({
filter: request.filter,
authorizationToken: request.authorizationToken,
});
const facets: EntityFacetsResponse['facets'] = {};
for (const facet of request.facets) {
const values = entities
.map(entity => {
// TODO(freben): Generalize this code to handle any field that may
// have dots in its key?
if (facet.startsWith('metadata.annotations.')) {
return entity.metadata.annotations?.[
facet.substring('metadata.annotations.'.length)
];
} else if (facet.startsWith('metadata.labels.')) {
return entity.metadata.labels?.[
facet.substring('metadata.labels.'.length)
];
}
return lodash.get(entity, facet);
})
.flatMap(field => {
if (typeof field === 'string') {
return [field];
} else if (Array.isArray(field)) {
return field.filter(i => typeof i === 'string');
}
return [];
})
.sort();
const counts = lodash.countBy(values, lodash.identity);
facets[facet] = Object.entries(counts).map(([value, count]) => ({
value,
count,
}));
}
return { facets };
}
}
plugins/catalog-backend/src/service/createRouter.test.ts
+3
View File
@@ -40,6 +40,7 @@ describe('createRouter readonly disabled', () => {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
};
locationService = {
getLocation: jest.fn(),
@@ -394,6 +395,7 @@ describe('createRouter readonly enabled', () => {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
};
locationService = {
getLocation: jest.fn(),
@@ -578,6 +580,7 @@ describe('NextRouter permissioning', () => {
entities: jest.fn(),
removeEntityByUid: jest.fn(),
entityAncestry: jest.fn(),
facets: jest.fn(),
};
locationService = {
getLocation: jest.fn(),
plugins/catalog-backend/src/service/createRouter.ts
+10 -1
View File
@@ -37,6 +37,7 @@ import {
} from './util';
import { RefreshOptions, LocationService, RefreshService } from './types';
import { z } from 'zod';
import { parseEntityFacetParams } from './request/parseEntityFacetParams';
/**
* Options used by {@link createRouter}.
@@ -160,7 +161,15 @@ export async function createRouter(
const response = await entitiesCatalog.entityAncestry(entityRef);
res.status(200).json(response);
},
);
)
.get('/entity-facets', async (req, res) => {
const response = await entitiesCatalog.facets({
filter: parseEntityFilterParams(req.query),
facets: parseEntityFacetParams(req.query),
authorizationToken: getBearerToken(req.header('authorization')),
});
res.status(200).json(response);
});
}
if (locationService) {
plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts
+37
View File
@@ -0,0 +1,37 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError } from '@backstage/errors';
import { parseStringsParam } from './common';
/**
* Parses the facets part of a facet query, like
* /entity-facets?filter=metadata.namespace=default,kind=Component&facet=metadata.namespace
*/
export function parseEntityFacetParams(
params: Record<string, unknown>,
): string[] {
// Each facet string is on the form a.b.c
const facetStrings = parseStringsParam(params.facet, 'facet');
if (facetStrings) {
const filtered = facetStrings.filter(Boolean);
if (filtered.length) {
return filtered;
}
}
throw new InputError('Missing facet parameter');
}