diff --git a/.changeset/dry-ties-collect.md b/.changeset/dry-ties-collect.md new file mode 100644 index 0000000000..9ad3afb208 --- /dev/null +++ b/.changeset/dry-ties-collect.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Added an `/entity-facets` endpoint, which lets you query the distribution of +possible values for fields of entities. + +This can be useful for example when populating a dropdown in the user interface, +such as listing all tag values that are actually being used right now in your +catalog instance, along with putting the most common ones at the top. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index d97df9967c..21cf1c524f 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -600,6 +600,7 @@ export type EntitiesCatalog = { authorizationToken?: string; }, ): Promise; + facets(request: EntityFacetsRequest): Promise; }; // Warning: (ae-missing-release-tag) "EntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -645,6 +646,24 @@ export type EntityAncestryResponse = { }>; }; +// @public +export interface EntityFacetsRequest { + authorizationToken?: string; + facets: string[]; + filter?: EntityFilter; +} + +// @public +export interface EntityFacetsResponse { + facets: Record< + string, + Array<{ + value: string; + count: number; + }> + >; +} + // Warning: (ae-missing-release-tag) "EntityFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index 95c6572b10..59f67b2f52 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -18,9 +18,11 @@ export type { EntitiesCatalog, EntitiesRequest, EntitiesResponse, - EntityAncestryResponse, - PageInfo, EntitiesSearchFilter, + EntityAncestryResponse, + EntityFacetsRequest, + EntityFacetsResponse, EntityFilter, EntityPagination, + PageInfo, } from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index a79229b9a1..e285d4c100 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -87,6 +87,44 @@ export type EntityAncestryResponse = { }>; }; +/** + * The request shape for {@link EntitiesCatalog.facets}. + * + * @public + */ +export interface EntityFacetsRequest { + /** + * A filter to apply on the full list of entities before computing the facets. + */ + filter?: EntityFilter; + /** + * The facets to compute. + * + * @remarks + * + * This is a list of strings corresponding to paths within individual entity + * shapes. For example, to compute the facets for all available tags, you + * would pass in the string 'metadata.tags'. + */ + facets: string[]; + /** + * The optional token that authorizes the action. + */ + authorizationToken?: string; +} + +/** + * The response shape for {@link EntitiesCatalog.facets}. + * + * @public + */ +export interface EntityFacetsResponse { + /** + * The computed facets, one entry per facet in the request. + */ + facets: Record>; +} + /** @public */ export type EntitiesCatalog = { /** @@ -115,4 +153,12 @@ export type EntitiesCatalog = { entityRef: string, options?: { authorizationToken?: string }, ): Promise; + + /** + * Computes facets for a set of entities, e.g. for populating filter lists + * or driving insights or similar. + * + * @param request - Request options + */ + facets(request: EntityFacetsRequest): Promise; }; diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 156db8e8ce..1f0e5ea644 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -30,6 +30,7 @@ describe('AuthorizedEntitiesCatalog', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; const fakePermissionApi = { authorize: jest.fn(), @@ -254,4 +255,54 @@ describe('AuthorizedEntitiesCatalog', () => { }); }); }); + + describe('facets', () => { + it('returns empty response on DENY', async () => { + fakePermissionApi.authorize.mockResolvedValue([ + { result: AuthorizeResult.DENY }, + ]); + const catalog = createCatalog(); + + expect( + await catalog.facets({ + facets: ['a'], + authorizationToken: 'abcd', + }), + ).toEqual({ + facets: { a: [] }, + }); + }); + + it('calls underlying catalog method with correct filter on CONDITIONAL', async () => { + fakePermissionApi.authorize.mockResolvedValue([ + { + result: AuthorizeResult.CONDITIONAL, + conditions: { rule: 'IS_ENTITY_KIND', params: [['b']] }, + }, + ]); + const catalog = createCatalog(isEntityKind); + + await catalog.facets({ facets: ['a'], authorizationToken: 'abcd' }); + + expect(fakeCatalog.facets).toHaveBeenCalledWith({ + facets: ['a'], + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + }); + + it('calls underlying catalog method on ALLOW', async () => { + fakePermissionApi.authorize.mockResolvedValue([ + { result: AuthorizeResult.ALLOW }, + ]); + const catalog = createCatalog(); + + await catalog.facets({ facets: ['a'], authorizationToken: 'abcd' }); + + expect(fakeCatalog.facets).toHaveBeenCalledWith({ + facets: ['a'], + authorizationToken: 'abcd', + }); + }); + }); }); diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index 99fc3ebc8e..2f94e23303 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -30,6 +30,8 @@ import { EntitiesRequest, EntitiesResponse, EntityAncestryResponse, + EntityFacetsRequest, + EntityFacetsResponse, EntityFilter, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; @@ -151,6 +153,35 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { }; } + async facets(request: EntityFacetsRequest): Promise { + const authorizeDecision = ( + await this.permissionApi.authorize( + [{ permission: catalogEntityReadPermission }], + { token: request?.authorizationToken }, + ) + )[0]; + + if (authorizeDecision.result === AuthorizeResult.DENY) { + return { + facets: Object.fromEntries(request.facets.map(f => [f, []])), + }; + } + + if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) { + const permissionFilter: EntityFilter = this.transformConditions( + authorizeDecision.conditions, + ); + return this.entitiesCatalog.facets({ + ...request, + filter: request?.filter + ? { allOf: [permissionFilter, request.filter] } + : permissionFilter, + }); + } + + return this.entitiesCatalog.facets(request); + } + private findParents( entityRef: string, allAncestryItems: { entity: Entity; parentEntityRefs: string[] }[], diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index d6d4e957f6..7094cfd1c2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -535,4 +535,129 @@ describe('DefaultEntitiesCatalog', () => { }, ); }); + + describe('facets', () => { + it.each(databases.eachSupportedId())( + 'can filter and collect properly', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'one' }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'two' }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k2', + metadata: { name: 'two' }, + spec: {}, + }); + const catalog = new DefaultEntitiesCatalog(knex); + + await expect(catalog.facets({ facets: ['kind'] })).resolves.toEqual({ + facets: { + kind: [ + { value: 'k', count: 2 }, + { value: 'k2', count: 1 }, + ], + }, + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'can match on annotations and labels with dots in them', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'one', + annotations: { 'a.b/c.d': 'annotation1' }, + labels: { 'e.f/g.h': 'label1' }, + }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'two', + annotations: { 'a.b/c.d': 'annotation2' }, + labels: { 'e.f/g.h': 'label2' }, + }, + spec: {}, + }); + const catalog = new DefaultEntitiesCatalog(knex); + + await expect( + catalog.facets({ + facets: ['metadata.annotations.a.b/c.d', 'metadata.labels.e.f/g.h'], + }), + ).resolves.toEqual({ + facets: { + 'metadata.annotations.a.b/c.d': [ + { value: 'annotation1', count: 1 }, + { value: 'annotation2', count: 1 }, + ], + 'metadata.labels.e.f/g.h': [ + { value: 'label1', count: 1 }, + { value: 'label2', count: 1 }, + ], + }, + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'can match on strings in arrays', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'one', + tags: ['java', 'rust'], + }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'two', + tags: ['java', 'node'], + }, + spec: {}, + }); + const catalog = new DefaultEntitiesCatalog(knex); + + await expect( + catalog.facets({ + facets: ['metadata.tags'], + }), + ).resolves.toEqual({ + facets: { + 'metadata.tags': expect.arrayContaining([ + { value: 'java', count: 2 }, + { value: 'rust', count: 1 }, + { value: 'node', count: 1 }, + ]), + }, + }); + }, + ); + }); }); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 6979f94ba7..1ac70bda30 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -17,21 +17,24 @@ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; +import lodash from 'lodash'; import { EntitiesCatalog, EntitiesRequest, EntitiesResponse, - EntityAncestryResponse, - EntityPagination, - EntityFilter, EntitiesSearchFilter, + EntityAncestryResponse, + EntityFacetsRequest, + EntityFacetsResponse, + EntityFilter, + EntityPagination, } from '../catalog/types'; import { DbFinalEntitiesRow, + DbPageInfo, DbRefreshStateReferencesRow, DbRefreshStateRow, DbSearchRow, - DbPageInfo, } from '../database/tables'; function parsePagination(input?: EntityPagination): { @@ -295,4 +298,49 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { items, }; } + + async facets(request: EntityFacetsRequest): Promise { + const { entities } = await this.entities({ + filter: request.filter, + authorizationToken: request.authorizationToken, + }); + + const facets: EntityFacetsResponse['facets'] = {}; + + for (const facet of request.facets) { + const values = entities + .map(entity => { + // TODO(freben): Generalize this code to handle any field that may + // have dots in its key? + if (facet.startsWith('metadata.annotations.')) { + return entity.metadata.annotations?.[ + facet.substring('metadata.annotations.'.length) + ]; + } else if (facet.startsWith('metadata.labels.')) { + return entity.metadata.labels?.[ + facet.substring('metadata.labels.'.length) + ]; + } + return lodash.get(entity, facet); + }) + .flatMap(field => { + if (typeof field === 'string') { + return [field]; + } else if (Array.isArray(field)) { + return field.filter(i => typeof i === 'string'); + } + return []; + }) + .sort(); + + const counts = lodash.countBy(values, lodash.identity); + + facets[facet] = Object.entries(counts).map(([value, count]) => ({ + value, + count, + })); + } + + return { facets }; + } } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 6ae24ecd27..755134c3d9 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -40,6 +40,7 @@ describe('createRouter readonly disabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -394,6 +395,7 @@ describe('createRouter readonly enabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -578,6 +580,7 @@ describe('NextRouter permissioning', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; locationService = { getLocation: jest.fn(), diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 0dce765507..a4d911a421 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -37,6 +37,7 @@ import { } from './util'; import { RefreshOptions, LocationService, RefreshService } from './types'; import { z } from 'zod'; +import { parseEntityFacetParams } from './request/parseEntityFacetParams'; /** * Options used by {@link createRouter}. @@ -160,7 +161,15 @@ export async function createRouter( const response = await entitiesCatalog.entityAncestry(entityRef); res.status(200).json(response); }, - ); + ) + .get('/entity-facets', async (req, res) => { + const response = await entitiesCatalog.facets({ + filter: parseEntityFilterParams(req.query), + facets: parseEntityFacetParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); + res.status(200).json(response); + }); } if (locationService) { diff --git a/plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts b/plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts new file mode 100644 index 0000000000..00493f068f --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { parseStringsParam } from './common'; + +/** + * Parses the facets part of a facet query, like + * /entity-facets?filter=metadata.namespace=default,kind=Component&facet=metadata.namespace + */ +export function parseEntityFacetParams( + params: Record, +): string[] { + // Each facet string is on the form a.b.c + const facetStrings = parseStringsParam(params.facet, 'facet'); + if (facetStrings) { + const filtered = facetStrings.filter(Boolean); + if (filtered.length) { + return filtered; + } + } + + throw new InputError('Missing facet parameter'); +}