diff --git a/.changeset/dry-ties-collect.md b/.changeset/dry-ties-collect.md new file mode 100644 index 0000000000..9ad3afb208 --- /dev/null +++ b/.changeset/dry-ties-collect.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Added an `/entity-facets` endpoint, which lets you query the distribution of +possible values for fields of entities. + +This can be useful for example when populating a dropdown in the user interface, +such as listing all tag values that are actually being used right now in your +catalog instance, along with putting the most common ones at the top. diff --git a/.changeset/lucky-bulldogs-own.md b/.changeset/lucky-bulldogs-own.md new file mode 100644 index 0000000000..4fb27d90b4 --- /dev/null +++ b/.changeset/lucky-bulldogs-own.md @@ -0,0 +1,15 @@ +--- +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-badges-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-todo-backend': patch +--- + +Updated according to the new `getEntityFacets` catalog API method diff --git a/.changeset/tiny-ads-knock.md b/.changeset/tiny-ads-knock.md new file mode 100644 index 0000000000..ba24ad6906 --- /dev/null +++ b/.changeset/tiny-ads-knock.md @@ -0,0 +1,8 @@ +--- +'@backstage/catalog-client': patch +--- + +Added `CatalogApi.getEntityFacets`. Marking this as a breaking change since it +is a non-optional addition to the API and depends on the backend being in place. +If you are mocking this interface in your tests, you will need to add an extra +`getEntityFacets: jest.fn()` or similar to that interface. diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index 56ba4a834f..d5991303eb 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -43,6 +43,10 @@ export interface CatalogApi { name: EntityName, options?: CatalogRequestOptions, ): Promise; + getEntityFacets( + request: GetEntityFacetsRequest, + options?: CatalogRequestOptions, + ): Promise; getLocationById( id: string, options?: CatalogRequestOptions, @@ -91,6 +95,10 @@ export class CatalogClient implements CatalogApi { compoundName: EntityName, options?: CatalogRequestOptions, ): Promise; + getEntityFacets( + request: GetEntityFacetsRequest, + options?: CatalogRequestOptions, + ): Promise; // @deprecated (undocumented) getLocationByEntity( entity: Entity, @@ -180,6 +188,26 @@ export interface GetEntityAncestorsResponse { rootEntityRef: string; } +// @public +export interface GetEntityFacetsRequest { + facets: string[]; + filter?: + | Record[] + | Record + | undefined; +} + +// @public +export interface GetEntityFacetsResponse { + facets: Record< + string, + Array<{ + value: string; + count: number; + }> + >; +} + // @public type Location_2 = { id: string; diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index 583f5a85bd..bb3c9604b3 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -36,6 +36,8 @@ import { GetEntityAncestorsRequest, GetEntityAncestorsResponse, Location, + GetEntityFacetsRequest, + GetEntityFacetsResponse, } from './types/api'; import { DiscoveryApi } from './types/discovery'; import { FetchApi } from './types/fetch'; @@ -97,14 +99,13 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise { const { filter = [], fields = [], offset, limit, after } = request ?? {}; - const filterItems = [filter].flat(); const params: string[] = []; // filter param can occur multiple times, for example // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters // the "inner array" defined within a `filter` param corresponds to "allOf" filters - for (const filterItem of filterItems) { + for (const filterItem of [filter].flat()) { const filterParts: string[] = []; for (const [key, value] of Object.entries(filterItem)) { for (const v of [value].flat()) { @@ -207,6 +208,47 @@ export class CatalogClient implements CatalogApi { } } + /** + * {@inheritdoc CatalogApi.getEntityFacets} + */ + async getEntityFacets( + request: GetEntityFacetsRequest, + options?: CatalogRequestOptions, + ): Promise { + const { filter = [], facets } = request; + const params: string[] = []; + + // filter param can occur multiple times, for example + // /api/catalog/entities?filter=metadata.name=wayback-search,kind=component&filter=metadata.name=www-artist,kind=component' + // the "outer array" defined by `filter` occurrences corresponds to "anyOf" filters + // the "inner array" defined within a `filter` param corresponds to "allOf" filters + for (const filterItem of [filter].flat()) { + const filterParts: string[] = []; + for (const [key, value] of Object.entries(filterItem)) { + for (const v of [value].flat()) { + if (v === CATALOG_FILTER_EXISTS) { + filterParts.push(encodeURIComponent(key)); + } else if (typeof v === 'string') { + filterParts.push( + `${encodeURIComponent(key)}=${encodeURIComponent(v)}`, + ); + } + } + } + + if (filterParts.length) { + params.push(`filter=${filterParts.join(',')}`); + } + } + + for (const facet of facets) { + params.push(`facet=${encodeURIComponent(facet)}`); + } + + const query = params.length ? `?${params.join('&')}` : ''; + return await this.requestOptional('GET', `/entity-facets${query}`, options); + } + /** * {@inheritdoc CatalogApi.addLocation} */ diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index b3d4bc2397..b070820c8b 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -142,6 +142,90 @@ export interface GetEntityAncestorsResponse { }>; } +/** + * The request type for {@link CatalogClient.getEntityFacets}. + * + * @public + */ +export interface GetEntityFacetsRequest { + /** + * If given, return only entities that match the given patterns. + * + * @remarks + * + * If multiple filter sets are given as an array, then there is effectively an + * OR between each filter set. + * + * Within one filter set, there is effectively an AND between the various + * keys. + * + * Within one key, if there are more than one value, then there is effectively + * an OR between them. + * + * Example: For an input of + * + * ``` + * [ + * { kind: ['API', 'Component'] }, + * { 'metadata.name': 'a', 'metadata.namespace': 'b' } + * ] + * ``` + * + * This effectively means + * + * ``` + * (kind = EITHER 'API' OR 'Component') + * OR + * (metadata.name = 'a' AND metadata.namespace = 'b' ) + * ``` + * + * Each key is a dot separated path in each object. + * + * As a value you can also pass in the symbol `CATALOG_FILTER_EXISTS` + * (exported from this package), which means that you assert on the existence + * of that key, no matter what its value is. + */ + filter?: + | Record[] + | Record + | undefined; + /** + * Dot separated paths for the facets to extract from each entity. + * + * @remarks + * + * Example: For an input of `['kind', 'metadata.annotations.backstage.io/orphan']`, then the + * response will be shaped like + * + * ``` + * { + * "facets": { + * "kind": [ + * { "key": "Component", "count": 22 }, + * { "key": "API", "count": 13 } + * ], + * "metadata.annotations.backstage.io/orphan": [ + * { "key": "true", "count": 2 } + * ] + * } + * } + * ``` + */ + facets: string[]; +} + +/** + * The response type for {@link CatalogClient.getEntityFacets}. + * + * @public + */ +export interface GetEntityFacetsResponse { + /** + * The computed facets, one entry per facet in the request. + */ + facets: Record>; +} + /** * Options you can pass into a catalog request for additional information. * @@ -248,6 +332,17 @@ export interface CatalogApi { options?: CatalogRequestOptions, ): Promise; + /** + * Gets a summary of field facets of entities in the catalog. + * + * @param request - Request parameters + * @param options - Additional options + */ + getEntityFacets( + request: GetEntityFacetsRequest, + options?: CatalogRequestOptions, + ): Promise; + // Locations /** diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 580bb7330d..39c8962b68 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -25,6 +25,8 @@ export type { GetEntityAncestorsRequest, GetEntityAncestorsResponse, Location, + GetEntityFacetsRequest, + GetEntityFacetsResponse, } from './api'; -export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; export * from './deprecated'; +export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts index 83ebb5de73..4388c2f82f 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts @@ -34,6 +34,7 @@ describe('CatalogIdentityClient', () => { removeEntityByUid: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; const tokenManager: jest.Mocked = { getToken: jest.fn(), diff --git a/plugins/badges-backend/src/service/router.test.ts b/plugins/badges-backend/src/service/router.test.ts index 4b0ae2a44a..56dbfb0655 100644 --- a/plugins/badges-backend/src/service/router.test.ts +++ b/plugins/badges-backend/src/service/router.test.ts @@ -67,6 +67,7 @@ describe('createRouter', () => { removeEntityByUid: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; config = new ConfigReader({ diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 0538554d6d..5ae56ac79c 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -559,6 +559,7 @@ export type EntitiesCatalog = { authorizationToken?: string; }, ): Promise; + facets(request: EntityFacetsRequest): Promise; }; // Warning: (ae-missing-release-tag) "EntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -604,6 +605,24 @@ export type EntityAncestryResponse = { }>; }; +// @public +export interface EntityFacetsRequest { + authorizationToken?: string; + facets: string[]; + filter?: EntityFilter; +} + +// @public +export interface EntityFacetsResponse { + facets: Record< + string, + Array<{ + value: string; + count: number; + }> + >; +} + // Warning: (ae-missing-release-tag) "EntityFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index 95c6572b10..59f67b2f52 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -18,9 +18,11 @@ export type { EntitiesCatalog, EntitiesRequest, EntitiesResponse, - EntityAncestryResponse, - PageInfo, EntitiesSearchFilter, + EntityAncestryResponse, + EntityFacetsRequest, + EntityFacetsResponse, EntityFilter, EntityPagination, + PageInfo, } from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index a79229b9a1..e285d4c100 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -87,6 +87,44 @@ export type EntityAncestryResponse = { }>; }; +/** + * The request shape for {@link EntitiesCatalog.facets}. + * + * @public + */ +export interface EntityFacetsRequest { + /** + * A filter to apply on the full list of entities before computing the facets. + */ + filter?: EntityFilter; + /** + * The facets to compute. + * + * @remarks + * + * This is a list of strings corresponding to paths within individual entity + * shapes. For example, to compute the facets for all available tags, you + * would pass in the string 'metadata.tags'. + */ + facets: string[]; + /** + * The optional token that authorizes the action. + */ + authorizationToken?: string; +} + +/** + * The response shape for {@link EntitiesCatalog.facets}. + * + * @public + */ +export interface EntityFacetsResponse { + /** + * The computed facets, one entry per facet in the request. + */ + facets: Record>; +} + /** @public */ export type EntitiesCatalog = { /** @@ -115,4 +153,12 @@ export type EntitiesCatalog = { entityRef: string, options?: { authorizationToken?: string }, ): Promise; + + /** + * Computes facets for a set of entities, e.g. for populating filter lists + * or driving insights or similar. + * + * @param request - Request options + */ + facets(request: EntityFacetsRequest): Promise; }; diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts index 156db8e8ce..1f0e5ea644 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.test.ts @@ -30,6 +30,7 @@ describe('AuthorizedEntitiesCatalog', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; const fakePermissionApi = { authorize: jest.fn(), @@ -254,4 +255,54 @@ describe('AuthorizedEntitiesCatalog', () => { }); }); }); + + describe('facets', () => { + it('returns empty response on DENY', async () => { + fakePermissionApi.authorize.mockResolvedValue([ + { result: AuthorizeResult.DENY }, + ]); + const catalog = createCatalog(); + + expect( + await catalog.facets({ + facets: ['a'], + authorizationToken: 'abcd', + }), + ).toEqual({ + facets: { a: [] }, + }); + }); + + it('calls underlying catalog method with correct filter on CONDITIONAL', async () => { + fakePermissionApi.authorize.mockResolvedValue([ + { + result: AuthorizeResult.CONDITIONAL, + conditions: { rule: 'IS_ENTITY_KIND', params: [['b']] }, + }, + ]); + const catalog = createCatalog(isEntityKind); + + await catalog.facets({ facets: ['a'], authorizationToken: 'abcd' }); + + expect(fakeCatalog.facets).toHaveBeenCalledWith({ + facets: ['a'], + authorizationToken: 'abcd', + filter: { key: 'kind', values: ['b'] }, + }); + }); + + it('calls underlying catalog method on ALLOW', async () => { + fakePermissionApi.authorize.mockResolvedValue([ + { result: AuthorizeResult.ALLOW }, + ]); + const catalog = createCatalog(); + + await catalog.facets({ facets: ['a'], authorizationToken: 'abcd' }); + + expect(fakeCatalog.facets).toHaveBeenCalledWith({ + facets: ['a'], + authorizationToken: 'abcd', + }); + }); + }); }); diff --git a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts index 99fc3ebc8e..2f94e23303 100644 --- a/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/AuthorizedEntitiesCatalog.ts @@ -30,6 +30,8 @@ import { EntitiesRequest, EntitiesResponse, EntityAncestryResponse, + EntityFacetsRequest, + EntityFacetsResponse, EntityFilter, } from '../catalog/types'; import { basicEntityFilter } from './request/basicEntityFilter'; @@ -151,6 +153,35 @@ export class AuthorizedEntitiesCatalog implements EntitiesCatalog { }; } + async facets(request: EntityFacetsRequest): Promise { + const authorizeDecision = ( + await this.permissionApi.authorize( + [{ permission: catalogEntityReadPermission }], + { token: request?.authorizationToken }, + ) + )[0]; + + if (authorizeDecision.result === AuthorizeResult.DENY) { + return { + facets: Object.fromEntries(request.facets.map(f => [f, []])), + }; + } + + if (authorizeDecision.result === AuthorizeResult.CONDITIONAL) { + const permissionFilter: EntityFilter = this.transformConditions( + authorizeDecision.conditions, + ); + return this.entitiesCatalog.facets({ + ...request, + filter: request?.filter + ? { allOf: [permissionFilter, request.filter] } + : permissionFilter, + }); + } + + return this.entitiesCatalog.facets(request); + } + private findParents( entityRef: string, allAncestryItems: { entity: Entity; parentEntityRefs: string[] }[], diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts index d6d4e957f6..7094cfd1c2 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.test.ts @@ -535,4 +535,129 @@ describe('DefaultEntitiesCatalog', () => { }, ); }); + + describe('facets', () => { + it.each(databases.eachSupportedId())( + 'can filter and collect properly', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'one' }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'two' }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k2', + metadata: { name: 'two' }, + spec: {}, + }); + const catalog = new DefaultEntitiesCatalog(knex); + + await expect(catalog.facets({ facets: ['kind'] })).resolves.toEqual({ + facets: { + kind: [ + { value: 'k', count: 2 }, + { value: 'k2', count: 1 }, + ], + }, + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'can match on annotations and labels with dots in them', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'one', + annotations: { 'a.b/c.d': 'annotation1' }, + labels: { 'e.f/g.h': 'label1' }, + }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'two', + annotations: { 'a.b/c.d': 'annotation2' }, + labels: { 'e.f/g.h': 'label2' }, + }, + spec: {}, + }); + const catalog = new DefaultEntitiesCatalog(knex); + + await expect( + catalog.facets({ + facets: ['metadata.annotations.a.b/c.d', 'metadata.labels.e.f/g.h'], + }), + ).resolves.toEqual({ + facets: { + 'metadata.annotations.a.b/c.d': [ + { value: 'annotation1', count: 1 }, + { value: 'annotation2', count: 1 }, + ], + 'metadata.labels.e.f/g.h': [ + { value: 'label1', count: 1 }, + { value: 'label2', count: 1 }, + ], + }, + }); + }, + ); + + it.each(databases.eachSupportedId())( + 'can match on strings in arrays', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'one', + tags: ['java', 'rust'], + }, + spec: {}, + }); + await addEntityToSearch(knex, { + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'two', + tags: ['java', 'node'], + }, + spec: {}, + }); + const catalog = new DefaultEntitiesCatalog(knex); + + await expect( + catalog.facets({ + facets: ['metadata.tags'], + }), + ).resolves.toEqual({ + facets: { + 'metadata.tags': expect.arrayContaining([ + { value: 'java', count: 2 }, + { value: 'rust', count: 1 }, + { value: 'node', count: 1 }, + ]), + }, + }); + }, + ); + }); }); diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 6979f94ba7..1ac70bda30 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -17,21 +17,24 @@ import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; +import lodash from 'lodash'; import { EntitiesCatalog, EntitiesRequest, EntitiesResponse, - EntityAncestryResponse, - EntityPagination, - EntityFilter, EntitiesSearchFilter, + EntityAncestryResponse, + EntityFacetsRequest, + EntityFacetsResponse, + EntityFilter, + EntityPagination, } from '../catalog/types'; import { DbFinalEntitiesRow, + DbPageInfo, DbRefreshStateReferencesRow, DbRefreshStateRow, DbSearchRow, - DbPageInfo, } from '../database/tables'; function parsePagination(input?: EntityPagination): { @@ -295,4 +298,49 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { items, }; } + + async facets(request: EntityFacetsRequest): Promise { + const { entities } = await this.entities({ + filter: request.filter, + authorizationToken: request.authorizationToken, + }); + + const facets: EntityFacetsResponse['facets'] = {}; + + for (const facet of request.facets) { + const values = entities + .map(entity => { + // TODO(freben): Generalize this code to handle any field that may + // have dots in its key? + if (facet.startsWith('metadata.annotations.')) { + return entity.metadata.annotations?.[ + facet.substring('metadata.annotations.'.length) + ]; + } else if (facet.startsWith('metadata.labels.')) { + return entity.metadata.labels?.[ + facet.substring('metadata.labels.'.length) + ]; + } + return lodash.get(entity, facet); + }) + .flatMap(field => { + if (typeof field === 'string') { + return [field]; + } else if (Array.isArray(field)) { + return field.filter(i => typeof i === 'string'); + } + return []; + }) + .sort(); + + const counts = lodash.countBy(values, lodash.identity); + + facets[facet] = Object.entries(counts).map(([value, count]) => ({ + value, + count, + })); + } + + return { facets }; + } } diff --git a/plugins/catalog-backend/src/service/createRouter.test.ts b/plugins/catalog-backend/src/service/createRouter.test.ts index 6ae24ecd27..755134c3d9 100644 --- a/plugins/catalog-backend/src/service/createRouter.test.ts +++ b/plugins/catalog-backend/src/service/createRouter.test.ts @@ -40,6 +40,7 @@ describe('createRouter readonly disabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -394,6 +395,7 @@ describe('createRouter readonly enabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -578,6 +580,7 @@ describe('NextRouter permissioning', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), entityAncestry: jest.fn(), + facets: jest.fn(), }; locationService = { getLocation: jest.fn(), diff --git a/plugins/catalog-backend/src/service/createRouter.ts b/plugins/catalog-backend/src/service/createRouter.ts index 0dce765507..a4d911a421 100644 --- a/plugins/catalog-backend/src/service/createRouter.ts +++ b/plugins/catalog-backend/src/service/createRouter.ts @@ -37,6 +37,7 @@ import { } from './util'; import { RefreshOptions, LocationService, RefreshService } from './types'; import { z } from 'zod'; +import { parseEntityFacetParams } from './request/parseEntityFacetParams'; /** * Options used by {@link createRouter}. @@ -160,7 +161,15 @@ export async function createRouter( const response = await entitiesCatalog.entityAncestry(entityRef); res.status(200).json(response); }, - ); + ) + .get('/entity-facets', async (req, res) => { + const response = await entitiesCatalog.facets({ + filter: parseEntityFilterParams(req.query), + facets: parseEntityFacetParams(req.query), + authorizationToken: getBearerToken(req.header('authorization')), + }); + res.status(200).json(response); + }); } if (locationService) { diff --git a/plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts b/plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts new file mode 100644 index 0000000000..00493f068f --- /dev/null +++ b/plugins/catalog-backend/src/service/request/parseEntityFacetParams.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/errors'; +import { parseStringsParam } from './common'; + +/** + * Parses the facets part of a facet query, like + * /entity-facets?filter=metadata.namespace=default,kind=Component&facet=metadata.namespace + */ +export function parseEntityFacetParams( + params: Record, +): string[] { + // Each facet string is on the form a.b.c + const facetStrings = parseStringsParam(params.facet, 'facet'); + if (facetStrings) { + const filtered = facetStrings.filter(Boolean); + if (filtered.length) { + return filtered; + } + } + + throw new InputError('Missing facet parameter'); +} diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx index 0e2fa723dc..7cebdee692 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx @@ -65,6 +65,7 @@ describe('', () => { removeLocationById: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; apis = TestApiRegistry.from([catalogApiRef, catalog]); diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx index c509abd712..f98aca0081 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx @@ -94,6 +94,7 @@ describe('', () => { removeLocationById: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; wrapper = ( diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx index 715f30e8c9..f16d06f809 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx @@ -155,6 +155,7 @@ describe('', () => { removeLocationById: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; Wrapper = ({ children }) => ( diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts index 214f1a8fb9..a5dc82eab9 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts @@ -37,6 +37,7 @@ describe('useEntityStore', () => { removeLocationById: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; useApi.mockReturnValue(catalogApi); diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts index d2c7771804..f689c0a883 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.test.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -99,6 +99,7 @@ describe('CatalogImportClient', () => { removeEntityByUid: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; let catalogImportClient: CatalogImportClient; diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx index 40c1152fe8..fc9b35136a 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx @@ -45,6 +45,7 @@ describe('', () => { removeEntityByUid: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; const errorApi: jest.Mocked = { diff --git a/plugins/catalog-react/src/components/EntityTypePicker/EntityTypePicker.test.tsx b/plugins/catalog-react/src/components/EntityTypePicker/EntityTypePicker.test.tsx index 9ed27bbef8..4623cf5d9e 100644 --- a/plugins/catalog-react/src/components/EntityTypePicker/EntityTypePicker.test.tsx +++ b/plugins/catalog-react/src/components/EntityTypePicker/EntityTypePicker.test.tsx @@ -26,6 +26,7 @@ import { EntityKindFilter, EntityTypeFilter } from '../../filters'; import { alertApiRef } from '@backstage/core-plugin-api'; import { ApiProvider } from '@backstage/core-app-api'; import { renderWithEffects, TestApiRegistry } from '@backstage/test-utils'; +import { GetEntityFacetsResponse } from '@backstage/catalog-client'; const entities: Entity[] = [ { @@ -64,7 +65,14 @@ const apis = TestApiRegistry.from( [ catalogApiRef, { - getEntities: jest.fn().mockResolvedValue({ items: entities }), + getEntityFacets: jest.fn().mockResolvedValue({ + facets: { + 'spec.type': entities.map(e => ({ + value: (e.spec as any).type, + count: 1, + })), + }, + } as GetEntityFacetsResponse), }, ], [ diff --git a/plugins/catalog-react/src/hooks/useEntityKinds.test.tsx b/plugins/catalog-react/src/hooks/useEntityKinds.test.tsx index cdcac8c194..c96301b3c4 100644 --- a/plugins/catalog-react/src/hooks/useEntityKinds.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityKinds.test.tsx @@ -16,45 +16,21 @@ import React, { PropsWithChildren } from 'react'; import { CatalogApi } from '@backstage/catalog-client'; -import { Entity } from '@backstage/catalog-model'; import { catalogApiRef } from '../api'; import { renderHook } from '@testing-library/react-hooks'; import { useEntityKinds } from './useEntityKinds'; import { TestApiProvider } from '@backstage/test-utils'; -const entities: Entity[] = [ - { - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'component-1', - }, - }, - { - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'component-2', - }, - }, - { - apiVersion: '1', - kind: 'Template', - metadata: { - name: 'template', - }, - }, - { - apiVersion: '1', - kind: 'System', - metadata: { - name: 'system', - }, - }, -]; - const mockCatalogApi: Partial = { - getEntities: jest.fn().mockResolvedValue({ items: entities }), + getEntityFacets: jest.fn().mockResolvedValue({ + facets: { + kind: [ + { value: 'Template', count: 2 }, + { value: 'System', count: 1 }, + { value: 'Component', count: 3 }, + ], + }, + }), }; const wrapper = ({ children }: PropsWithChildren<{}>) => { @@ -66,24 +42,10 @@ const wrapper = ({ children }: PropsWithChildren<{}>) => { }; describe('useEntityKinds', () => { - it('does not return duplicate kinds', async () => { + it('gets entity kinds', async () => { const { result, waitForValueToChange } = renderHook( () => useEntityKinds(), - { - wrapper, - }, - ); - await waitForValueToChange(() => result.current); - expect(result.current.kinds).toBeDefined(); - expect(result.current.kinds!.length).toBe(3); - }); - - it('sorts entity kinds', async () => { - const { result, waitForValueToChange } = renderHook( - () => useEntityKinds(), - { - wrapper, - }, + { wrapper }, ); await waitForValueToChange(() => result.current); expect(result.current.kinds).toEqual(['Component', 'System', 'Template']); diff --git a/plugins/catalog-react/src/hooks/useEntityKinds.ts b/plugins/catalog-react/src/hooks/useEntityKinds.ts index 7bc0dd47ae..5fad49df8e 100644 --- a/plugins/catalog-react/src/hooks/useEntityKinds.ts +++ b/plugins/catalog-react/src/hooks/useEntityKinds.ts @@ -30,11 +30,9 @@ export function useEntityKinds() { loading, value: kinds, } = useAsync(async () => { - const entities = await catalogApi - .getEntities({ fields: ['kind'] }) - .then(response => response.items); - - return [...new Set(entities.map(e => e.kind))].sort(); + return await catalogApi + .getEntityFacets({ facets: ['kind'] }) + .then(response => response.facets.kind?.map(f => f.value).sort() || []); }); return { error, loading, kinds }; } diff --git a/plugins/catalog-react/src/hooks/useEntityTypeFilter.tsx b/plugins/catalog-react/src/hooks/useEntityTypeFilter.tsx index 8e5be53950..cfe821b4f7 100644 --- a/plugins/catalog-react/src/hooks/useEntityTypeFilter.tsx +++ b/plugins/catalog-react/src/hooks/useEntityTypeFilter.tsx @@ -17,6 +17,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import useAsync from 'react-use/lib/useAsync'; import isEqual from 'lodash/isEqual'; +import sortBy from 'lodash/sortBy'; import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef } from '../api'; import { useEntityListProvider } from './useEntityListProvider'; @@ -69,51 +70,34 @@ export function useEntityTypeFilter(): EntityTypeReturn { const { error, loading, - value: entities, + value: facets, } = useAsync(async () => { if (kind) { const items = await catalogApi - .getEntities({ + .getEntityFacets({ filter: { kind }, - fields: ['spec.type'], + facets: ['spec.type'], }) - .then(response => response.items); + .then(response => response.facets['spec.type'] || []); return items; } return []; }, [kind, catalogApi]); - const entitiesRef = useRef(entities); + const facetsRef = useRef(facets); useEffect(() => { - const oldEntities = entitiesRef.current; - entitiesRef.current = entities; - // Delay processing hook until kind and entity load updates have settled to generate list of types; - // This prevents reseting the type filter due to saved type value from query params not matching the + const oldFacets = facetsRef.current; + facetsRef.current = facets; + // Delay processing hook until kind and facets load updates have settled to generate list of types; + // This prevents resetting the type filter due to saved type value from query params not matching the // empty set of type values while values are still being loaded; also only run this hook on changes - // to entities - if (loading || !kind || oldEntities === entities) { + // to facets + if (loading || !kind || oldFacets === facets || !facets) { return; } - // Resolve the unique set of types from returned entities; could be optimized by a new endpoint - // in the catalog-backend that does this, rather than loading entities with redundant types. - if (!entities) return; - - // Sort by entity count descending, so the most common types appear on top - const countByType = entities.reduce((acc, entity) => { - if (typeof entity.spec?.type !== 'string') return acc; - - const entityType = entity.spec.type.toLocaleLowerCase('en-US'); - if (!acc[entityType]) { - acc[entityType] = 0; - } - acc[entityType] += 1; - return acc; - }, {} as Record); - - const newTypes = Object.entries(countByType) - .sort(([, count1], [, count2]) => count2 - count1) - .map(([type]) => type); + // Sort by facet count descending, so the most common types appear on top + const newTypes = sortBy(facets, f => -f.count).map(f => f.value); setAvailableTypes(newTypes); // Update type filter to only valid values when the list of available types has changed @@ -123,7 +107,7 @@ export function useEntityTypeFilter(): EntityTypeReturn { if (!isEqual(selectedTypes, stillValidTypes)) { setSelectedTypes(stillValidTypes); } - }, [loading, kind, selectedTypes, setSelectedTypes, entities]); + }, [loading, kind, selectedTypes, setSelectedTypes, facets]); useEffect(() => { updateFilters({ diff --git a/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.test.tsx b/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.test.tsx index b320d6d220..795d57df47 100644 --- a/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.test.tsx +++ b/plugins/catalog/src/components/CatalogKindHeader/CatalogKindHeader.test.tsx @@ -16,6 +16,7 @@ import React from 'react'; import { fireEvent } from '@testing-library/react'; +import { GetEntityFacetsResponse } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import { catalogApiRef, @@ -60,7 +61,15 @@ const entities: Entity[] = [ const apis = TestApiRegistry.from([ catalogApiRef, { - getEntities: jest.fn().mockResolvedValue({ items: entities }), + getEntityFacets: jest.fn().mockResolvedValue({ + facets: { + kind: [ + { value: 'Component', count: 2 }, + { value: 'Template', count: 1 }, + { value: 'System', count: 1 }, + ], + }, + } as GetEntityFacetsResponse), }, ]); diff --git a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx index db59e28f40..53b225daaf 100644 --- a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx +++ b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx @@ -31,6 +31,7 @@ describe('', () => { getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; const Wrapper = ({ children }: { children?: React.ReactNode }) => ( diff --git a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx index c431bd0c6c..1310d9f31e 100644 --- a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx +++ b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx @@ -32,6 +32,7 @@ describe('', () => { getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; const Wrapper = ({ children }: { children?: React.ReactNode }) => ( diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx index 23ae2a3e6d..255db061aa 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx @@ -32,6 +32,7 @@ describe('', () => { getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; const Wrapper = ({ children }: { children?: React.ReactNode }) => ( diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx index df376895bc..2759afd585 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx @@ -36,6 +36,7 @@ describe('', () => { removeLocationById: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; const fossaApi: jest.Mocked = { getFindingSummary: jest.fn(), diff --git a/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.test.tsx b/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.test.tsx index c70f251eac..aa6630d4aa 100644 --- a/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.test.tsx +++ b/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.test.tsx @@ -17,7 +17,6 @@ import React from 'react'; import { fireEvent } from '@testing-library/react'; import { capitalize } from 'lodash'; -import { CatalogApi } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import { TemplateTypePicker } from './TemplateTypePicker'; import { @@ -28,6 +27,7 @@ import { import { AlertApi, alertApiRef } from '@backstage/core-plugin-api'; import { ApiProvider } from '@backstage/core-app-api'; import { renderWithEffects, TestApiRegistry } from '@backstage/test-utils'; +import { GetEntityFacetsResponse } from '@backstage/catalog-client'; const entities: Entity[] = [ { @@ -66,10 +66,15 @@ const apis = TestApiRegistry.from( [ catalogApiRef, { - getEntities: jest - .fn() - .mockImplementation(() => Promise.resolve({ items: entities })), - } as unknown as CatalogApi, + getEntityFacets: jest.fn().mockResolvedValue({ + facets: { + 'spec.type': entities.map(e => ({ + value: (e.spec as any).type, + count: 1, + })), + }, + } as GetEntityFacetsResponse), + }, ], [ alertApiRef, diff --git a/plugins/todo-backend/src/service/TodoReaderService.test.ts b/plugins/todo-backend/src/service/TodoReaderService.test.ts index fbb1341e73..985687c6bb 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.test.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.test.ts @@ -51,6 +51,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { removeEntityByUid: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), + getEntityFacets: jest.fn(), }; if (entity) { mock.getEntityByName.mockReturnValue(entity);