diff --git a/.changeset/tidy-houses-jam.md b/.changeset/tidy-houses-jam.md new file mode 100644 index 0000000000..71a165912c --- /dev/null +++ b/.changeset/tidy-houses-jam.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Add `/entities/by-name/:kind/:namespace/:name/ancestry` to get the "processing parents" lineage of an entity. + +This involves a breaking change of adding the method `entityAncestry` to `EntitiesCatalog`. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 133deb96fc..fb27a2cc2a 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -588,6 +588,8 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { // (undocumented) entities(request?: EntitiesRequest): Promise; // (undocumented) + entityAncestry(): Promise; + // (undocumented) removeEntityByUid(uid: string): Promise; } @@ -805,8 +807,6 @@ export type DeferredEntity = { // @public export function durationText(startTimestamp: [number, number]): string; -// Warning: (ae-missing-release-tag) "EntitiesCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public (undocumented) export type EntitiesCatalog = { entities(request?: EntitiesRequest): Promise; @@ -819,6 +819,7 @@ export type EntitiesCatalog = { outputEntities?: boolean; }, ): Promise; + entityAncestry(entityRef: string): Promise; }; // Warning: (ae-missing-release-tag) "EntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -855,6 +856,15 @@ function entity( newEntity: Entity, ): CatalogProcessorResult; +// @public (undocumented) +export type EntityAncestryResponse = { + rootEntityRef: string; + items: Array<{ + entity: Entity; + parentEntityRefs: string[]; + }>; +}; + // Warning: (ae-missing-release-tag) "EntityFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public @@ -1501,12 +1511,9 @@ export class UrlReaderProcessor implements CatalogProcessor { // Warnings were encountered during analysis: // -// src/catalog/types.d.ts:30:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/catalog/types.d.ts:36:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/catalog/types.d.ts:42:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen -// src/catalog/types.d.ts:43:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// src/catalog/types.d.ts:44:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters -// src/catalog/types.d.ts:45:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// src/catalog/types.d.ts:52:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// src/catalog/types.d.ts:53:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters +// src/catalog/types.d.ts:54:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters // src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 69b8b500d0..0f9fe872a2 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -369,4 +369,8 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { return response.entity; } + + async entityAncestry(): Promise { + throw new Error('Not implemented'); + } } diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts index 04cc358568..6c20237a37 100644 --- a/plugins/catalog-backend/src/catalog/index.ts +++ b/plugins/catalog-backend/src/catalog/index.ts @@ -18,13 +18,14 @@ export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog'; export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog'; export type { EntitiesCatalog, - LocationsCatalog, - EntityUpsertRequest, - EntityUpsertResponse, EntitiesRequest, EntitiesResponse, + EntityAncestryResponse, + EntityUpsertRequest, + EntityUpsertResponse, LocationResponse, - PageInfo, + LocationsCatalog, LocationUpdateLogEvent, LocationUpdateStatus, + PageInfo, } from './types'; diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 2c08482a96..217e077434 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -51,28 +51,38 @@ export type EntityUpsertResponse = { entity?: Entity; }; +/** @public */ +export type EntityAncestryResponse = { + rootEntityRef: string; + items: Array<{ + entity: Entity; + parentEntityRefs: string[]; + }>; +}; + +/** @public */ export type EntitiesCatalog = { /** * Fetch entities. * - * @param request Request options + * @param request - Request options */ entities(request?: EntitiesRequest): Promise; /** * Removes a single entity. * - * @param uid The metadata.uid of the entity + * @param uid - The metadata.uid of the entity */ removeEntityByUid(uid: string): Promise; /** * Writes a number of entities efficiently to storage. * - * @param requests The entities and their relations - * @param options.locationId The location that they all belong to (default none) - * @param options.dryRun Whether to throw away the results (default false) - * @param options.outputEntities Whether to return the resulting entities (default false) + * @param requests - The entities and their relations + * @param options.locationId - The location that they all belong to (default none) + * @param options.dryRun - Whether to throw away the results (default false) + * @param options.outputEntities - Whether to return the resulting entities (default false) */ batchAddOrUpdateEntities( requests: EntityUpsertRequest[], @@ -82,6 +92,13 @@ export type EntitiesCatalog = { outputEntities?: boolean; }, ): Promise; + + /** + * Returns the full ancestry tree upward along reference edges. + * + * @param entityRef - An entity reference to the root of the tree + */ + entityAncestry(entityRef: string): Promise; }; // @@ -93,6 +110,7 @@ export type LocationUpdateStatus = { status: string | null; message: string | null; }; + export type LocationUpdateLogEvent = { id: string; status: 'fail' | 'success'; diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts index e023e3ae9e..f49c553ad3 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts @@ -33,6 +33,7 @@ describe('HigherOrderOperations', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), batchAddOrUpdateEntities: jest.fn(), + entityAncestry: jest.fn(), }; locationsCatalog = { addLocation: jest.fn(), diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts new file mode 100644 index 0000000000..c173c926e8 --- /dev/null +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts @@ -0,0 +1,212 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { Knex } from 'knex'; +import { v4 as uuid } from 'uuid'; +import { DatabaseManager } from './database/DatabaseManager'; +import { + DbFinalEntitiesRow, + DbRefreshStateReferencesRow, + DbRefreshStateRow, +} from './database/tables'; +import { NextEntitiesCatalog } from './NextEntitiesCatalog'; + +describe('NextEntitiesCatalog', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + async function createDatabase(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await DatabaseManager.createDatabase(knex); + return { knex }; + } + + async function addEntity( + knex: Knex, + entity: Entity, + parents: { source?: string; entity?: Entity }[], + ) { + const id = uuid(); + const entityRef = stringifyEntityRef(entity); + const entityJson = JSON.stringify(entity); + + await knex('refresh_state').insert({ + entity_id: id, + entity_ref: entityRef, + unprocessed_entity: entityJson, + errors: '[]', + next_update_at: '2031-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + await knex('final_entities').insert({ + entity_id: id, + final_entity: entityJson, + hash: 'h', + stitch_ticket: '', + }); + + for (const parent of parents) { + await knex( + 'refresh_state_references', + ).insert({ + source_key: parent.source, + source_entity_ref: parent.entity && stringifyEntityRef(parent.entity), + target_entity_ref: stringifyEntityRef(entity), + }); + } + } + + describe('entityAncestry', () => { + it.each(databases.eachSupportedId())( + 'should return the ancestry with one parent, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + const grandparent: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'grandparent' }, + spec: {}, + }; + const parent: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'parent' }, + spec: {}, + }; + const root: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'root' }, + spec: {}, + }; + + await addEntity(knex, grandparent, [{ source: 's' }]); + await addEntity(knex, parent, [{ entity: grandparent }]); + await addEntity(knex, root, [{ entity: parent }]); + + const catalog = new NextEntitiesCatalog(knex); + const result = await catalog.entityAncestry('k:default/root'); + expect(result.rootEntityRef).toEqual('k:default/root'); + + expect(result.items).toEqual( + expect.arrayContaining([ + { + entity: expect.objectContaining({ metadata: { name: 'root' } }), + parentEntityRefs: ['k:default/parent'], + }, + { + entity: expect.objectContaining({ metadata: { name: 'parent' } }), + parentEntityRefs: ['k:default/grandparent'], + }, + { + entity: expect.objectContaining({ + metadata: { name: 'grandparent' }, + }), + parentEntityRefs: [], + }, + ]), + ); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should throw error if the entity does not exist, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + const catalog = new NextEntitiesCatalog(knex); + await expect(() => + catalog.entityAncestry('k:default/root'), + ).rejects.toThrow('No such entity k:default/root'); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should return the ancestry with multiple parents, %p', + async databaseId => { + const { knex } = await createDatabase(databaseId); + + const grandparent: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'grandparent' }, + spec: {}, + }; + const parent1: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'parent1' }, + spec: {}, + }; + const parent2: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'parent2' }, + spec: {}, + }; + const root: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'root' }, + spec: {}, + }; + + await addEntity(knex, grandparent, [{ source: 's' }]); + await addEntity(knex, parent1, [{ entity: grandparent }]); + await addEntity(knex, parent2, [{ entity: grandparent }]); + await addEntity(knex, root, [{ entity: parent1 }, { entity: parent2 }]); + + const catalog = new NextEntitiesCatalog(knex); + const result = await catalog.entityAncestry('k:default/root'); + expect(result.rootEntityRef).toEqual('k:default/root'); + + expect(result.items).toEqual( + expect.arrayContaining([ + { + entity: expect.objectContaining({ metadata: { name: 'root' } }), + parentEntityRefs: ['k:default/parent1', 'k:default/parent2'], + }, + { + entity: expect.objectContaining({ + metadata: { name: 'parent1' }, + }), + parentEntityRefs: ['k:default/grandparent'], + }, + { + entity: expect.objectContaining({ + metadata: { name: 'parent2' }, + }), + parentEntityRefs: ['k:default/grandparent'], + }, + { + entity: expect.objectContaining({ + metadata: { name: 'grandparent' }, + }), + parentEntityRefs: [], + }, + ]), + ); + }, + 60_000, + ); + }); +}); diff --git a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts index e9772b4aa3..14fda4ea24 100644 --- a/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/next/NextEntitiesCatalog.ts @@ -14,16 +14,19 @@ * limitations under the License. */ -import { InputError } from '@backstage/errors'; +import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; +import { InputError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { EntitiesCatalog, EntitiesRequest, EntitiesResponse, + EntityAncestryResponse, } from '../catalog/types'; import { DbPageInfo, EntityPagination } from '../database/types'; import { DbFinalEntitiesRow, + DbRefreshStateReferencesRow, DbRefreshStateRow, DbSearchRow, } from './database/tables'; @@ -161,6 +164,70 @@ export class NextEntitiesCatalog implements EntitiesCatalog { .delete(); } + async entityAncestry(rootRef: string): Promise { + const [rootRow] = await this.database('refresh_state') + .leftJoin('final_entities', { + 'refresh_state.entity_id': 'final_entities.entity_id', + }) + .where('refresh_state.entity_ref', '=', rootRef) + .select({ + entityJson: 'final_entities.final_entity', + }); + + if (!rootRow) { + throw new NotFoundError(`No such entity ${rootRef}`); + } + + const rootEntity = JSON.parse(rootRow.entityJson) as Entity; + const seenEntityRefs = new Set(); + const todo = new Array(); + const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>(); + + for ( + let current: Entity | undefined = rootEntity; + current; + current = todo.pop() + ) { + const currentRef = stringifyEntityRef(current); + seenEntityRefs.add(currentRef); + + const parentRows = await this.database( + 'refresh_state_references', + ) + .innerJoin('refresh_state', { + 'refresh_state_references.source_entity_ref': + 'refresh_state.entity_ref', + }) + .innerJoin('final_entities', { + 'refresh_state.entity_id': 'final_entities.entity_id', + }) + .where('refresh_state_references.target_entity_ref', '=', currentRef) + .select({ + parentEntityRef: 'refresh_state.entity_ref', + parentEntityJson: 'final_entities.final_entity', + }); + + const parentRefs: string[] = []; + for (const { parentEntityRef, parentEntityJson } of parentRows) { + parentRefs.push(parentEntityRef); + if (!seenEntityRefs.has(parentEntityRef)) { + seenEntityRefs.add(parentEntityRef); + todo.push(JSON.parse(parentEntityJson)); + } + } + + items.push({ + entity: current, + parentEntityRefs: parentRefs, + }); + } + + return { + rootEntityRef: stringifyEntityRef(rootEntity), + items, + }; + } + async batchAddOrUpdateEntities(): Promise { throw new Error('Not implemented'); } diff --git a/plugins/catalog-backend/src/next/NextRouter.test.ts b/plugins/catalog-backend/src/next/NextRouter.test.ts index 15c8716004..489e7efdb5 100644 --- a/plugins/catalog-backend/src/next/NextRouter.test.ts +++ b/plugins/catalog-backend/src/next/NextRouter.test.ts @@ -36,6 +36,7 @@ describe('createNextRouter readonly disabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), batchAddOrUpdateEntities: jest.fn(), + entityAncestry: jest.fn(), }; locationService = { getLocation: jest.fn(), @@ -318,6 +319,7 @@ describe('createNextRouter readonly enabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), batchAddOrUpdateEntities: jest.fn(), + entityAncestry: jest.fn(), }; locationService = { getLocation: jest.fn(), diff --git a/plugins/catalog-backend/src/next/NextRouter.ts b/plugins/catalog-backend/src/next/NextRouter.ts index b9da91b30a..6352f96c59 100644 --- a/plugins/catalog-backend/src/next/NextRouter.ts +++ b/plugins/catalog-backend/src/next/NextRouter.ts @@ -18,6 +18,7 @@ import { errorHandler } from '@backstage/backend-common'; import { analyzeLocationSchema, locationSpecSchema, + stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { NotFoundError } from '@backstage/errors'; @@ -124,7 +125,16 @@ export async function createNextRouter( ); } res.status(200).json(entities[0]); - }); + }) + .get( + '/entities/by-name/:kind/:namespace/:name/ancestry', + async (req, res) => { + const { kind, namespace, name } = req.params; + const entityRef = stringifyEntityRef({ kind, namespace, name }); + const response = await entitiesCatalog.entityAncestry(entityRef); + res.status(200).json(response); + }, + ); } if (locationService) { diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index fe23e15dd3..29b49ff157 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -39,6 +39,7 @@ describe('createRouter readonly disabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), batchAddOrUpdateEntities: jest.fn(), + entityAncestry: jest.fn(), }; locationsCatalog = { addLocation: jest.fn(), @@ -383,6 +384,7 @@ describe('createRouter readonly enabled', () => { entities: jest.fn(), removeEntityByUid: jest.fn(), batchAddOrUpdateEntities: jest.fn(), + entityAncestry: jest.fn(), }; locationsCatalog = { addLocation: jest.fn(),