Merge pull request #7377 from backstage/mob/ancestry

Add a processing ancestry endpoint to the catalog

.changeset/tidy-houses-jam.md

@@ -0,0 +1,7 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Add `/entities/by-name/:kind/:namespace/:name/ancestry` to get the "processing parents" lineage of an entity.
+
+This involves a breaking change of adding the method `entityAncestry` to `EntitiesCatalog`.

plugins/catalog-backend/api-report.md

@@ -588,6 +588,8 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
   // (undocumented)
   entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
   // (undocumented)
+  entityAncestry(): Promise<never>;
+  // (undocumented)
   removeEntityByUid(uid: string): Promise<void>;
 }
 
@@ -805,8 +807,6 @@ export type DeferredEntity = {
 // @public
 export function durationText(startTimestamp: [number, number]): string;
 
-// Warning: (ae-missing-release-tag) "EntitiesCatalog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
 // @public (undocumented)
 export type EntitiesCatalog = {
   entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
@@ -819,6 +819,7 @@ export type EntitiesCatalog = {
       outputEntities?: boolean;
     },
   ): Promise<EntityUpsertResponse[]>;
+  entityAncestry(entityRef: string): Promise<EntityAncestryResponse>;
 };
 
 // Warning: (ae-missing-release-tag) "EntitiesRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -855,6 +856,15 @@ function entity(
   newEntity: Entity,
 ): CatalogProcessorResult;
 
+// @public (undocumented)
+export type EntityAncestryResponse = {
+  rootEntityRef: string;
+  items: Array<{
+    entity: Entity;
+    parentEntityRefs: string[];
+  }>;
+};
+
 // Warning: (ae-missing-release-tag) "EntityFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public
@@ -1501,12 +1511,9 @@ export class UrlReaderProcessor implements CatalogProcessor {
 
 // Warnings were encountered during analysis:
 //
-// src/catalog/types.d.ts:30:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/catalog/types.d.ts:36:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/catalog/types.d.ts:42:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-// src/catalog/types.d.ts:43:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
-// src/catalog/types.d.ts:44:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
-// src/catalog/types.d.ts:45:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
+// src/catalog/types.d.ts:52:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
+// src/catalog/types.d.ts:53:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
+// src/catalog/types.d.ts:54:8 - (tsdoc-param-tag-with-invalid-name) The @param block should be followed by a valid parameter name: The identifier cannot non-word characters
 // src/database/types.d.ts:125:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/database/types.d.ts:131:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/database/types.d.ts:132:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -369,4 +369,8 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
 
     return response.entity;
   }
+
+  async entityAncestry(): Promise<never> {
+    throw new Error('Not implemented');
+  }
 }

plugins/catalog-backend/src/catalog/index.ts

@@ -18,13 +18,14 @@ export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
 export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
 export type {
   EntitiesCatalog,
-  LocationsCatalog,
-  EntityUpsertRequest,
-  EntityUpsertResponse,
   EntitiesRequest,
   EntitiesResponse,
+  EntityAncestryResponse,
+  EntityUpsertRequest,
+  EntityUpsertResponse,
   LocationResponse,
-  PageInfo,
+  LocationsCatalog,
   LocationUpdateLogEvent,
   LocationUpdateStatus,
+  PageInfo,
 } from './types';

plugins/catalog-backend/src/catalog/types.ts

@@ -51,28 +51,38 @@ export type EntityUpsertResponse = {
   entity?: Entity;
 };
 
+/** @public */
+export type EntityAncestryResponse = {
+  rootEntityRef: string;
+  items: Array<{
+    entity: Entity;
+    parentEntityRefs: string[];
+  }>;
+};
+
+/** @public */
 export type EntitiesCatalog = {
   /**
    * Fetch entities.
    *
-   * @param request Request options
+   * @param request - Request options
    */
   entities(request?: EntitiesRequest): Promise<EntitiesResponse>;
 
   /**
    * Removes a single entity.
    *
-   * @param uid The metadata.uid of the entity
+   * @param uid - The metadata.uid of the entity
    */
   removeEntityByUid(uid: string): Promise<void>;
 
   /**
    * Writes a number of entities efficiently to storage.
    *
-   * @param requests The entities and their relations
-   * @param options.locationId The location that they all belong to (default none)
-   * @param options.dryRun Whether to throw away the results (default false)
-   * @param options.outputEntities Whether to return the resulting entities (default false)
+   * @param requests - The entities and their relations
+   * @param options.locationId - The location that they all belong to (default none)
+   * @param options.dryRun - Whether to throw away the results (default false)
+   * @param options.outputEntities - Whether to return the resulting entities (default false)
    */
   batchAddOrUpdateEntities(
     requests: EntityUpsertRequest[],
@@ -82,6 +92,13 @@ export type EntitiesCatalog = {
       outputEntities?: boolean;
     },
   ): Promise<EntityUpsertResponse[]>;
+
+  /**
+   * Returns the full ancestry tree upward along reference edges.
+   *
+   * @param entityRef - An entity reference to the root of the tree
+   */
+  entityAncestry(entityRef: string): Promise<EntityAncestryResponse>;
 };
 
 //
@@ -93,6 +110,7 @@ export type LocationUpdateStatus = {
   status: string | null;
   message: string | null;
 };
+
 export type LocationUpdateLogEvent = {
   id: string;
   status: 'fail' | 'success';

plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts

@@ -33,6 +33,7 @@ describe('HigherOrderOperations', () => {
       entities: jest.fn(),
       removeEntityByUid: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
+      entityAncestry: jest.fn(),
     };
     locationsCatalog = {
       addLocation: jest.fn(),

plugins/catalog-backend/src/next/NextEntitiesCatalog.test.ts

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
+import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
+import { Knex } from 'knex';
+import { v4 as uuid } from 'uuid';
+import { DatabaseManager } from './database/DatabaseManager';
+import {
+  DbFinalEntitiesRow,
+  DbRefreshStateReferencesRow,
+  DbRefreshStateRow,
+} from './database/tables';
+import { NextEntitiesCatalog } from './NextEntitiesCatalog';
+
+describe('NextEntitiesCatalog', () => {
+  const databases = TestDatabases.create({
+    ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
+  });
+
+  async function createDatabase(databaseId: TestDatabaseId) {
+    const knex = await databases.init(databaseId);
+    await DatabaseManager.createDatabase(knex);
+    return { knex };
+  }
+
+  async function addEntity(
+    knex: Knex,
+    entity: Entity,
+    parents: { source?: string; entity?: Entity }[],
+  ) {
+    const id = uuid();
+    const entityRef = stringifyEntityRef(entity);
+    const entityJson = JSON.stringify(entity);
+
+    await knex<DbRefreshStateRow>('refresh_state').insert({
+      entity_id: id,
+      entity_ref: entityRef,
+      unprocessed_entity: entityJson,
+      errors: '[]',
+      next_update_at: '2031-01-01 23:00:00',
+      last_discovery_at: '2021-04-01 13:37:00',
+    });
+
+    await knex<DbFinalEntitiesRow>('final_entities').insert({
+      entity_id: id,
+      final_entity: entityJson,
+      hash: 'h',
+      stitch_ticket: '',
+    });
+
+    for (const parent of parents) {
+      await knex<DbRefreshStateReferencesRow>(
+        'refresh_state_references',
+      ).insert({
+        source_key: parent.source,
+        source_entity_ref: parent.entity && stringifyEntityRef(parent.entity),
+        target_entity_ref: stringifyEntityRef(entity),
+      });
+    }
+  }
+
+  describe('entityAncestry', () => {
+    it.each(databases.eachSupportedId())(
+      'should return the ancestry with one parent, %p',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+
+        const grandparent: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'grandparent' },
+          spec: {},
+        };
+        const parent: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'parent' },
+          spec: {},
+        };
+        const root: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'root' },
+          spec: {},
+        };
+
+        await addEntity(knex, grandparent, [{ source: 's' }]);
+        await addEntity(knex, parent, [{ entity: grandparent }]);
+        await addEntity(knex, root, [{ entity: parent }]);
+
+        const catalog = new NextEntitiesCatalog(knex);
+        const result = await catalog.entityAncestry('k:default/root');
+        expect(result.rootEntityRef).toEqual('k:default/root');
+
+        expect(result.items).toEqual(
+          expect.arrayContaining([
+            {
+              entity: expect.objectContaining({ metadata: { name: 'root' } }),
+              parentEntityRefs: ['k:default/parent'],
+            },
+            {
+              entity: expect.objectContaining({ metadata: { name: 'parent' } }),
+              parentEntityRefs: ['k:default/grandparent'],
+            },
+            {
+              entity: expect.objectContaining({
+                metadata: { name: 'grandparent' },
+              }),
+              parentEntityRefs: [],
+            },
+          ]),
+        );
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should throw error if the entity does not exist, %p',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+        const catalog = new NextEntitiesCatalog(knex);
+        await expect(() =>
+          catalog.entityAncestry('k:default/root'),
+        ).rejects.toThrow('No such entity k:default/root');
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should return the ancestry with multiple parents, %p',
+      async databaseId => {
+        const { knex } = await createDatabase(databaseId);
+
+        const grandparent: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'grandparent' },
+          spec: {},
+        };
+        const parent1: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'parent1' },
+          spec: {},
+        };
+        const parent2: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'parent2' },
+          spec: {},
+        };
+        const root: Entity = {
+          apiVersion: 'a',
+          kind: 'k',
+          metadata: { name: 'root' },
+          spec: {},
+        };
+
+        await addEntity(knex, grandparent, [{ source: 's' }]);
+        await addEntity(knex, parent1, [{ entity: grandparent }]);
+        await addEntity(knex, parent2, [{ entity: grandparent }]);
+        await addEntity(knex, root, [{ entity: parent1 }, { entity: parent2 }]);
+
+        const catalog = new NextEntitiesCatalog(knex);
+        const result = await catalog.entityAncestry('k:default/root');
+        expect(result.rootEntityRef).toEqual('k:default/root');
+
+        expect(result.items).toEqual(
+          expect.arrayContaining([
+            {
+              entity: expect.objectContaining({ metadata: { name: 'root' } }),
+              parentEntityRefs: ['k:default/parent1', 'k:default/parent2'],
+            },
+            {
+              entity: expect.objectContaining({
+                metadata: { name: 'parent1' },
+              }),
+              parentEntityRefs: ['k:default/grandparent'],
+            },
+            {
+              entity: expect.objectContaining({
+                metadata: { name: 'parent2' },
+              }),
+              parentEntityRefs: ['k:default/grandparent'],
+            },
+            {
+              entity: expect.objectContaining({
+                metadata: { name: 'grandparent' },
+              }),
+              parentEntityRefs: [],
+            },
+          ]),
+        );
+      },
+      60_000,
+    );
+  });
+});

plugins/catalog-backend/src/next/NextEntitiesCatalog.ts

@@ -14,16 +14,19 @@
  * limitations under the License.
  */
 
-import { InputError } from '@backstage/errors';
+import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
+import { InputError, NotFoundError } from '@backstage/errors';
 import { Knex } from 'knex';
 import {
   EntitiesCatalog,
   EntitiesRequest,
   EntitiesResponse,
+  EntityAncestryResponse,
 } from '../catalog/types';
 import { DbPageInfo, EntityPagination } from '../database/types';
 import {
   DbFinalEntitiesRow,
+  DbRefreshStateReferencesRow,
   DbRefreshStateRow,
   DbSearchRow,
 } from './database/tables';
@@ -161,6 +164,70 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
       .delete();
   }
 
+  async entityAncestry(rootRef: string): Promise<EntityAncestryResponse> {
+    const [rootRow] = await this.database<DbRefreshStateRow>('refresh_state')
+      .leftJoin<DbFinalEntitiesRow>('final_entities', {
+        'refresh_state.entity_id': 'final_entities.entity_id',
+      })
+      .where('refresh_state.entity_ref', '=', rootRef)
+      .select({
+        entityJson: 'final_entities.final_entity',
+      });
+
+    if (!rootRow) {
+      throw new NotFoundError(`No such entity ${rootRef}`);
+    }
+
+    const rootEntity = JSON.parse(rootRow.entityJson) as Entity;
+    const seenEntityRefs = new Set<string>();
+    const todo = new Array<Entity>();
+    const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>();
+
+    for (
+      let current: Entity | undefined = rootEntity;
+      current;
+      current = todo.pop()
+    ) {
+      const currentRef = stringifyEntityRef(current);
+      seenEntityRefs.add(currentRef);
+
+      const parentRows = await this.database<DbRefreshStateReferencesRow>(
+        'refresh_state_references',
+      )
+        .innerJoin<DbRefreshStateRow>('refresh_state', {
+          'refresh_state_references.source_entity_ref':
+            'refresh_state.entity_ref',
+        })
+        .innerJoin<DbFinalEntitiesRow>('final_entities', {
+          'refresh_state.entity_id': 'final_entities.entity_id',
+        })
+        .where('refresh_state_references.target_entity_ref', '=', currentRef)
+        .select({
+          parentEntityRef: 'refresh_state.entity_ref',
+          parentEntityJson: 'final_entities.final_entity',
+        });
+
+      const parentRefs: string[] = [];
+      for (const { parentEntityRef, parentEntityJson } of parentRows) {
+        parentRefs.push(parentEntityRef);
+        if (!seenEntityRefs.has(parentEntityRef)) {
+          seenEntityRefs.add(parentEntityRef);
+          todo.push(JSON.parse(parentEntityJson));
+        }
+      }
+
+      items.push({
+        entity: current,
+        parentEntityRefs: parentRefs,
+      });
+    }
+
+    return {
+      rootEntityRef: stringifyEntityRef(rootEntity),
+      items,
+    };
+  }
+
   async batchAddOrUpdateEntities(): Promise<never> {
     throw new Error('Not implemented');
   }

plugins/catalog-backend/src/next/NextRouter.test.ts

@@ -36,6 +36,7 @@ describe('createNextRouter readonly disabled', () => {
       entities: jest.fn(),
       removeEntityByUid: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
+      entityAncestry: jest.fn(),
     };
     locationService = {
       getLocation: jest.fn(),
@@ -318,6 +319,7 @@ describe('createNextRouter readonly enabled', () => {
       entities: jest.fn(),
       removeEntityByUid: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
+      entityAncestry: jest.fn(),
     };
     locationService = {
       getLocation: jest.fn(),

plugins/catalog-backend/src/next/NextRouter.ts

@@ -18,6 +18,7 @@ import { errorHandler } from '@backstage/backend-common';
 import {
   analyzeLocationSchema,
   locationSpecSchema,
+  stringifyEntityRef,
 } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { NotFoundError } from '@backstage/errors';
@@ -124,7 +125,16 @@ export async function createNextRouter(
           );
         }
         res.status(200).json(entities[0]);
-      });
+      })
+      .get(
+        '/entities/by-name/:kind/:namespace/:name/ancestry',
+        async (req, res) => {
+          const { kind, namespace, name } = req.params;
+          const entityRef = stringifyEntityRef({ kind, namespace, name });
+          const response = await entitiesCatalog.entityAncestry(entityRef);
+          res.status(200).json(response);
+        },
+      );
   }
 
   if (locationService) {

plugins/catalog-backend/src/service/router.test.ts

@@ -39,6 +39,7 @@ describe('createRouter readonly disabled', () => {
       entities: jest.fn(),
       removeEntityByUid: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
+      entityAncestry: jest.fn(),
     };
     locationsCatalog = {
       addLocation: jest.fn(),
@@ -383,6 +384,7 @@ describe('createRouter readonly enabled', () => {
       entities: jest.fn(),
       removeEntityByUid: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
+      entityAncestry: jest.fn(),
     };
     locationsCatalog = {
       addLocation: jest.fn(),