Merge pull request #2970 from spotify/mob/relations-db

catalog-backend: initial implementation of relations storage

plugins/catalog-backend/migrations/20201019130742_add_relations_table.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.createTable('entities_relations', table => {
+    table.comment('All relations between entities in the catalog');
+    table
+      .uuid('originating_entity_id')
+      .references('id')
+      .inTable('entities')
+      .onDelete('CASCADE')
+      .notNullable()
+      .comment('The originating entity of the relation');
+    table
+      .string('source_full_name')
+      .notNullable()
+      .comment('The full name of the source entity of the relation');
+    table
+      .string('type')
+      .notNullable()
+      .comment('The type of the relation between the entities');
+    table
+      .string('target_full_name')
+      .notNullable()
+      .comment('The full name of the target entity of the relation');
+
+    table.primary(['source_full_name', 'type', 'target_full_name']);
+  });
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.dropTable('entities_relations');
+};

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts

@@ -31,6 +31,7 @@ describe('DatabaseEntitiesCatalog', () => {
       entityByName: jest.fn(),
       entityByUid: jest.fn(),
       removeEntityByUid: jest.fn(),
+      setRelations: jest.fn(),
       addLocation: jest.fn(),
       removeLocation: jest.fn(),
       location: jest.fn(),

plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts

@@ -15,13 +15,14 @@
  */
 
 import { ConflictError, NotFoundError } from '@backstage/backend-common';
-import type { Entity } from '@backstage/catalog-model';
 import {
   entityHasChanges,
   generateUpdatedEntity,
   getEntityName,
   LOCATION_ANNOTATION,
   serializeEntityRef,
+  Entity,
+  EntityRelationSpec,
 } from '@backstage/catalog-model';
 import { chunk, groupBy } from 'lodash';
 import limiterFactory from 'p-limit';
@@ -191,6 +192,16 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
     await Promise.all(tasks);
   }
 
+  // Set the relations originating from an entity using the DB layer
+  async setRelations(
+    originatingEntityId: string,
+    relations: EntityRelationSpec[],
+  ): Promise<void> {
+    return await this.database.transaction(tx =>
+      this.database.setRelations(tx, originatingEntityId, relations),
+    );
+  }
+
   // Given a batch of entities that were just read from a location, take them
   // into consideration by comparing against the existing catalog entities and
   // produce the list of entities to be added, and the list of entities to be

plugins/catalog-backend/src/catalog/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Entity, Location } from '@backstage/catalog-model';
+import { Entity, Location, EntityRelationSpec } from '@backstage/catalog-model';
 import type { EntityFilters } from '../database';
 
 //
@@ -37,6 +37,12 @@ export type EntitiesCatalog = {
     entities: Entity[],
     locationId?: string,
   ): Promise<void>;
+
+  // Same as the DB layer
+  setRelations(
+    entityUid: string,
+    relations: EntityRelationSpec[],
+  ): Promise<void>;
 };
 
 //

plugins/catalog-backend/src/database/CommonDatabase.test.ts

@@ -15,7 +15,7 @@
  */
 
 import { ConflictError } from '@backstage/backend-common';
-import type { Entity, Location } from '@backstage/catalog-model';
+import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
 import { DatabaseManager } from './DatabaseManager';
 import type {
   DbEntityRequest,
@@ -494,6 +494,196 @@ describe('CommonDatabase', () => {
     });
   });
 
+  describe('setRelations', () => {
+    it('adds a relation for an entity', async () => {
+      const mockRelations = [
+        {
+          source: {
+            kind: entityRequest.entity.kind,
+            namespace: entityRequest.entity.metadata.namespace!,
+            name: entityRequest.entity.metadata.name,
+          },
+          target: {
+            kind: 'component',
+            namespace: 'asd',
+            name: 'bleb',
+          },
+          type: 'child',
+        },
+      ];
+
+      const entityId = await db.transaction(async tx => {
+        const [{ entity }] = await db.addEntities(tx, [entityRequest]);
+
+        await db.setRelations(tx, entity?.metadata?.uid!, mockRelations);
+        return entity.metadata.uid;
+      });
+
+      const returnedEntity1 = await db.transaction(tx =>
+        db.entityByUid(tx, entityId!),
+      );
+      expect(returnedEntity1?.entity.relations).toEqual([
+        { target: mockRelations[0].target, type: 'child' },
+      ]);
+
+      const returnedEntity2 = await db.transaction(tx =>
+        db.entityByName(tx, mockRelations[0].source),
+      );
+      expect(returnedEntity2?.entity.relations).toEqual([
+        { target: mockRelations[0].target, type: 'child' },
+      ]);
+
+      const [returnedEntity3] = await db.transaction(tx => db.entities(tx));
+      expect(returnedEntity3?.entity.relations).toEqual([
+        { target: mockRelations[0].target, type: 'child' },
+      ]);
+    });
+
+    function makeRelation(source: string, type: string, target: string) {
+      return {
+        source: parseEntityRef(source, {
+          defaultKind: 'x',
+          defaultNamespace: 'x',
+        }),
+        type,
+        target: parseEntityRef(target, {
+          defaultKind: 'x',
+          defaultNamespace: 'x',
+        }),
+      };
+    }
+
+    it('should not allow setting relations on nonexistent entities', async () => {
+      await expect(
+        db.transaction(async tx => {
+          await db.setRelations(tx, 'nonexistent', [
+            makeRelation('a:b/c', 'rel1', 'x:y/z'),
+          ]);
+        }),
+      ).rejects.toThrow(/constraint failed/);
+    });
+
+    it('should allow setting relations on nonexistent entities without any relations', async () => {
+      await expect(
+        db.transaction(async tx => {
+          await db.setRelations(tx, 'nonexistent', []);
+        }),
+      ).resolves.toBeUndefined();
+    });
+
+    it('adds multiple relations for entities', async () => {
+      const entity1 = {
+        apiVersion: 'v1',
+        kind: 'a',
+        metadata: {
+          name: 'c',
+          namespace: 'b',
+        },
+      };
+      const entity2 = {
+        apiVersion: 'v1',
+        kind: 'x',
+        metadata: {
+          name: 'z',
+          namespace: 'y',
+        },
+      };
+      const fromEntity1 = [
+        makeRelation('a:b/c', 'rel1', 'x:y/z'),
+        makeRelation('x:y/z', 'rel2', 'a:b/c'),
+        makeRelation('a:b/c', 'rel2', 'x:y/z'),
+      ];
+      const fromEntity2 = [
+        makeRelation('a:b/c', 'rel4', 'x:y/z'),
+        makeRelation('a:b/c', 'rel5', 'x:y/z'),
+        makeRelation('x:y/z', 'rel6', 'a:b/c'),
+        // relations don't have to reference the originating entity, so this should be fine, but not show up
+        makeRelation('g:h/i', 'rel8', 'd:e/f'),
+      ];
+
+      const { id2: secondEntityId } = await db.transaction(async tx => {
+        const [{ entity: e1 }, { entity: e2 }] = await db.addEntities(tx, [
+          { entity: entity1 },
+          { entity: entity2 },
+        ]);
+        const id1 = e1?.metadata?.uid!;
+        const id2 = e2?.metadata?.uid!;
+
+        await db.setRelations(tx, id1, fromEntity1);
+        await db.setRelations(tx, id2, fromEntity2);
+
+        return { id1, id2 };
+      });
+
+      const res = await db.transaction(tx => db.entities(tx));
+      expect(
+        res.map(r => ({
+          name: r.entity.metadata.name,
+          relations: r.entity.relations,
+        })),
+      ).toEqual([
+        {
+          name: 'c',
+          relations: [
+            {
+              type: 'rel1',
+              target: { kind: 'x', namespace: 'y', name: 'z' },
+            },
+            {
+              type: 'rel2',
+              target: { kind: 'x', namespace: 'y', name: 'z' },
+            },
+            {
+              type: 'rel4',
+              target: { kind: 'x', namespace: 'y', name: 'z' },
+            },
+            {
+              type: 'rel5',
+              target: { kind: 'x', namespace: 'y', name: 'z' },
+            },
+          ],
+        },
+        {
+          name: 'z',
+          relations: [
+            {
+              type: 'rel2',
+              target: { kind: 'a', namespace: 'b', name: 'c' },
+            },
+            {
+              type: 'rel6',
+              target: { kind: 'a', namespace: 'b', name: 'c' },
+            },
+          ],
+        },
+      ]);
+
+      await db.transaction(tx => db.removeEntityByUid(tx, secondEntityId));
+
+      const res2 = await db.transaction(tx => db.entities(tx));
+      expect(
+        res2.map(r => ({
+          name: r.entity.metadata.name,
+          relations: r.entity.relations,
+        })),
+      ).toEqual([
+        {
+          name: 'c',
+          relations: [
+            {
+              type: 'rel1',
+              target: { kind: 'x', namespace: 'y', name: 'z' },
+            },
+            {
+              type: 'rel2',
+              target: { kind: 'x', namespace: 'y', name: 'z' },
+            },
+          ],
+        },
+      ]);
+    });
+  });
+
   describe('entityByName', () => {
     it('can get entities case insensitively', async () => {
       const entities: Entity[] = [

plugins/catalog-backend/src/database/CommonDatabase.ts

@@ -27,15 +27,18 @@ import {
   generateEntityEtag,
   generateEntityUid,
   Location,
+  EntityRelationSpec,
+  parseEntityName,
 } from '@backstage/catalog-model';
 import Knex from 'knex';
 import lodash from 'lodash';
 import type { Logger } from 'winston';
 import { buildEntitySearch } from './search';
-import type {
+import {
   Database,
   DatabaseLocationUpdateLogEvent,
   DatabaseLocationUpdateLogStatus,
+  DbEntitiesRelationsRow,
   DbEntitiesRow,
   DbEntitiesSearchRow,
   DbEntityRequest,
@@ -123,6 +126,8 @@ export class CommonDatabase implements Database {
         throw new InputError('May not specify etag for new entities');
       } else if (entity.metadata.generation !== undefined) {
         throw new InputError('May not specify generation for new entities');
+      } else if (entity.relations !== undefined) {
+        throw new InputError('May not specify relations for new entities');
       }
 
       const newEntity = {
@@ -281,7 +286,7 @@ export class CommonDatabase implements Database {
       .select('entities.*')
       .orderBy('full_name', 'asc');
 
-    return rows.map(row => this.toEntityResponse(row));
+    return Promise.all(rows.map(row => this.toEntityResponse(tx, row)));
   }
 
   async entityByName(
@@ -300,7 +305,7 @@ export class CommonDatabase implements Database {
       return undefined;
     }
 
-    return this.toEntityResponse(rows[0]);
+    return this.toEntityResponse(tx, rows[0]);
   }
 
   async entityByUid(
@@ -317,7 +322,7 @@ export class CommonDatabase implements Database {
       return undefined;
     }
 
-    return this.toEntityResponse(rows[0]);
+    return this.toEntityResponse(tx, rows[0]);
   }
 
   async removeEntityByUid(txOpaque: unknown, uid: string): Promise<void> {
@@ -330,6 +335,34 @@ export class CommonDatabase implements Database {
     }
   }
 
+  async setRelations(
+    txOpaque: unknown,
+    originatingEntityId: string,
+    relations: EntityRelationSpec[],
+  ): Promise<void> {
+    const tx = txOpaque as Knex.Transaction<any, any>;
+
+    // remove all relations that exist for the originating entity id.
+    await tx<DbEntitiesRelationsRow>('entities_relations')
+      .where({ originating_entity_id: originatingEntityId })
+      .del();
+
+    const serializeName = (e: EntityName) =>
+      `${e.kind}:${e.namespace}/${e.name}`.toLowerCase();
+
+    const relationsRows: DbEntitiesRelationsRow[] = relations.map(
+      ({ source, target, type }) => ({
+        originating_entity_id: originatingEntityId,
+        source_full_name: serializeName(source),
+        target_full_name: serializeName(target),
+        type,
+      }),
+    );
+
+    // TODO(blam): translate constraint failures to sane NotFoundError instead
+    await tx.batchInsert('entities_relations', relationsRows, BATCH_SIZE);
+  }
+
   async addLocation(location: Location): Promise<DbLocationsRow> {
     return await this.database.transaction<DbLocationsRow>(async tx => {
       const row: DbLocationsRow = {
@@ -469,12 +502,27 @@ export class CommonDatabase implements Database {
     };
   }
 
-  private toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
+  private async toEntityResponse(
+    tx: Knex.Transaction<any, any>,
+    row: DbEntitiesRow,
+  ): Promise<DbEntityResponse> {
     const entity = JSON.parse(row.data) as Entity;
     entity.metadata.uid = row.id;
     entity.metadata.etag = row.etag;
     entity.metadata.generation = Number(row.generation); // cast due to sqlite
 
+    // TODO(Rugvip): This is here because it's simple for now, but we likely
+    //               need to refactor this to be more efficient or introduce pagination.
+    const relations = await tx<DbEntitiesRelationsRow>('entities_relations')
+      .where({ source_full_name: row.full_name })
+      .orderBy(['type', 'target_full_name'])
+      .select();
+
+    entity.relations = relations.map(r => ({
+      target: parseEntityName(r.target_full_name),
+      type: r.type,
+    }));
+
     return {
       locationId: row.location_id || undefined,
       entity,

plugins/catalog-backend/src/database/types.ts

@@ -14,7 +14,12 @@
  * limitations under the License.
  */
 
-import type { Entity, EntityName, Location } from '@backstage/catalog-model';
+import type {
+  Entity,
+  EntityName,
+  Location,
+  EntityRelationSpec,
+} from '@backstage/catalog-model';
 
 export type DbEntitiesRow = {
   id: string;
@@ -35,6 +40,13 @@ export type DbEntityResponse = {
   entity: Entity;
 };
 
+export type DbEntitiesRelationsRow = {
+  originating_entity_id: string;
+  source_full_name: string;
+  type: string;
+  target_full_name: string;
+};
+
 export type DbEntitiesSearchRow = {
   entity_id: string;
   key: string;
@@ -152,6 +164,18 @@ export type Database = {
 
   removeEntityByUid(tx: unknown, uid: string): Promise<void>;
 
+  /**
+   * Remove current relations for the entity and replace them with the new relations array
+   * @param tx An ongoing transaction
+   * @param entityUid the entity uid
+   * @param relations the relationships to be set
+   */
+  setRelations(
+    tx: unknown,
+    entityUid: string,
+    relations: EntityRelationSpec[],
+  ): Promise<void>;
+
   addLocation(location: Location): Promise<DbLocationsRow>;
 
   removeLocation(tx: unknown, id: string): Promise<void>;

plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts

@@ -34,6 +34,7 @@ describe('HigherOrderOperations', () => {
       addOrUpdateEntity: jest.fn(),
       addEntities: jest.fn(),
       removeEntityByUid: jest.fn(),
+      setRelations: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
     };
     locationsCatalog = {

plugins/catalog-backend/src/service/router.test.ts

@@ -35,6 +35,7 @@ describe('createRouter', () => {
       addOrUpdateEntity: jest.fn(),
       addEntities: jest.fn(),
       removeEntityByUid: jest.fn(),
+      setRelations: jest.fn(),
       batchAddOrUpdateEntities: jest.fn(),
     };
     locationsCatalog = {