diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index 9e0aec7de6..9e7ece4122 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -15,6 +15,7 @@ */ import { JsonObject } from '@backstage/config'; +import { EntityName } from '../types'; /** * The format envelope that's common to all versions/kinds of entity. @@ -42,6 +43,11 @@ export type Entity = { * The specification data describing the entity itself. */ spec?: JsonObject; + + /** + * The relations that this entity has with other entities. + */ + relations?: EntityRelation[]; }; /** @@ -120,3 +126,37 @@ export type EntityMeta = JsonObject & { */ tags?: string[]; }; + +/** + * A relation of a specific type to another entity in the catalog. + */ +export type EntityRelation = { + /** + * The type of the relation. + */ + type: string; + + /** + * The target entity of this relation. + */ + target: EntityName; +}; + +/** + * Holds the relationship data for entities + */ +export type EntityRelationSpec = { + /** + * The source entity of this relation. + */ + source: EntityName; + /** + * The type of the relation. + */ + type: string; + + /** + * The target entity of this relation. + */ + target: EntityName; +}; diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts index 759a94ca67..d450119f0a 100644 --- a/packages/catalog-model/src/entity/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -18,7 +18,12 @@ export { ENTITY_DEFAULT_NAMESPACE, ENTITY_META_GENERATED_FIELDS, } from './constants'; -export type { Entity, EntityMeta } from './Entity'; +export type { + Entity, + EntityMeta, + EntityRelation, + EntityRelationSpec, +} from './Entity'; export * from './policies'; export { getEntityName, diff --git a/plugins/catalog-backend/migrations/20201019130742_add_relations_table.js b/plugins/catalog-backend/migrations/20201019130742_add_relations_table.js new file mode 100644 index 0000000000..accf9035e7 --- /dev/null +++ b/plugins/catalog-backend/migrations/20201019130742_add_relations_table.js @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The originating entity of the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + + table.primary(['source_full_name', 'type', 'target_full_name']); + }); +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('entities_relations'); +}; diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts index 3d1f887e1f..d84a8e071c 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts @@ -31,6 +31,7 @@ describe('DatabaseEntitiesCatalog', () => { entityByName: jest.fn(), entityByUid: jest.fn(), removeEntityByUid: jest.fn(), + setRelations: jest.fn(), addLocation: jest.fn(), removeLocation: jest.fn(), location: jest.fn(), diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts index 0a213dd186..efb541727a 100644 --- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts @@ -15,13 +15,14 @@ */ import { ConflictError, NotFoundError } from '@backstage/backend-common'; -import type { Entity } from '@backstage/catalog-model'; import { entityHasChanges, generateUpdatedEntity, getEntityName, LOCATION_ANNOTATION, serializeEntityRef, + Entity, + EntityRelationSpec, } from '@backstage/catalog-model'; import { chunk, groupBy } from 'lodash'; import limiterFactory from 'p-limit'; @@ -191,6 +192,16 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog { await Promise.all(tasks); } + // Set the relations originating from an entity using the DB layer + async setRelations( + originatingEntityId: string, + relations: EntityRelationSpec[], + ): Promise { + return await this.database.transaction(tx => + this.database.setRelations(tx, originatingEntityId, relations), + ); + } + // Given a batch of entities that were just read from a location, take them // into consideration by comparing against the existing catalog entities and // produce the list of entities to be added, and the list of entities to be diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index ca73eddce9..e50b5f15db 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, Location } from '@backstage/catalog-model'; +import { Entity, Location, EntityRelationSpec } from '@backstage/catalog-model'; import type { EntityFilters } from '../database'; // @@ -37,6 +37,12 @@ export type EntitiesCatalog = { entities: Entity[], locationId?: string, ): Promise; + + // Same as the DB layer + setRelations( + entityUid: string, + relations: EntityRelationSpec[], + ): Promise; }; // diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts index f3ca212df0..0e59e0ff5a 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts @@ -15,7 +15,7 @@ */ import { ConflictError } from '@backstage/backend-common'; -import type { Entity, Location } from '@backstage/catalog-model'; +import { Entity, Location, parseEntityRef } from '@backstage/catalog-model'; import { DatabaseManager } from './DatabaseManager'; import type { DbEntityRequest, @@ -494,6 +494,196 @@ describe('CommonDatabase', () => { }); }); + describe('setRelations', () => { + it('adds a relation for an entity', async () => { + const mockRelations = [ + { + source: { + kind: entityRequest.entity.kind, + namespace: entityRequest.entity.metadata.namespace!, + name: entityRequest.entity.metadata.name, + }, + target: { + kind: 'component', + namespace: 'asd', + name: 'bleb', + }, + type: 'child', + }, + ]; + + const entityId = await db.transaction(async tx => { + const [{ entity }] = await db.addEntities(tx, [entityRequest]); + + await db.setRelations(tx, entity?.metadata?.uid!, mockRelations); + return entity.metadata.uid; + }); + + const returnedEntity1 = await db.transaction(tx => + db.entityByUid(tx, entityId!), + ); + expect(returnedEntity1?.entity.relations).toEqual([ + { target: mockRelations[0].target, type: 'child' }, + ]); + + const returnedEntity2 = await db.transaction(tx => + db.entityByName(tx, mockRelations[0].source), + ); + expect(returnedEntity2?.entity.relations).toEqual([ + { target: mockRelations[0].target, type: 'child' }, + ]); + + const [returnedEntity3] = await db.transaction(tx => db.entities(tx)); + expect(returnedEntity3?.entity.relations).toEqual([ + { target: mockRelations[0].target, type: 'child' }, + ]); + }); + + function makeRelation(source: string, type: string, target: string) { + return { + source: parseEntityRef(source, { + defaultKind: 'x', + defaultNamespace: 'x', + }), + type, + target: parseEntityRef(target, { + defaultKind: 'x', + defaultNamespace: 'x', + }), + }; + } + + it('should not allow setting relations on nonexistent entities', async () => { + await expect( + db.transaction(async tx => { + await db.setRelations(tx, 'nonexistent', [ + makeRelation('a:b/c', 'rel1', 'x:y/z'), + ]); + }), + ).rejects.toThrow(/constraint failed/); + }); + + it('should allow setting relations on nonexistent entities without any relations', async () => { + await expect( + db.transaction(async tx => { + await db.setRelations(tx, 'nonexistent', []); + }), + ).resolves.toBeUndefined(); + }); + + it('adds multiple relations for entities', async () => { + const entity1 = { + apiVersion: 'v1', + kind: 'a', + metadata: { + name: 'c', + namespace: 'b', + }, + }; + const entity2 = { + apiVersion: 'v1', + kind: 'x', + metadata: { + name: 'z', + namespace: 'y', + }, + }; + const fromEntity1 = [ + makeRelation('a:b/c', 'rel1', 'x:y/z'), + makeRelation('x:y/z', 'rel2', 'a:b/c'), + makeRelation('a:b/c', 'rel2', 'x:y/z'), + ]; + const fromEntity2 = [ + makeRelation('a:b/c', 'rel4', 'x:y/z'), + makeRelation('a:b/c', 'rel5', 'x:y/z'), + makeRelation('x:y/z', 'rel6', 'a:b/c'), + // relations don't have to reference the originating entity, so this should be fine, but not show up + makeRelation('g:h/i', 'rel8', 'd:e/f'), + ]; + + const { id2: secondEntityId } = await db.transaction(async tx => { + const [{ entity: e1 }, { entity: e2 }] = await db.addEntities(tx, [ + { entity: entity1 }, + { entity: entity2 }, + ]); + const id1 = e1?.metadata?.uid!; + const id2 = e2?.metadata?.uid!; + + await db.setRelations(tx, id1, fromEntity1); + await db.setRelations(tx, id2, fromEntity2); + + return { id1, id2 }; + }); + + const res = await db.transaction(tx => db.entities(tx)); + expect( + res.map(r => ({ + name: r.entity.metadata.name, + relations: r.entity.relations, + })), + ).toEqual([ + { + name: 'c', + relations: [ + { + type: 'rel1', + target: { kind: 'x', namespace: 'y', name: 'z' }, + }, + { + type: 'rel2', + target: { kind: 'x', namespace: 'y', name: 'z' }, + }, + { + type: 'rel4', + target: { kind: 'x', namespace: 'y', name: 'z' }, + }, + { + type: 'rel5', + target: { kind: 'x', namespace: 'y', name: 'z' }, + }, + ], + }, + { + name: 'z', + relations: [ + { + type: 'rel2', + target: { kind: 'a', namespace: 'b', name: 'c' }, + }, + { + type: 'rel6', + target: { kind: 'a', namespace: 'b', name: 'c' }, + }, + ], + }, + ]); + + await db.transaction(tx => db.removeEntityByUid(tx, secondEntityId)); + + const res2 = await db.transaction(tx => db.entities(tx)); + expect( + res2.map(r => ({ + name: r.entity.metadata.name, + relations: r.entity.relations, + })), + ).toEqual([ + { + name: 'c', + relations: [ + { + type: 'rel1', + target: { kind: 'x', namespace: 'y', name: 'z' }, + }, + { + type: 'rel2', + target: { kind: 'x', namespace: 'y', name: 'z' }, + }, + ], + }, + ]); + }); + }); + describe('entityByName', () => { it('can get entities case insensitively', async () => { const entities: Entity[] = [ diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index dca5072b30..e72926b13a 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -27,15 +27,18 @@ import { generateEntityEtag, generateEntityUid, Location, + EntityRelationSpec, + parseEntityName, } from '@backstage/catalog-model'; import Knex from 'knex'; import lodash from 'lodash'; import type { Logger } from 'winston'; import { buildEntitySearch } from './search'; -import type { +import { Database, DatabaseLocationUpdateLogEvent, DatabaseLocationUpdateLogStatus, + DbEntitiesRelationsRow, DbEntitiesRow, DbEntitiesSearchRow, DbEntityRequest, @@ -123,6 +126,8 @@ export class CommonDatabase implements Database { throw new InputError('May not specify etag for new entities'); } else if (entity.metadata.generation !== undefined) { throw new InputError('May not specify generation for new entities'); + } else if (entity.relations !== undefined) { + throw new InputError('May not specify relations for new entities'); } const newEntity = { @@ -281,7 +286,7 @@ export class CommonDatabase implements Database { .select('entities.*') .orderBy('full_name', 'asc'); - return rows.map(row => this.toEntityResponse(row)); + return Promise.all(rows.map(row => this.toEntityResponse(tx, row))); } async entityByName( @@ -300,7 +305,7 @@ export class CommonDatabase implements Database { return undefined; } - return this.toEntityResponse(rows[0]); + return this.toEntityResponse(tx, rows[0]); } async entityByUid( @@ -317,7 +322,7 @@ export class CommonDatabase implements Database { return undefined; } - return this.toEntityResponse(rows[0]); + return this.toEntityResponse(tx, rows[0]); } async removeEntityByUid(txOpaque: unknown, uid: string): Promise { @@ -330,6 +335,34 @@ export class CommonDatabase implements Database { } } + async setRelations( + txOpaque: unknown, + originatingEntityId: string, + relations: EntityRelationSpec[], + ): Promise { + const tx = txOpaque as Knex.Transaction; + + // remove all relations that exist for the originating entity id. + await tx('entities_relations') + .where({ originating_entity_id: originatingEntityId }) + .del(); + + const serializeName = (e: EntityName) => + `${e.kind}:${e.namespace}/${e.name}`.toLowerCase(); + + const relationsRows: DbEntitiesRelationsRow[] = relations.map( + ({ source, target, type }) => ({ + originating_entity_id: originatingEntityId, + source_full_name: serializeName(source), + target_full_name: serializeName(target), + type, + }), + ); + + // TODO(blam): translate constraint failures to sane NotFoundError instead + await tx.batchInsert('entities_relations', relationsRows, BATCH_SIZE); + } + async addLocation(location: Location): Promise { return await this.database.transaction(async tx => { const row: DbLocationsRow = { @@ -469,12 +502,27 @@ export class CommonDatabase implements Database { }; } - private toEntityResponse(row: DbEntitiesRow): DbEntityResponse { + private async toEntityResponse( + tx: Knex.Transaction, + row: DbEntitiesRow, + ): Promise { const entity = JSON.parse(row.data) as Entity; entity.metadata.uid = row.id; entity.metadata.etag = row.etag; entity.metadata.generation = Number(row.generation); // cast due to sqlite + // TODO(Rugvip): This is here because it's simple for now, but we likely + // need to refactor this to be more efficient or introduce pagination. + const relations = await tx('entities_relations') + .where({ source_full_name: row.full_name }) + .orderBy(['type', 'target_full_name']) + .select(); + + entity.relations = relations.map(r => ({ + target: parseEntityName(r.target_full_name), + type: r.type, + })); + return { locationId: row.location_id || undefined, entity, diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 14e8d5be64..fa038c0e1d 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -14,7 +14,12 @@ * limitations under the License. */ -import type { Entity, EntityName, Location } from '@backstage/catalog-model'; +import type { + Entity, + EntityName, + Location, + EntityRelationSpec, +} from '@backstage/catalog-model'; export type DbEntitiesRow = { id: string; @@ -35,6 +40,13 @@ export type DbEntityResponse = { entity: Entity; }; +export type DbEntitiesRelationsRow = { + originating_entity_id: string; + source_full_name: string; + type: string; + target_full_name: string; +}; + export type DbEntitiesSearchRow = { entity_id: string; key: string; @@ -152,6 +164,18 @@ export type Database = { removeEntityByUid(tx: unknown, uid: string): Promise; + /** + * Remove current relations for the entity and replace them with the new relations array + * @param tx An ongoing transaction + * @param entityUid the entity uid + * @param relations the relationships to be set + */ + setRelations( + tx: unknown, + entityUid: string, + relations: EntityRelationSpec[], + ): Promise; + addLocation(location: Location): Promise; removeLocation(tx: unknown, id: string): Promise; diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts index 7fe164d05f..34320fe18f 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts @@ -34,6 +34,7 @@ describe('HigherOrderOperations', () => { addOrUpdateEntity: jest.fn(), addEntities: jest.fn(), removeEntityByUid: jest.fn(), + setRelations: jest.fn(), batchAddOrUpdateEntities: jest.fn(), }; locationsCatalog = { diff --git a/plugins/catalog-backend/src/service/router.test.ts b/plugins/catalog-backend/src/service/router.test.ts index 414f8a93de..a2b17ba360 100644 --- a/plugins/catalog-backend/src/service/router.test.ts +++ b/plugins/catalog-backend/src/service/router.test.ts @@ -35,6 +35,7 @@ describe('createRouter', () => { addOrUpdateEntity: jest.fn(), addEntities: jest.fn(), removeEntityByUid: jest.fn(), + setRelations: jest.fn(), batchAddOrUpdateEntities: jest.fn(), }; locationsCatalog = {