From 2daf18e8095c86ed71b0f059e9b895f3cdf7d6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 25 Nov 2020 10:51:22 +0100 Subject: [PATCH] catalog: emit all common relations based on specs (#3408) --- .changeset/pink-goats-sort.md | 6 + packages/catalog-model/src/entity/ref.ts | 16 +- packages/catalog-model/src/kinds/relations.ts | 2 + .../20201123205611_relations_table_uniq.js | 93 +++++++++ plugins/catalog-backend/package.json | 2 +- .../src/database/CommonDatabase.ts | 17 +- .../BuiltinKindsEntityProcessor.test.ts | 180 ++++++++++++++++++ .../processors/BuiltinKindsEntityProcessor.ts | 128 ++++++++++++- .../processors/OwnerRelationProcessor.ts | 74 ------- .../src/ingestion/processors/index.ts | 3 +- .../src/service/CatalogBuilder.test.ts | 2 +- .../src/service/CatalogBuilder.ts | 6 +- 12 files changed, 437 insertions(+), 92 deletions(-) create mode 100644 .changeset/pink-goats-sort.md create mode 100644 plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js delete mode 100644 plugins/catalog-backend/src/ingestion/processors/OwnerRelationProcessor.ts diff --git a/.changeset/pink-goats-sort.md b/.changeset/pink-goats-sort.md new file mode 100644 index 0000000000..828cccba84 --- /dev/null +++ b/.changeset/pink-goats-sort.md @@ -0,0 +1,6 @@ +--- +'@backstage/catalog-model': patch +'@backstage/plugin-catalog-backend': patch +--- + +Start emitting all known relation types from the core entity kinds, based on their spec data. diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index 2cacfce53a..9f6f175d8d 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -84,6 +84,14 @@ export function parseEntityName( * @param context The context of defaults that the parsing happens within * @returns The compound form of the reference */ +export function parseEntityRef( + ref: EntityRef, + context?: { defaultKind: string; defaultNamespace: string }, +): { + kind: string; + namespace: string; + name: string; +}; export function parseEntityRef( ref: EntityRef, context?: { defaultKind: string }, @@ -100,14 +108,6 @@ export function parseEntityRef( namespace: string; name: string; }; -export function parseEntityRef( - ref: EntityRef, - context?: { defaultKind: string; defaultNamespace: string }, -): { - kind: string; - namespace: string; - name: string; -}; export function parseEntityRef( ref: EntityRef, context: EntityRefContext = {}, diff --git a/packages/catalog-model/src/kinds/relations.ts b/packages/catalog-model/src/kinds/relations.ts index 846313ec8c..3d5d629b9e 100644 --- a/packages/catalog-model/src/kinds/relations.ts +++ b/packages/catalog-model/src/kinds/relations.ts @@ -33,7 +33,9 @@ export const RELATION_OWNER_OF = 'ownerOf'; * A relation with an API entity, typically from a component or system */ export const RELATION_CONSUMES_API = 'consumesApi'; +export const RELATION_API_CONSUMED_BY = 'apiConsumedBy'; export const RELATION_PROVIDES_API = 'providesApi'; +export const RELATION_API_PROVIDED_BY = 'apiProvidedBy'; /** * A relation denoting a dependency on another entity. diff --git a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js new file mode 100644 index 0000000000..a8cf62b4c4 --- /dev/null +++ b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client === 'sqlite3') { + // sqlite doesn't support dropPrimary so we recreate it properly instead + await knex.schema.dropTable('entities_relations'); + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + table.index('source_full_name', 'source_full_name_idx'); + }); + } else { + await knex.schema.alterTable('entities_relations', table => { + table.dropPrimary(); + table.index('source_full_name', 'source_full_name_idx'); + }); + } +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client === 'sqlite3') { + await knex.schema.dropTable('entities_relations'); + await knex.schema.createTable('entities_relations', table => { + table.comment('All relations between entities in the catalog'); + table + .uuid('originating_entity_id') + .references('id') + .inTable('entities') + .onDelete('CASCADE') + .notNullable() + .comment('The entity that provided the relation'); + table + .string('source_full_name') + .notNullable() + .comment('The full name of the source entity of the relation'); + table + .string('type') + .notNullable() + .comment('The type of the relation between the entities'); + table + .string('target_full_name') + .notNullable() + .comment('The full name of the target entity of the relation'); + + table.primary(['source_full_name', 'type', 'target_full_name']); + }); + } else { + await knex.schema.alterTable('entities_relations', table => { + table.dropIndex([], 'source_full_name_idx'); + table.primary(['source_full_name', 'type', 'target_full_name']); + }); + } +}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 51a9978f30..26f7180d43 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -26,6 +26,7 @@ "@backstage/config": "^0.1.1", "@octokit/graphql": "^4.5.6", "@types/express": "^4.17.6", + "@types/ldapjs": "^1.0.9", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", @@ -51,7 +52,6 @@ "@backstage/test-utils": "^0.1.3", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", - "@types/ldapjs": "^1.0.9", "@types/lodash": "^4.14.151", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts index f86ab9e598..287756b0a8 100644 --- a/plugins/catalog-backend/src/database/CommonDatabase.ts +++ b/plugins/catalog-backend/src/database/CommonDatabase.ts @@ -345,7 +345,11 @@ export class CommonDatabase implements Database { ); // TODO(blam): translate constraint failures to sane NotFoundError instead - await tx.batchInsert('entities_relations', relationsRows, BATCH_SIZE); + await tx.batchInsert( + 'entities_relations', + deduplicateRelations(relationsRows), + BATCH_SIZE, + ); } async addLocation( @@ -506,7 +510,7 @@ export class CommonDatabase implements Database { .orderBy(['type', 'target_full_name']) .select(); - entity.relations = relations.map(r => ({ + entity.relations = deduplicateRelations(relations).map(r => ({ target: parseEntityName(r.target_full_name), type: r.type, })); @@ -517,3 +521,12 @@ export class CommonDatabase implements Database { }; } } + +function deduplicateRelations( + rows: DbEntitiesRelationsRow[], +): DbEntitiesRelationsRow[] { + return lodash.uniqBy( + rows, + r => `${r.source_full_name}:${r.target_full_name}:${r.type}`, + ); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts index e1328e827a..94d35cc109 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts @@ -14,6 +14,12 @@ * limitations under the License. */ +import { + ApiEntity, + ComponentEntity, + GroupEntity, + UserEntity, +} from '@backstage/catalog-model'; import { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; describe('BuiltinKindsEntityProcessor', () => { @@ -44,4 +50,178 @@ describe('BuiltinKindsEntityProcessor', () => { }, }); }); + + describe('postProcessEntity', () => { + const processor = new BuiltinKindsEntityProcessor(); + const location = { type: 'a', target: 'b' }; + const emit = jest.fn(); + + afterEach(() => jest.resetAllMocks()); + + it('generates relations for component entities', async () => { + const entity: ComponentEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'n' }, + spec: { + type: 'service', + owner: 'o', + lifecycle: 'l', + implementsApis: ['a'], + }, + }; + + await processor.postProcessEntity(entity, location, emit); + + expect(emit).toBeCalledTimes(4); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'o' }, + type: 'ownerOf', + target: { kind: 'Component', namespace: 'default', name: 'n' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Component', namespace: 'default', name: 'n' }, + type: 'ownedBy', + target: { kind: 'Group', namespace: 'default', name: 'o' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'API', namespace: 'default', name: 'a' }, + type: 'apiProvidedBy', + target: { kind: 'Component', namespace: 'default', name: 'n' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Component', namespace: 'default', name: 'n' }, + type: 'providesApi', + target: { kind: 'API', namespace: 'default', name: 'a' }, + }, + }); + }); + + it('generates relations for api entities', async () => { + const entity: ApiEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'API', + metadata: { name: 'n' }, + spec: { + type: 'service', + owner: 'o', + lifecycle: 'l', + definition: 'd', + }, + }; + + await processor.postProcessEntity(entity, location, emit); + + expect(emit).toBeCalledTimes(2); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'o' }, + type: 'ownerOf', + target: { kind: 'API', namespace: 'default', name: 'n' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'API', namespace: 'default', name: 'n' }, + type: 'ownedBy', + target: { kind: 'Group', namespace: 'default', name: 'o' }, + }, + }); + }); + + it('generates relations for user entities', async () => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'n' }, + spec: { + memberOf: ['g'], + }, + }; + + await processor.postProcessEntity(entity, location, emit); + + expect(emit).toBeCalledTimes(2); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'User', namespace: 'default', name: 'n' }, + type: 'memberOf', + target: { kind: 'Group', namespace: 'default', name: 'g' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'g' }, + type: 'hasMember', + target: { kind: 'User', namespace: 'default', name: 'n' }, + }, + }); + }); + + it('generates relations for group entities', async () => { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { name: 'n' }, + spec: { + type: 't', + parent: 'p', + ancestors: [], + children: ['c'], + descendants: [], + }, + }; + + await processor.postProcessEntity(entity, location, emit); + + expect(emit).toBeCalledTimes(4); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'n' }, + type: 'childOf', + target: { kind: 'Group', namespace: 'default', name: 'p' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'p' }, + type: 'parentOf', + target: { kind: 'Group', namespace: 'default', name: 'n' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'c' }, + type: 'childOf', + target: { kind: 'Group', namespace: 'default', name: 'n' }, + }, + }); + expect(emit).toBeCalledWith({ + type: 'relation', + relation: { + source: { kind: 'Group', namespace: 'default', name: 'n' }, + type: 'parentOf', + target: { kind: 'Group', namespace: 'default', name: 'c' }, + }, + }); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts index 472dffeb16..a4e62e5b8c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts @@ -15,15 +15,31 @@ */ import { + ApiEntity, apiEntityV1alpha1Validator, + ComponentEntity, componentEntityV1alpha1Validator, Entity, + getEntityName, + GroupEntity, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, + LocationSpec, + parseEntityRef, + RELATION_API_PROVIDED_BY, + RELATION_CHILD_OF, + RELATION_HAS_MEMBER, + RELATION_MEMBER_OF, + RELATION_OWNED_BY, + RELATION_OWNER_OF, + RELATION_PARENT_OF, + RELATION_PROVIDES_API, templateEntityV1alpha1Validator, + UserEntity, userEntityV1alpha1Validator, } from '@backstage/catalog-model'; -import { CatalogProcessor } from './types'; +import * as result from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; export class BuiltinKindsEntityProcessor implements CatalogProcessor { private readonly validators = [ @@ -64,4 +80,114 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { return false; } + + async postProcessEntity( + entity: Entity, + _location: LocationSpec, + emit: CatalogProcessorEmit, + ): Promise { + const selfRef = getEntityName(entity); + + /* + * Utilities + */ + + function doEmit( + targets: string | string[] | undefined, + context: { defaultKind: string; defaultNamespace: string }, + outgoingRelation: string, + incomingRelation: string, + ): void { + if (!targets) { + return; + } + for (const target of [targets].flat()) { + const targetRef = parseEntityRef(target, context); + emit( + result.relation({ + source: selfRef, + type: outgoingRelation, + target: targetRef, + }), + ); + emit( + result.relation({ + source: targetRef, + type: incomingRelation, + target: selfRef, + }), + ); + } + } + + /* + * Emit relations for the Component kind + */ + + if (entity.kind === 'Component') { + const component = entity as ComponentEntity; + doEmit( + component.spec.owner, + { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, + RELATION_OWNED_BY, + RELATION_OWNER_OF, + ); + doEmit( + component.spec.implementsApis, + { defaultKind: 'API', defaultNamespace: selfRef.namespace }, + RELATION_PROVIDES_API, + RELATION_API_PROVIDED_BY, + ); + } + + /* + * Emit relations for the API kind + */ + + if (entity.kind === 'API') { + const api = entity as ApiEntity; + doEmit( + api.spec.owner, + { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, + RELATION_OWNED_BY, + RELATION_OWNER_OF, + ); + } + + /* + * Emit relations for the User kind + */ + + if (entity.kind === 'User') { + const user = entity as UserEntity; + doEmit( + user.spec.memberOf, + { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, + RELATION_MEMBER_OF, + RELATION_HAS_MEMBER, + ); + } + + /* + * Emit relations for the Group kind + */ + + if (entity.kind === 'Group') { + const group = entity as GroupEntity; + doEmit( + group.spec.parent, + { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, + RELATION_CHILD_OF, + RELATION_PARENT_OF, + ); + doEmit( + group.spec.children, + { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, + RELATION_PARENT_OF, + RELATION_CHILD_OF, + ); + } + + return entity; + } } diff --git a/plugins/catalog-backend/src/ingestion/processors/OwnerRelationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/OwnerRelationProcessor.ts deleted file mode 100644 index 4c67589789..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/OwnerRelationProcessor.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Entity, - ENTITY_DEFAULT_NAMESPACE, - LocationSpec, - parseEntityRef, - ApiEntityV1alpha1, - ComponentEntityV1alpha1, - RELATION_OWNED_BY, - RELATION_OWNER_OF, - getEntityName, -} from '@backstage/catalog-model'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import * as result from './results'; - -const includedKinds = new Set(['api', 'component']); - -export class OwnerRelationProcessor implements CatalogProcessor { - async postProcessEntity( - entity: Entity, - _location: LocationSpec, - emit: CatalogProcessorEmit, - ): Promise { - if (!includedKinds.has(entity.kind.toLowerCase())) { - return entity; - } - const apiOrComponentEntity = entity as - | ApiEntityV1alpha1 - | ComponentEntityV1alpha1; - - const owner = apiOrComponentEntity.spec?.owner; - if (owner) { - const namespace = entity.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE; - - const selfRef = getEntityName(entity); - const ownerRef = parseEntityRef(owner, { - defaultKind: 'group', - defaultNamespace: namespace, - }); - - emit( - result.relation({ - source: selfRef, - type: RELATION_OWNED_BY, - target: ownerRef, - }), - ); - emit( - result.relation({ - source: ownerRef, - type: RELATION_OWNER_OF, - target: selfRef, - }), - ); - } - - return entity; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 118f977125..49d4e03b64 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -22,10 +22,11 @@ export * from './types'; export { parseEntityYaml } from './util/parse'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; +export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -export { OwnerRelationProcessor } from './OwnerRelationProcessor'; +export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; export { LocationRefProcessor } from './LocationEntityProcessor'; export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index ee7c6f2246..58897a10a7 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -144,7 +144,7 @@ describe('CatalogBuilder', () => { owner: 'o', lifecycle: 'l', }, - relations: [], + relations: expect.anything(), }, ]); }); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 3f95dcd0d3..4be0f1799b 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -37,15 +37,16 @@ import { import { DatabaseManager } from '../database'; import { AnnotateLocationEntityProcessor, + BuiltinKindsEntityProcessor, CatalogProcessor, CodeOwnersProcessor, FileReaderProcessor, GithubOrgReaderProcessor, HigherOrderOperation, HigherOrderOperations, + LdapOrgReaderProcessor, LocationReaders, LocationRefProcessor, - OwnerRelationProcessor, MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, @@ -53,8 +54,6 @@ import { UrlReaderProcessor, } from '../ingestion'; import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; -import { BuiltinKindsEntityProcessor } from '../ingestion/processors/BuiltinKindsEntityProcessor'; -import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor'; import { jsonPlaceholderResolver, textPlaceholderResolver, @@ -283,7 +282,6 @@ export class CatalogBuilder { new UrlReaderProcessor({ reader, logger }), new CodeOwnersProcessor({ reader }), new LocationRefProcessor(), - new OwnerRelationProcessor(), new AnnotateLocationEntityProcessor(), ); }