paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

catalog: emit all common relations based on specs (#3408)

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-11-25 10:51:22 +01:00
committed by GitHub
parent fae3fafcfa
commit 2daf18e809
12 changed files with 437 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/pink-goats-sort.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/catalog-model': patch
'@backstage/plugin-catalog-backend': patch
---
Start emitting all known relation types from the core entity kinds, based on their spec data.
packages/catalog-model/src/entity/ref.ts
+8 -8
View File
@@ -84,6 +84,14 @@ export function parseEntityName(
* @param context The context of defaults that the parsing happens within
* @returns The compound form of the reference
*/
export function parseEntityRef(
ref: EntityRef,
context?: { defaultKind: string; defaultNamespace: string },
): {
kind: string;
namespace: string;
name: string;
};
export function parseEntityRef(
ref: EntityRef,
context?: { defaultKind: string },
@@ -100,14 +108,6 @@ export function parseEntityRef(
namespace: string;
name: string;
};
export function parseEntityRef(
ref: EntityRef,
context?: { defaultKind: string; defaultNamespace: string },
): {
kind: string;
namespace: string;
name: string;
};
export function parseEntityRef(
ref: EntityRef,
context: EntityRefContext = {},
packages/catalog-model/src/kinds/relations.ts
+2
View File
@@ -33,7 +33,9 @@ export const RELATION_OWNER_OF = 'ownerOf';
* A relation with an API entity, typically from a component or system
*/
export const RELATION_CONSUMES_API = 'consumesApi';
export const RELATION_API_CONSUMED_BY = 'apiConsumedBy';
export const RELATION_PROVIDES_API = 'providesApi';
export const RELATION_API_PROVIDED_BY = 'apiProvidedBy';
/**
* A relation denoting a dependency on another entity.
plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js
+93
View File
@@ -0,0 +1,93 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// @ts-check
/**
* @param {import('knex')} knex
*/
exports.up = async function up(knex) {
if (knex.client.config.client === 'sqlite3') {
// sqlite doesn't support dropPrimary so we recreate it properly instead
await knex.schema.dropTable('entities_relations');
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.index('source_full_name', 'source_full_name_idx');
});
} else {
await knex.schema.alterTable('entities_relations', table => {
table.dropPrimary();
table.index('source_full_name', 'source_full_name_idx');
});
}
};
/**
* @param {import('knex')} knex
*/
exports.down = async function down(knex) {
if (knex.client.config.client === 'sqlite3') {
await knex.schema.dropTable('entities_relations');
await knex.schema.createTable('entities_relations', table => {
table.comment('All relations between entities in the catalog');
table
.uuid('originating_entity_id')
.references('id')
.inTable('entities')
.onDelete('CASCADE')
.notNullable()
.comment('The entity that provided the relation');
table
.string('source_full_name')
.notNullable()
.comment('The full name of the source entity of the relation');
table
.string('type')
.notNullable()
.comment('The type of the relation between the entities');
table
.string('target_full_name')
.notNullable()
.comment('The full name of the target entity of the relation');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
} else {
await knex.schema.alterTable('entities_relations', table => {
table.dropIndex([], 'source_full_name_idx');
table.primary(['source_full_name', 'type', 'target_full_name']);
});
}
};
plugins/catalog-backend/package.json
+1 -1
View File
@@ -26,6 +26,7 @@
"@backstage/config": "^0.1.1",
"@octokit/graphql": "^4.5.6",
"@types/express": "^4.17.6",
"@types/ldapjs": "^1.0.9",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
"cross-fetch": "^3.0.6",
@@ -51,7 +52,6 @@
"@backstage/test-utils": "^0.1.3",
"@types/core-js": "^2.5.4",
"@types/git-url-parse": "^9.0.0",
"@types/ldapjs": "^1.0.9",
"@types/lodash": "^4.14.151",
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
plugins/catalog-backend/src/database/CommonDatabase.ts
+15 -2
View File
@@ -345,7 +345,11 @@ export class CommonDatabase implements Database {
);
// TODO(blam): translate constraint failures to sane NotFoundError instead
await tx.batchInsert('entities_relations', relationsRows, BATCH_SIZE);
await tx.batchInsert(
'entities_relations',
deduplicateRelations(relationsRows),
BATCH_SIZE,
);
}
async addLocation(
@@ -506,7 +510,7 @@ export class CommonDatabase implements Database {
.orderBy(['type', 'target_full_name'])
.select();
entity.relations = relations.map(r => ({
entity.relations = deduplicateRelations(relations).map(r => ({
target: parseEntityName(r.target_full_name),
type: r.type,
}));
@@ -517,3 +521,12 @@ export class CommonDatabase implements Database {
};
}
}
function deduplicateRelations(
rows: DbEntitiesRelationsRow[],
): DbEntitiesRelationsRow[] {
return lodash.uniqBy(
rows,
r => `${r.source_full_name}:${r.target_full_name}:${r.type}`,
);
}
plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts
+180
View File
@@ -14,6 +14,12 @@
* limitations under the License.
*/
import {
ApiEntity,
ComponentEntity,
GroupEntity,
UserEntity,
} from '@backstage/catalog-model';
import { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
describe('BuiltinKindsEntityProcessor', () => {
@@ -44,4 +50,178 @@ describe('BuiltinKindsEntityProcessor', () => {
},
});
});
describe('postProcessEntity', () => {
const processor = new BuiltinKindsEntityProcessor();
const location = { type: 'a', target: 'b' };
const emit = jest.fn();
afterEach(() => jest.resetAllMocks());
it('generates relations for component entities', async () => {
const entity: ComponentEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: { name: 'n' },
spec: {
type: 'service',
owner: 'o',
lifecycle: 'l',
implementsApis: ['a'],
},
};
await processor.postProcessEntity(entity, location, emit);
expect(emit).toBeCalledTimes(4);
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'o' },
type: 'ownerOf',
target: { kind: 'Component', namespace: 'default', name: 'n' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Component', namespace: 'default', name: 'n' },
type: 'ownedBy',
target: { kind: 'Group', namespace: 'default', name: 'o' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'API', namespace: 'default', name: 'a' },
type: 'apiProvidedBy',
target: { kind: 'Component', namespace: 'default', name: 'n' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Component', namespace: 'default', name: 'n' },
type: 'providesApi',
target: { kind: 'API', namespace: 'default', name: 'a' },
},
});
});
it('generates relations for api entities', async () => {
const entity: ApiEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'API',
metadata: { name: 'n' },
spec: {
type: 'service',
owner: 'o',
lifecycle: 'l',
definition: 'd',
},
};
await processor.postProcessEntity(entity, location, emit);
expect(emit).toBeCalledTimes(2);
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'o' },
type: 'ownerOf',
target: { kind: 'API', namespace: 'default', name: 'n' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'API', namespace: 'default', name: 'n' },
type: 'ownedBy',
target: { kind: 'Group', namespace: 'default', name: 'o' },
},
});
});
it('generates relations for user entities', async () => {
const entity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name: 'n' },
spec: {
memberOf: ['g'],
},
};
await processor.postProcessEntity(entity, location, emit);
expect(emit).toBeCalledTimes(2);
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'User', namespace: 'default', name: 'n' },
type: 'memberOf',
target: { kind: 'Group', namespace: 'default', name: 'g' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'g' },
type: 'hasMember',
target: { kind: 'User', namespace: 'default', name: 'n' },
},
});
});
it('generates relations for group entities', async () => {
const entity: GroupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: { name: 'n' },
spec: {
type: 't',
parent: 'p',
ancestors: [],
children: ['c'],
descendants: [],
},
};
await processor.postProcessEntity(entity, location, emit);
expect(emit).toBeCalledTimes(4);
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'n' },
type: 'childOf',
target: { kind: 'Group', namespace: 'default', name: 'p' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'p' },
type: 'parentOf',
target: { kind: 'Group', namespace: 'default', name: 'n' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'c' },
type: 'childOf',
target: { kind: 'Group', namespace: 'default', name: 'n' },
},
});
expect(emit).toBeCalledWith({
type: 'relation',
relation: {
source: { kind: 'Group', namespace: 'default', name: 'n' },
type: 'parentOf',
target: { kind: 'Group', namespace: 'default', name: 'c' },
},
});
});
});
});
plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts
+127 -1
View File
@@ -15,15 +15,31 @@
*/
import {
ApiEntity,
apiEntityV1alpha1Validator,
ComponentEntity,
componentEntityV1alpha1Validator,
Entity,
getEntityName,
GroupEntity,
groupEntityV1alpha1Validator,
locationEntityV1alpha1Validator,
LocationSpec,
parseEntityRef,
RELATION_API_PROVIDED_BY,
RELATION_CHILD_OF,
RELATION_HAS_MEMBER,
RELATION_MEMBER_OF,
RELATION_OWNED_BY,
RELATION_OWNER_OF,
RELATION_PARENT_OF,
RELATION_PROVIDES_API,
templateEntityV1alpha1Validator,
UserEntity,
userEntityV1alpha1Validator,
} from '@backstage/catalog-model';
import { CatalogProcessor } from './types';
import * as result from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
export class BuiltinKindsEntityProcessor implements CatalogProcessor {
private readonly validators = [
@@ -64,4 +80,114 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor {
return false;
}
async postProcessEntity(
entity: Entity,
_location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<Entity> {
const selfRef = getEntityName(entity);
/*
* Utilities
*/
function doEmit(
targets: string | string[] | undefined,
context: { defaultKind: string; defaultNamespace: string },
outgoingRelation: string,
incomingRelation: string,
): void {
if (!targets) {
return;
}
for (const target of [targets].flat()) {
const targetRef = parseEntityRef(target, context);
emit(
result.relation({
source: selfRef,
type: outgoingRelation,
target: targetRef,
}),
);
emit(
result.relation({
source: targetRef,
type: incomingRelation,
target: selfRef,
}),
);
}
}
/*
* Emit relations for the Component kind
*/
if (entity.kind === 'Component') {
const component = entity as ComponentEntity;
doEmit(
component.spec.owner,
{ defaultKind: 'Group', defaultNamespace: selfRef.namespace },
RELATION_OWNED_BY,
RELATION_OWNER_OF,
);
doEmit(
component.spec.implementsApis,
{ defaultKind: 'API', defaultNamespace: selfRef.namespace },
RELATION_PROVIDES_API,
RELATION_API_PROVIDED_BY,
);
}
/*
* Emit relations for the API kind
*/
if (entity.kind === 'API') {
const api = entity as ApiEntity;
doEmit(
api.spec.owner,
{ defaultKind: 'Group', defaultNamespace: selfRef.namespace },
RELATION_OWNED_BY,
RELATION_OWNER_OF,
);
}
/*
* Emit relations for the User kind
*/
if (entity.kind === 'User') {
const user = entity as UserEntity;
doEmit(
user.spec.memberOf,
{ defaultKind: 'Group', defaultNamespace: selfRef.namespace },
RELATION_MEMBER_OF,
RELATION_HAS_MEMBER,
);
}
/*
* Emit relations for the Group kind
*/
if (entity.kind === 'Group') {
const group = entity as GroupEntity;
doEmit(
group.spec.parent,
{ defaultKind: 'Group', defaultNamespace: selfRef.namespace },
RELATION_CHILD_OF,
RELATION_PARENT_OF,
);
doEmit(
group.spec.children,
{ defaultKind: 'Group', defaultNamespace: selfRef.namespace },
RELATION_PARENT_OF,
RELATION_CHILD_OF,
);
}
return entity;
}
}
plugins/catalog-backend/src/ingestion/processors/OwnerRelationProcessor.ts
-74
View File
@@ -1,74 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
parseEntityRef,
ApiEntityV1alpha1,
ComponentEntityV1alpha1,
RELATION_OWNED_BY,
RELATION_OWNER_OF,
getEntityName,
} from '@backstage/catalog-model';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import * as result from './results';
const includedKinds = new Set(['api', 'component']);
export class OwnerRelationProcessor implements CatalogProcessor {
async postProcessEntity(
entity: Entity,
_location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<Entity> {
if (!includedKinds.has(entity.kind.toLowerCase())) {
return entity;
}
const apiOrComponentEntity = entity as
| ApiEntityV1alpha1
| ComponentEntityV1alpha1;
const owner = apiOrComponentEntity.spec?.owner;
if (owner) {
const namespace = entity.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const selfRef = getEntityName(entity);
const ownerRef = parseEntityRef(owner, {
defaultKind: 'group',
defaultNamespace: namespace,
});
emit(
result.relation({
source: selfRef,
type: RELATION_OWNED_BY,
target: ownerRef,
}),
);
emit(
result.relation({
source: ownerRef,
type: RELATION_OWNER_OF,
target: selfRef,
}),
);
}
return entity;
}
}
plugins/catalog-backend/src/ingestion/processors/index.ts
+2 -1
View File
@@ -22,10 +22,11 @@ export * from './types';
export { parseEntityYaml } from './util/parse';
export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
export { CodeOwnersProcessor } from './CodeOwnersProcessor';
export { FileReaderProcessor } from './FileReaderProcessor';
export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
export { OwnerRelationProcessor } from './OwnerRelationProcessor';
export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor';
export { LocationRefProcessor } from './LocationEntityProcessor';
export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';
export { PlaceholderProcessor } from './PlaceholderProcessor';
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+1 -1
View File
@@ -144,7 +144,7 @@ describe('CatalogBuilder', () => {
owner: 'o',
lifecycle: 'l',
},
relations: [],
relations: expect.anything(),
},
]);
});
plugins/catalog-backend/src/service/CatalogBuilder.ts
+2 -4
View File
@@ -37,15 +37,16 @@ import {
import { DatabaseManager } from '../database';
import {
AnnotateLocationEntityProcessor,
BuiltinKindsEntityProcessor,
CatalogProcessor,
CodeOwnersProcessor,
FileReaderProcessor,
GithubOrgReaderProcessor,
HigherOrderOperation,
HigherOrderOperations,
LdapOrgReaderProcessor,
LocationReaders,
LocationRefProcessor,
OwnerRelationProcessor,
MicrosoftGraphOrgReaderProcessor,
PlaceholderProcessor,
PlaceholderResolver,
@@ -53,8 +54,6 @@ import {
UrlReaderProcessor,
} from '../ingestion';
import { CatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { BuiltinKindsEntityProcessor } from '../ingestion/processors/BuiltinKindsEntityProcessor';
import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor';
import {
jsonPlaceholderResolver,
textPlaceholderResolver,
@@ -283,7 +282,6 @@ export class CatalogBuilder {
new UrlReaderProcessor({ reader, logger }),
new CodeOwnersProcessor({ reader }),
new LocationRefProcessor(),
new OwnerRelationProcessor(),
new AnnotateLocationEntityProcessor(),
);
}