Merge pull request #2118 from spotify/rugvip/rules
catalog-backend: initial catalog ingestion rules implementation
This commit is contained in:
Marcus Eide
@@ -0,0 +1,57 @@
|
||||
---
|
||||
id: software-catalog-configuration
|
||||
title: Catalog Configuration
|
||||
---
|
||||
|
||||
## Static Location Configuration
|
||||
|
||||
To enable declarative catalog setups, it is possible to add locations to the
|
||||
catalog via [static configuration](../../conf/index.md). Locations are added to
|
||||
the catalog under the `catalog.locations` key, for example:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
locations:
|
||||
- type: github
|
||||
target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml
|
||||
```
|
||||
|
||||
The locations added through static configuration can not be removed through the
|
||||
catalog locations API. To remove the locations, you have to remove them from the
|
||||
configuration.
|
||||
|
||||
## Catalog Rules
|
||||
|
||||
By default the catalog will only allow ingestion of entities with the kind
|
||||
`Component` and `API`. In order to allow entities of other kinds to be added,
|
||||
you need to add rules to the catalog. Rules are added either in a separate
|
||||
`catalog.rules` key, or added to statically configured locations.
|
||||
|
||||
For example, given the following configuration:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
rules:
|
||||
- allow: [Component, API, System]
|
||||
|
||||
locations:
|
||||
- type: github
|
||||
target: https://github.com/org/example/blob/master/org-data.yaml
|
||||
allow: [Group]
|
||||
```
|
||||
|
||||
We are able to add entities of kind `Component`, `API`, or `System` from any
|
||||
location, and `Group` entities from the `org-data.yaml`, which will also be read
|
||||
as statically configured location.
|
||||
|
||||
Note that if the `catalog.rules` key is present it will replace the default
|
||||
value, meaning that you need to add rules for `Component` and `API` kinds if you
|
||||
want those to be allowed.
|
||||
|
||||
The following configuration will reject any kind of entities from being added to
|
||||
the catalog:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
rules: []
|
||||
```
|
||||
@@ -79,6 +79,22 @@ All software created through the
|
||||
[Backstage Software Templates](../software-templates/index.md) are automatically
|
||||
registered in the catalog.
|
||||
|
||||
### Static catalog configuration
|
||||
|
||||
In addition to manually registering components, it is also possible to register
|
||||
components though [static configuration](../../conf/index.md). For example, the
|
||||
above example can be added using the following configuration:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
locations:
|
||||
- type: github
|
||||
target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml
|
||||
```
|
||||
|
||||
More information about catalog configuration can be found
|
||||
[here](configuration.md).
|
||||
|
||||
### Updating component metadata
|
||||
|
||||
Teams owning the components are responsible for maintaining the metadata about
|
||||
|
||||
@@ -28,6 +28,7 @@ nav:
|
||||
- Overview: 'features/software-catalog/index.md'
|
||||
- System model: 'features/software-catalog/system-model.md'
|
||||
- YAML File Format: 'features/software-catalog/descriptor-format.md'
|
||||
- Configuration: 'features/software-catalog/configuration.md'
|
||||
- Extending the model: 'features/software-catalog/extending-the-model.md'
|
||||
- External integrations: 'features/software-catalog/external-integrations.md'
|
||||
- API: 'features/software-catalog/api.md'
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { LocationSpec, Entity } from '@backstage/catalog-model';
|
||||
import { CatalogRulesEnforcer } from './CatalogRules';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
|
||||
const entity = {
|
||||
user: {
|
||||
kind: 'User',
|
||||
} as Entity,
|
||||
group: {
|
||||
kind: 'Group',
|
||||
} as Entity,
|
||||
component: {
|
||||
kind: 'component',
|
||||
} as Entity,
|
||||
};
|
||||
|
||||
const location: Record<string, LocationSpec> = {
|
||||
x: {
|
||||
type: 'github',
|
||||
target: 'https://github.com/a/b/blob/master/x.yaml',
|
||||
},
|
||||
y: {
|
||||
type: 'github',
|
||||
target: 'https://github.com/a/b/blob/master/y.yaml',
|
||||
},
|
||||
z: {
|
||||
type: 'file',
|
||||
target: '/root/z.yaml',
|
||||
},
|
||||
};
|
||||
|
||||
describe('CatalogRulesEnforcer', () => {
|
||||
it('should deny by default', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
|
||||
});
|
||||
|
||||
it('should deny all', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([{ allow: [] }]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
|
||||
});
|
||||
|
||||
it('should allow all', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([
|
||||
{ allow: [{ kind: 'User' }, { kind: 'Group' }, { kind: 'Component' }] },
|
||||
]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
it('should deny groups', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([
|
||||
{ allow: [{ kind: 'User' }, { kind: 'Component' }] },
|
||||
]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
it('should deny groups from github', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([
|
||||
{ allow: [{ kind: 'User' }, { kind: 'Component' }] },
|
||||
{ allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] },
|
||||
]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
it('should allow groups from files', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([
|
||||
{ allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] },
|
||||
]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
|
||||
});
|
||||
|
||||
it('should not be sensitive to kind case', () => {
|
||||
const enforcer = new CatalogRulesEnforcer([
|
||||
{ allow: [{ kind: 'group' }] },
|
||||
{ allow: [{ kind: 'Component' }] },
|
||||
]);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
describe('fromConfig', () => {
|
||||
it('should allow components by default', () => {
|
||||
const enforcer = CatalogRulesEnforcer.fromConfig(new ConfigReader({}));
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
it('should deny all', () => {
|
||||
const enforcer = CatalogRulesEnforcer.fromConfig(
|
||||
new ConfigReader({ catalog: { rules: [] } }),
|
||||
);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
|
||||
});
|
||||
|
||||
it('should allow all', () => {
|
||||
const enforcer = CatalogRulesEnforcer.fromConfig(
|
||||
new ConfigReader({
|
||||
catalog: {
|
||||
rules: [{ allow: ['User', 'Group'] }, { allow: ['Component'] }],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
it('should deny groups', () => {
|
||||
const enforcer = CatalogRulesEnforcer.fromConfig(
|
||||
new ConfigReader({
|
||||
catalog: { rules: [{ allow: ['User'] }, { allow: ['Component'] }] },
|
||||
}),
|
||||
);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
|
||||
});
|
||||
|
||||
it('should allow groups from a specific github location', () => {
|
||||
const enforcer = CatalogRulesEnforcer.fromConfig(
|
||||
new ConfigReader({
|
||||
catalog: {
|
||||
rules: [{ allow: ['user'] }],
|
||||
locations: [
|
||||
{
|
||||
type: 'github',
|
||||
target: 'https://github.com/a/b/blob/master/x.yaml',
|
||||
allow: ['Group'],
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
|
||||
});
|
||||
|
||||
it('should not care about location configuration in catalog.rules', () => {
|
||||
const enforcer = CatalogRulesEnforcer.fromConfig(
|
||||
new ConfigReader({
|
||||
catalog: {
|
||||
rules: [{ allow: ['Group'], locations: [{ type: 'github' }] }],
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
|
||||
expect(enforcer.isAllowed(entity.group, location.x)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.y)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.group, location.z)).toBe(true);
|
||||
expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Config } from '@backstage/config';
|
||||
import { LocationSpec, Entity } from '@backstage/catalog-model';
|
||||
|
||||
/**
|
||||
* A structure for matching entities to a given rule.
|
||||
*/
|
||||
type EntityMatcher = {
|
||||
kind: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A structure for matching locations to a given rule.
|
||||
*/
|
||||
type LocationMatcher = {
|
||||
target?: string;
|
||||
type: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Rules to apply to catalog entities
|
||||
*
|
||||
* An undefined list of matchers means match all, an empty list of matchers means match none
|
||||
*/
|
||||
type CatalogRule = {
|
||||
allow: EntityMatcher[];
|
||||
locations?: LocationMatcher[];
|
||||
};
|
||||
|
||||
export class CatalogRulesEnforcer {
|
||||
/**
|
||||
* Default rules used by the catalog.
|
||||
*
|
||||
* Denies any location from specifying user or group entities.
|
||||
*/
|
||||
static readonly defaultRules: CatalogRule[] = [
|
||||
{
|
||||
allow: [{ kind: 'Component' }, { kind: 'API' }],
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Loads catalog rules from config.
|
||||
*
|
||||
* This reads `catalog.rules` and defaults to the default rules if no value is present.
|
||||
* The value of the config should be a list of config objects, each with a single `allow`
|
||||
* field which in turn is a list of entity kinds to allow.
|
||||
*
|
||||
* If there is no matching rule to allow an ingested entity, it will be rejected by the catalog.
|
||||
*
|
||||
* It also reads in rules from `catalog.locations`, where each location can have a list
|
||||
* of allowed entity for the location, specified in an `allow` field.
|
||||
*
|
||||
* For example:
|
||||
*
|
||||
* ```yaml
|
||||
* catalog:
|
||||
* rules:
|
||||
* - allow: [Component, API]
|
||||
*
|
||||
* locations:
|
||||
* - type: github
|
||||
* target: https://github.com/org/repo/blob/master/users.yaml
|
||||
* allow: [User, Group]
|
||||
* - type: github
|
||||
* target: https://github.com/org/repo/blob/master/systems.yaml
|
||||
* allow: [System]
|
||||
* ```
|
||||
*/
|
||||
static fromConfig(config: Config) {
|
||||
const rules = new Array<CatalogRule>();
|
||||
|
||||
if (config.has('catalog.rules')) {
|
||||
const globalRules = config.getConfigArray('catalog.rules').map(sub => ({
|
||||
allow: sub.getStringArray('allow').map(kind => ({ kind })),
|
||||
}));
|
||||
rules.push(...globalRules);
|
||||
} else {
|
||||
rules.push(...CatalogRulesEnforcer.defaultRules);
|
||||
}
|
||||
|
||||
if (config.has('catalog.locations')) {
|
||||
const locationRules = config
|
||||
.getConfigArray('catalog.locations')
|
||||
.flatMap(sub => {
|
||||
if (!sub.has('allow')) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
allow: sub.getStringArray('allow').map(kind => ({ kind })),
|
||||
locations: [
|
||||
{
|
||||
type: sub.getString('type'),
|
||||
target: sub.getString('target'),
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
rules.push(...locationRules);
|
||||
}
|
||||
|
||||
return new CatalogRulesEnforcer(rules);
|
||||
}
|
||||
|
||||
constructor(private readonly rules: CatalogRule[]) {}
|
||||
|
||||
/**
|
||||
* Checks wether a specific entity/location combination is allowed
|
||||
* according to the configured rules.
|
||||
*/
|
||||
isAllowed(entity: Entity, location: LocationSpec) {
|
||||
for (const rule of this.rules) {
|
||||
if (!this.matchLocation(location, rule.locations)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (this.matchEntity(entity, rule.allow)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private matchLocation(
|
||||
location: LocationSpec,
|
||||
matchers?: LocationMatcher[],
|
||||
): boolean {
|
||||
if (!matchers) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (const matcher of matchers) {
|
||||
if (matcher.type !== location.type) {
|
||||
continue;
|
||||
}
|
||||
if (matcher.target && matcher.target !== location.target) {
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private matchEntity(entity: Entity, matchers?: EntityMatcher[]): boolean {
|
||||
if (!matchers) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (const matcher of matchers) {
|
||||
if (entity.kind.toLowerCase() !== matcher.kind.toLowerCase()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
} from './processors/types';
|
||||
import { YamlProcessor } from './processors/YamlProcessor';
|
||||
import { LocationReader, ReadLocationResult } from './types';
|
||||
import { CatalogRulesEnforcer } from './CatalogRules';
|
||||
|
||||
// The max amount of nesting depth of generated work items
|
||||
const MAX_DEPTH = 10;
|
||||
@@ -63,6 +64,7 @@ type Options = {
|
||||
export class LocationReaders implements LocationReader {
|
||||
private readonly logger: Logger;
|
||||
private readonly processors: LocationProcessor[];
|
||||
private readonly rulesEnforcer: CatalogRulesEnforcer;
|
||||
|
||||
static defaultProcessors(options: {
|
||||
config?: Config;
|
||||
@@ -96,6 +98,9 @@ export class LocationReaders implements LocationReader {
|
||||
}: Options) {
|
||||
this.logger = logger;
|
||||
this.processors = processors;
|
||||
this.rulesEnforcer = config
|
||||
? CatalogRulesEnforcer.fromConfig(config)
|
||||
: new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules);
|
||||
}
|
||||
|
||||
async read(location: LocationSpec): Promise<ReadLocationResult> {
|
||||
@@ -112,11 +117,20 @@ export class LocationReaders implements LocationReader {
|
||||
} else if (item.type === 'data') {
|
||||
await this.handleData(item, emit);
|
||||
} else if (item.type === 'entity') {
|
||||
const entity = await this.handleEntity(item, emit);
|
||||
output.entities.push({
|
||||
entity,
|
||||
location: item.location,
|
||||
});
|
||||
if (this.rulesEnforcer.isAllowed(item.entity, item.location)) {
|
||||
const entity = await this.handleEntity(item, emit);
|
||||
output.entities.push({
|
||||
entity,
|
||||
location: item.location,
|
||||
});
|
||||
} else {
|
||||
output.errors.push({
|
||||
location: item.location,
|
||||
error: new Error(
|
||||
`Entity of kind ${item.entity.kind} is not allowed from location ${item.location.target}:${item.location.type}`,
|
||||
),
|
||||
});
|
||||
}
|
||||
} else if (item.type === 'error') {
|
||||
await this.handleError(item, emit);
|
||||
output.errors.push({
|
||||
|
||||
Reference in New Issue
Block a user