diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md new file mode 100644 index 0000000000..d4cc7fc0c2 --- /dev/null +++ b/docs/features/software-catalog/configuration.md @@ -0,0 +1,57 @@ +--- +id: software-catalog-configuration +title: Catalog Configuration +--- + +## Static Location Configuration + +To enable declarative catalog setups, it is possible to add locations to the +catalog via [static configuration](../../conf/index.md). Locations are added to +the catalog under the `catalog.locations` key, for example: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +The locations added through static configuration can not be removed through the +catalog locations API. To remove the locations, you have to remove them from the +configuration. + +## Catalog Rules + +By default the catalog will only allow ingestion of entities with the kind +`Component` and `API`. In order to allow entities of other kinds to be added, +you need to add rules to the catalog. Rules are added either in a separate +`catalog.rules` key, or added to statically configured locations. + +For example, given the following configuration: + +```yaml +catalog: + rules: + - allow: [Component, API, System] + + locations: + - type: github + target: https://github.com/org/example/blob/master/org-data.yaml + allow: [Group] +``` + +We are able to add entities of kind `Component`, `API`, or `System` from any +location, and `Group` entities from the `org-data.yaml`, which will also be read +as statically configured location. + +Note that if the `catalog.rules` key is present it will replace the default +value, meaning that you need to add rules for `Component` and `API` kinds if you +want those to be allowed. + +The following configuration will reject any kind of entities from being added to +the catalog: + +```yaml +catalog: + rules: [] +``` diff --git a/docs/features/software-catalog/index.md b/docs/features/software-catalog/index.md index db8c183390..bfc3d53d4d 100644 --- a/docs/features/software-catalog/index.md +++ b/docs/features/software-catalog/index.md @@ -79,6 +79,22 @@ All software created through the [Backstage Software Templates](../software-templates/index.md) are automatically registered in the catalog. +### Static catalog configuration + +In addition to manually registering components, it is also possible to register +components though [static configuration](../../conf/index.md). For example, the +above example can be added using the following configuration: + +```yaml +catalog: + locations: + - type: github + target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml +``` + +More information about catalog configuration can be found +[here](configuration.md). + ### Updating component metadata Teams owning the components are responsible for maintaining the metadata about diff --git a/mkdocs.yml b/mkdocs.yml index 637e45c0f0..b12c8f81c7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - Overview: 'features/software-catalog/index.md' - System model: 'features/software-catalog/system-model.md' - YAML File Format: 'features/software-catalog/descriptor-format.md' + - Configuration: 'features/software-catalog/configuration.md' - Extending the model: 'features/software-catalog/extending-the-model.md' - External integrations: 'features/software-catalog/external-integrations.md' - API: 'features/software-catalog/api.md' diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts new file mode 100644 index 0000000000..34484f6b1e --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec, Entity } from '@backstage/catalog-model'; +import { CatalogRulesEnforcer } from './CatalogRules'; +import { ConfigReader } from '@backstage/config'; + +const entity = { + user: { + kind: 'User', + } as Entity, + group: { + kind: 'Group', + } as Entity, + component: { + kind: 'component', + } as Entity, +}; + +const location: Record = { + x: { + type: 'github', + target: 'https://github.com/a/b/blob/master/x.yaml', + }, + y: { + type: 'github', + target: 'https://github.com/a/b/blob/master/y.yaml', + }, + z: { + type: 'file', + target: '/root/z.yaml', + }, +}; + +describe('CatalogRulesEnforcer', () => { + it('should deny by default', () => { + const enforcer = new CatalogRulesEnforcer([]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should deny all', () => { + const enforcer = new CatalogRulesEnforcer([{ allow: [] }]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should allow all', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'User' }, { kind: 'Group' }, { kind: 'Component' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'User' }, { kind: 'Component' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups from github', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'User' }, { kind: 'Component' }] }, + { allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should allow groups from files', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'Group' }], locations: [{ type: 'file' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should not be sensitive to kind case', () => { + const enforcer = new CatalogRulesEnforcer([ + { allow: [{ kind: 'group' }] }, + { allow: [{ kind: 'Component' }] }, + ]); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + describe('fromConfig', () => { + it('should allow components by default', () => { + const enforcer = CatalogRulesEnforcer.fromConfig(new ConfigReader({})); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny all', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ catalog: { rules: [] } }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should allow all', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['User', 'Group'] }, { allow: ['Component'] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should deny groups', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { rules: [{ allow: ['User'] }, { allow: ['Component'] }] }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(true); + }); + + it('should allow groups from a specific github location', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['user'] }], + locations: [ + { + type: 'github', + target: 'https://github.com/a/b/blob/master/x.yaml', + allow: ['Group'], + }, + ], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + + it('should not care about location configuration in catalog.rules', () => { + const enforcer = CatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['Group'], locations: [{ type: 'github' }] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); + expect(enforcer.isAllowed(entity.group, location.x)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.y)).toBe(true); + expect(enforcer.isAllowed(entity.group, location.z)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts new file mode 100644 index 0000000000..eda93b552f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -0,0 +1,180 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { LocationSpec, Entity } from '@backstage/catalog-model'; + +/** + * A structure for matching entities to a given rule. + */ +type EntityMatcher = { + kind: string; +}; + +/** + * A structure for matching locations to a given rule. + */ +type LocationMatcher = { + target?: string; + type: string; +}; + +/** + * Rules to apply to catalog entities + * + * An undefined list of matchers means match all, an empty list of matchers means match none + */ +type CatalogRule = { + allow: EntityMatcher[]; + locations?: LocationMatcher[]; +}; + +export class CatalogRulesEnforcer { + /** + * Default rules used by the catalog. + * + * Denies any location from specifying user or group entities. + */ + static readonly defaultRules: CatalogRule[] = [ + { + allow: [{ kind: 'Component' }, { kind: 'API' }], + }, + ]; + + /** + * Loads catalog rules from config. + * + * This reads `catalog.rules` and defaults to the default rules if no value is present. + * The value of the config should be a list of config objects, each with a single `allow` + * field which in turn is a list of entity kinds to allow. + * + * If there is no matching rule to allow an ingested entity, it will be rejected by the catalog. + * + * It also reads in rules from `catalog.locations`, where each location can have a list + * of allowed entity for the location, specified in an `allow` field. + * + * For example: + * + * ```yaml + * catalog: + * rules: + * - allow: [Component, API] + * + * locations: + * - type: github + * target: https://github.com/org/repo/blob/master/users.yaml + * allow: [User, Group] + * - type: github + * target: https://github.com/org/repo/blob/master/systems.yaml + * allow: [System] + * ``` + */ + static fromConfig(config: Config) { + const rules = new Array(); + + if (config.has('catalog.rules')) { + const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ + allow: sub.getStringArray('allow').map(kind => ({ kind })), + })); + rules.push(...globalRules); + } else { + rules.push(...CatalogRulesEnforcer.defaultRules); + } + + if (config.has('catalog.locations')) { + const locationRules = config + .getConfigArray('catalog.locations') + .flatMap(sub => { + if (!sub.has('allow')) { + return []; + } + + return [ + { + allow: sub.getStringArray('allow').map(kind => ({ kind })), + locations: [ + { + type: sub.getString('type'), + target: sub.getString('target'), + }, + ], + }, + ]; + }); + + rules.push(...locationRules); + } + + return new CatalogRulesEnforcer(rules); + } + + constructor(private readonly rules: CatalogRule[]) {} + + /** + * Checks wether a specific entity/location combination is allowed + * according to the configured rules. + */ + isAllowed(entity: Entity, location: LocationSpec) { + for (const rule of this.rules) { + if (!this.matchLocation(location, rule.locations)) { + continue; + } + + if (this.matchEntity(entity, rule.allow)) { + return true; + } + } + + return false; + } + + private matchLocation( + location: LocationSpec, + matchers?: LocationMatcher[], + ): boolean { + if (!matchers) { + return true; + } + + for (const matcher of matchers) { + if (matcher.type !== location.type) { + continue; + } + if (matcher.target && matcher.target !== location.target) { + continue; + } + return true; + } + + return false; + } + + private matchEntity(entity: Entity, matchers?: EntityMatcher[]): boolean { + if (!matchers) { + return true; + } + + for (const matcher of matchers) { + if (entity.kind.toLowerCase() !== matcher.kind.toLowerCase()) { + continue; + } + + return true; + } + + return false; + } +} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 46fa6781f1..bddef6b4de 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -47,6 +47,7 @@ import { } from './processors/types'; import { YamlProcessor } from './processors/YamlProcessor'; import { LocationReader, ReadLocationResult } from './types'; +import { CatalogRulesEnforcer } from './CatalogRules'; // The max amount of nesting depth of generated work items const MAX_DEPTH = 10; @@ -63,6 +64,7 @@ type Options = { export class LocationReaders implements LocationReader { private readonly logger: Logger; private readonly processors: LocationProcessor[]; + private readonly rulesEnforcer: CatalogRulesEnforcer; static defaultProcessors(options: { config?: Config; @@ -96,6 +98,9 @@ export class LocationReaders implements LocationReader { }: Options) { this.logger = logger; this.processors = processors; + this.rulesEnforcer = config + ? CatalogRulesEnforcer.fromConfig(config) + : new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules); } async read(location: LocationSpec): Promise { @@ -112,11 +117,20 @@ export class LocationReaders implements LocationReader { } else if (item.type === 'data') { await this.handleData(item, emit); } else if (item.type === 'entity') { - const entity = await this.handleEntity(item, emit); - output.entities.push({ - entity, - location: item.location, - }); + if (this.rulesEnforcer.isAllowed(item.entity, item.location)) { + const entity = await this.handleEntity(item, emit); + output.entities.push({ + entity, + location: item.location, + }); + } else { + output.errors.push({ + location: item.location, + error: new Error( + `Entity of kind ${item.entity.kind} is not allowed from location ${item.location.target}:${item.location.type}`, + ), + }); + } } else if (item.type === 'error') { await this.handleError(item, emit); output.errors.push({