catalog-backend: initial catalog ingestion rules implementation

2020-08-25 18:25:56 +02:00
parent 119e9001f1
commit 6f3b0b8c20
3 changed files with 320 additions and 5 deletions
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec, Entity } from '@backstage/catalog-model';
+import { CatalogRulesEnforcer } from './CatalogRules';
+
+const entity = {
+  user: {
+    kind: 'User',
+  } as Entity,
+  group: {
+    kind: 'Group',
+  } as Entity,
+  component: {
+    kind: 'component',
+  } as Entity,
+};
+
+const location: Record<string, LocationSpec> = {
+  x: {
+    type: 'github',
+    target: 'https://github.com/a/b/blob/master/x.yaml',
+  },
+  y: {
+    type: 'github',
+    target: 'https://github.com/a/b/blob/master/y.yaml',
+  },
+  z: {
+    type: 'file',
+    target: '/root/z.yaml',
+  },
+};
+
+describe('CatalogRulesEnforcer', () => {
+  it('should allow by default', () => {
+    const enforcer = new CatalogRulesEnforcer([]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(true);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
+  });
+
+  it('should deny all', () => {
+    const enforcer = new CatalogRulesEnforcer([{ allow: [] }]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
+  });
+
+  it('should allow all with override', () => {
+    const enforcer = new CatalogRulesEnforcer([{ allow: [] }, { deny: [] }]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(true);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
+  });
+
+  it('should deny groups', () => {
+    const enforcer = new CatalogRulesEnforcer([
+      { allow: [], deny: [{ kind: 'Group' }] },
+    ]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
+    expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.z)).toBe(false);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
+  });
+
+  it('should deny groups from github', () => {
+    const enforcer = new CatalogRulesEnforcer([
+      { allow: [], deny: [{ kind: 'Group' }], locations: [{ type: 'github' }] },
+    ]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
+    expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.z)).toBe(true);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
+  });
+
+  it('should override to allow groups from files', () => {
+    const enforcer = new CatalogRulesEnforcer([
+      { allow: [], deny: [{ kind: 'Group' }] },
+      { allow: [{ kind: 'Group' }], deny: [], locations: [{ type: 'file' }] },
+    ]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
+    expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.z)).toBe(true);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(true);
+  });
+
+  it('should not be sensitive to kind case', () => {
+    const enforcer = new CatalogRulesEnforcer([
+      { allow: [], deny: [{ kind: 'group' }] },
+      { allow: [], deny: [{ kind: 'Component' }] },
+    ]);
+    expect(enforcer.isAllowed(entity.user, location.x)).toBe(true);
+    expect(enforcer.isAllowed(entity.group, location.x)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.y)).toBe(false);
+    expect(enforcer.isAllowed(entity.group, location.z)).toBe(false);
+    expect(enforcer.isAllowed(entity.component, location.z)).toBe(false);
+  });
+});
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import { LocationSpec, Entity } from '@backstage/catalog-model';
+
+/**
+ * A structure for matching entities to a given rule.
+ */
+type EntityMatcher = {
+  kind: string;
+};
+
+/**
+ * A structure for matching locations to a given rule.
+ */
+type LocationMatcher = {
+  target?: string;
+  type: string;
+};
+
+/**
+ * Rules to apply to catalog entities
+ *
+ * An undefined list of matchers means match all, an empty list of matchers means match none
+ */
+type CatalogRule = {
+  deny?: EntityMatcher[];
+  allow?: EntityMatcher[];
+  locations?: LocationMatcher[];
+};
+
+export class CatalogRulesEnforcer {
+  /**
+   * Default rules used by the catalog.
+   *
+   * Denies any location from specifying user or group entities.
+   */
+  static readonly defaultRules: CatalogRule[] = [
+    {
+      deny: [{ kind: 'User' }, { kind: 'Group' }],
+      allow: [],
+    },
+  ];
+
+  /**
+   * Loads catalog rules from config.
+   *
+   * This reads `catalog.rules` and defaults to the default rules if no value is present.
+   * The value of the config should be a list of config objects, each with a single `deny`
+   * field which in turn is a list of entity kind to deny.
+   *
+   * It also reads in rules from `catalog.locations`, where each location can have a list
+   * of allowed entity for the location, specified in an `allow` field.
+   *
+   * For example:
+   *
+   * ```yaml
+   * catalog:
+   *   rules:
+   *   - deny: [User, Group, System]
+   *
+   *   locations:
+   *   - type: github
+   *     target: https://github.com/org/repo/blob/master/users.yaml
+   *     allow: [User, Group]
+   *   - type: github
+   *     target: https://github.com/org/repo/blob/master/systems.yaml
+   *     allow: [System]
+   * ```
+   */
+  static fromConfig(config: Config) {
+    const rules = new Array<CatalogRule>();
+
+    if (config.has('catalog.rules')) {
+      const globalRules = config.getConfigArray('catalog.rules').map(sub => ({
+        deny: sub.getStringArray('deny').map(kind => ({ kind })),
+        allow: [],
+      }));
+      rules.push(...globalRules);
+    } else {
+      rules.push(...CatalogRulesEnforcer.defaultRules);
+    }
+
+    if (config.has('catalog.locations')) {
+      const locationRules = config
+        .getConfigArray('catalog.locations')
+        .flatMap(sub => {
+          if (!sub.has('allow')) {
+            return [];
+          }
+
+          return [
+            {
+              deny: [],
+              allow: sub.getStringArray('allow').map(kind => ({ kind })),
+              locations: [
+                {
+                  type: sub.getString('type'),
+                  target: sub.getString('target'),
+                },
+              ],
+            },
+          ];
+        });
+
+      rules.push(...locationRules);
+    }
+
+    return new CatalogRulesEnforcer(rules);
+  }
+
+  constructor(private readonly rules: CatalogRule[]) {}
+
+  /**
+   * Checks wether a specific entity/location combination is allowed
+   * according to the configured rules.
+   */
+  isAllowed(entity: Entity, location: LocationSpec) {
+    let result = true;
+
+    for (const rule of this.rules) {
+      if (!this.matchLocation(location, rule.locations)) {
+        continue;
+      }
+
+      if (this.matchEntity(entity, rule.allow)) {
+        result = true;
+      }
+      if (this.matchEntity(entity, rule.deny)) {
+        result = false;
+      }
+    }
+
+    return result;
+  }
+
+  private matchLocation(
+    location: LocationSpec,
+    matchers?: LocationMatcher[],
+  ): boolean {
+    if (!matchers) {
+      return true;
+    }
+
+    for (const matcher of matchers) {
+      if (matcher.type !== location.type) {
+        continue;
+      }
+      if (matcher.target && matcher.target !== location.target) {
+        continue;
+      }
+      return true;
+    }
+
+    return false;
+  }
+
+  private matchEntity(entity: Entity, matchers?: EntityMatcher[]): boolean {
+    if (!matchers) {
+      return true;
+    }
+
+    for (const matcher of matchers) {
+      if (entity.kind.toLowerCase() !== matcher.kind.toLowerCase()) {
+        continue;
+      }
+
+      return true;
+    }
+
+    return false;
+  }
+}
@@ -47,6 +47,7 @@ import {
 } from './processors/types';
 import { YamlProcessor } from './processors/YamlProcessor';
 import { LocationReader, ReadLocationResult } from './types';
+import { CatalogRulesEnforcer } from './CatalogRules';

 // The max amount of nesting depth of generated work items
 const MAX_DEPTH = 10;
@@ -63,6 +64,7 @@ type Options = {
 export class LocationReaders implements LocationReader {
  private readonly logger: Logger;
  private readonly processors: LocationProcessor[];
+  private readonly rulesEnforcer: CatalogRulesEnforcer;

  static defaultProcessors(options: {
    config?: Config;
@@ -96,6 +98,9 @@ export class LocationReaders implements LocationReader {
  }: Options) {
    this.logger = logger;
    this.processors = processors;
+    this.rulesEnforcer = config
+      ? CatalogRulesEnforcer.fromConfig(config)
+      : new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules);
  }

  async read(location: LocationSpec): Promise<ReadLocationResult> {
@@ -112,11 +117,20 @@ export class LocationReaders implements LocationReader {
        } else if (item.type === 'data') {
          await this.handleData(item, emit);
        } else if (item.type === 'entity') {
-          const entity = await this.handleEntity(item, emit);
-          output.entities.push({
-            entity,
-            location: item.location,
-          });
+          if (this.rulesEnforcer.isAllowed(item.entity, item.location)) {
+            const entity = await this.handleEntity(item, emit);
+            output.entities.push({
+              entity,
+              location: item.location,
+            });
+          } else {
+            output.errors.push({
+              location: item.location,
+              error: new Error(
+                `Entity of kind ${item.entity.kind} is not allowed from location ${item.location.target}:${item.location.type}`,
+              ),
+            });
+          }
        } else if (item.type === 'error') {
          await this.handleError(item, emit);
          output.errors.push({