feat(catalog-backend): implement github org entity ingestion

packages/backend/src/plugins/catalog.ts

@@ -45,7 +45,7 @@ export default async function createPlugin({
 
   useHotCleanup(
     module,
-    runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000),
+    runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
   );
 
   return await createRouter({

packages/catalog-model/src/EntityPolicies.ts

@@ -40,7 +40,10 @@ class AllEntityPolicies implements EntityPolicy {
   async enforce(entity: Entity): Promise<Entity> {
     let result = entity;
     for (const policy of this.policies) {
-      result = await policy.enforce(entity);
+      const output = await policy.enforce(entity);
+      if (output) {
+        result = output;
+      }
     }
     return result;
   }
@@ -51,12 +54,11 @@ class AllEntityPolicies implements EntityPolicy {
 class AnyEntityPolicy implements EntityPolicy {
   constructor(private readonly policies: EntityPolicy[]) {}
 
-  async enforce(entity: Entity): Promise<Entity> {
+  async enforce(entity: Entity): Promise<Entity | undefined> {
     for (const policy of this.policies) {
-      try {
-        return await policy.enforce(entity);
-      } catch {
-        continue;
+      const output = await policy.enforce(entity);
+      if (output !== null) {
+        return output;
       }
     }
     throw new Error(`The entity did not match any known policy`);
@@ -98,7 +100,7 @@ export class EntityPolicies implements EntityPolicy {
     this.policy = policy;
   }
 
-  enforce(entity: Entity): Promise<Entity> {
+  enforce(entity: Entity): Promise<Entity | undefined> {
     return this.policy.enforce(entity);
   }
 }

packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts

@@ -49,7 +49,7 @@ paths:
         '200':
           description: A paged array of pets
           content:
-            application/json:    
+            application/json:
               schema:
                 $ref: "#/components/schemas/Pets"
 components:
@@ -86,14 +86,14 @@ components:
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts

@@ -50,7 +50,13 @@ export class ApiEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts

@@ -50,14 +50,14 @@ describe('ComponentV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts

@@ -50,7 +50,13 @@ export class ComponentEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts

@@ -53,14 +53,14 @@ describe('GroupV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {
@@ -98,6 +98,11 @@ describe('GroupV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
+  it('accepts no ancestors', async () => {
+    (entity as any).spec.ancestors = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
+
   it('rejects missing children', async () => {
     delete (entity as any).spec.children;
     await expect(policy.enforce(entity)).rejects.toThrow(/children/);
@@ -108,6 +113,11 @@ describe('GroupV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
+  it('accepts no children', async () => {
+    (entity as any).spec.children = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
+
   it('rejects missing descendants', async () => {
     delete (entity as any).spec.descendants;
     await expect(policy.enforce(entity)).rejects.toThrow(/descendants/);
@@ -117,4 +127,9 @@ describe('GroupV1alpha1Policy', () => {
     (entity as any).spec.descendants = [''];
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
+
+  it('accepts no descendants', async () => {
+    (entity as any).spec.descendants = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
 });

packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts

@@ -44,15 +44,35 @@ export class GroupEntityV1alpha1Policy implements EntityPolicy {
         .object({
           type: yup.string().required().min(1),
           parent: yup.string().notRequired().min(1),
-          ancestors: yup.array(yup.string()).required(),
-          children: yup.array(yup.string()).required(),
-          descendants: yup.array(yup.string()).required(),
+          // Use these manual tests because yup .required() requires at least
+          // one element and there is no simple workaround -_-
+          ancestors: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'ancestors must be defined',
+            test: v => Boolean(v),
+          }),
+          children: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'children must be defined',
+            test: v => Boolean(v),
+          }),
+          descendants: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'descendants must be defined',
+            test: v => Boolean(v),
+          }),
         })
         .required(),
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts

@@ -47,14 +47,14 @@ describe('LocationV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts

@@ -48,7 +48,13 @@ export class LocationEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts

@@ -64,14 +64,14 @@ describe('TemplateEntityV1alpah1', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts

@@ -50,7 +50,13 @@ export class TemplateEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts

@@ -54,14 +54,14 @@ describe('UserV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('spec accepts unknown additional fields', async () => {
@@ -147,4 +147,14 @@ describe('UserV1alpha1Policy', () => {
     (entity as any).spec.memberOf[0] = 7;
     await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/);
   });
+
+  it('accepts empty memberOf', async () => {
+    (entity as any).spec.memberOf = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
+
+  it('rejects null memberOf', async () => {
+    (entity as any).spec.memberOf = null;
+    await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/);
+  });
 });

packages/catalog-model/src/kinds/UserEntityV1alpha1.ts

@@ -50,13 +50,25 @@ export class UserEntityV1alpha1Policy implements EntityPolicy {
               picture: yup.string().min(1).notRequired(),
             })
             .notRequired(),
-          memberOf: yup.array(yup.string()).required(),
+          // Use this manual test because yup .required() requires at least one
+          // element and there is no simple workaround -_-
+          memberOf: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'memberOf must be defined',
+            test: v => Boolean(v),
+          }),
         })
         .required(),
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/types.ts

@@ -27,10 +27,11 @@ export type EntityPolicy = {
    * Applies validation or mutation on an entity.
    *
    * @param entity The entity, as validated/mutated so far in the policy tree
-   * @returns The incoming entity, or a mutated version of the same
+   * @returns The incoming entity, or a mutated version of the same, or
+   *          undefined if this processor could not handle the entity
    * @throws An error if the entity should be rejected
    */
-  enforce(entity: Entity): Promise<Entity>;
+  enforce(entity: Entity): Promise<Entity | undefined>;
 };
 
 export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue };

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -66,7 +66,7 @@ scaffolder:
 
 catalog:
   rules:
-    - allow: [Component, API, Group, Template, Location]
+    - allow: [Component, API, Group, User, Template, Location]
   processors:
     github:
       providers: