feat(catalog-backend): implement github org entity ingestion

app-config.yaml

@@ -69,7 +69,7 @@ kubernetes:
 
 catalog:
   rules:
-    - allow: [Component, API, Group, Template, Location]
+    - allow: [Component, API, Group, User, Template, Location]
   processors:
     github:
       providers:
@@ -89,6 +89,18 @@ catalog:
         #   token:
         #     $secret:
         #       env: GHE_PRIVATE_TOKEN
+    githubOrg:
+      providers:
+        - target: https://github.com
+          token:
+            $secret:
+              env: GITHUB_PRIVATE_TOKEN
+        #### Example for how to add your GitHub Enterprise instance using the API:
+        # - target: https://ghe.example.net
+        #   apiBaseUrl: https://ghe.example.net/api/v3
+        #   token:
+        #     $secret:
+        #       env: GHE_PRIVATE_TOKEN
     bitbucketApi:
       username:
         $secret:
@@ -109,19 +121,15 @@ catalog:
     # Backstage example components
     - type: github
       target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml
-
     # Example component for github-actions
     - type: github
       target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml
-
     # Example component for techdocs
     - type: github
       target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml
-
     # Backstage example APIs
     - type: github
       target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml
-
     # Backstage example templates
     - type: github
       target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml

packages/backend/src/plugins/catalog.ts

@@ -45,7 +45,7 @@ export default async function createPlugin({
 
   useHotCleanup(
     module,
-    runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000),
+    runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
   );
 
   return await createRouter({

packages/catalog-model/src/EntityPolicies.ts

@@ -40,7 +40,10 @@ class AllEntityPolicies implements EntityPolicy {
   async enforce(entity: Entity): Promise<Entity> {
     let result = entity;
     for (const policy of this.policies) {
-      result = await policy.enforce(entity);
+      const output = await policy.enforce(entity);
+      if (output) {
+        result = output;
+      }
     }
     return result;
   }
@@ -51,12 +54,11 @@ class AllEntityPolicies implements EntityPolicy {
 class AnyEntityPolicy implements EntityPolicy {
   constructor(private readonly policies: EntityPolicy[]) {}
 
-  async enforce(entity: Entity): Promise<Entity> {
+  async enforce(entity: Entity): Promise<Entity | undefined> {
     for (const policy of this.policies) {
-      try {
-        return await policy.enforce(entity);
-      } catch {
-        continue;
+      const output = await policy.enforce(entity);
+      if (output !== null) {
+        return output;
       }
     }
     throw new Error(`The entity did not match any known policy`);
@@ -98,7 +100,7 @@ export class EntityPolicies implements EntityPolicy {
     this.policy = policy;
   }
 
-  enforce(entity: Entity): Promise<Entity> {
+  enforce(entity: Entity): Promise<Entity | undefined> {
     return this.policy.enforce(entity);
   }
 }

packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts

@@ -49,7 +49,7 @@ paths:
         '200':
           description: A paged array of pets
           content:
-            application/json:    
+            application/json:
               schema:
                 $ref: "#/components/schemas/Pets"
 components:
@@ -86,14 +86,14 @@ components:
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts

@@ -50,7 +50,13 @@ export class ApiEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts

@@ -50,14 +50,14 @@ describe('ComponentV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts

@@ -50,7 +50,13 @@ export class ComponentEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts

@@ -53,14 +53,14 @@ describe('GroupV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {
@@ -98,6 +98,11 @@ describe('GroupV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
+  it('accepts no ancestors', async () => {
+    (entity as any).spec.ancestors = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
+
   it('rejects missing children', async () => {
     delete (entity as any).spec.children;
     await expect(policy.enforce(entity)).rejects.toThrow(/children/);
@@ -108,6 +113,11 @@ describe('GroupV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
+  it('accepts no children', async () => {
+    (entity as any).spec.children = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
+
   it('rejects missing descendants', async () => {
     delete (entity as any).spec.descendants;
     await expect(policy.enforce(entity)).rejects.toThrow(/descendants/);
@@ -117,4 +127,9 @@ describe('GroupV1alpha1Policy', () => {
     (entity as any).spec.descendants = [''];
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
+
+  it('accepts no descendants', async () => {
+    (entity as any).spec.descendants = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
 });

packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts

@@ -44,15 +44,35 @@ export class GroupEntityV1alpha1Policy implements EntityPolicy {
         .object({
           type: yup.string().required().min(1),
           parent: yup.string().notRequired().min(1),
-          ancestors: yup.array(yup.string()).required(),
-          children: yup.array(yup.string()).required(),
-          descendants: yup.array(yup.string()).required(),
+          // Use these manual tests because yup .required() requires at least
+          // one element and there is no simple workaround -_-
+          ancestors: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'ancestors must be defined',
+            test: v => Boolean(v),
+          }),
+          children: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'children must be defined',
+            test: v => Boolean(v),
+          }),
+          descendants: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'descendants must be defined',
+            test: v => Boolean(v),
+          }),
         })
         .required(),
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts

@@ -47,14 +47,14 @@ describe('LocationV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts

@@ -48,7 +48,13 @@ export class LocationEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts

@@ -64,14 +64,14 @@ describe('TemplateEntityV1alpah1', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('rejects missing type', async () => {

packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts

@@ -50,7 +50,13 @@ export class TemplateEntityV1alpha1Policy implements EntityPolicy {
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts

@@ -54,14 +54,14 @@ describe('UserV1alpha1Policy', () => {
     await expect(policy.enforce(entity)).resolves.toBe(entity);
   });
 
-  it('rejects unknown apiVersion', async () => {
+  it('ignores unknown apiVersion', async () => {
     (entity as any).apiVersion = 'backstage.io/v1beta0';
-    await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
-  it('rejects unknown kind', async () => {
+  it('ignores unknown kind', async () => {
     (entity as any).kind = 'Wizard';
-    await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
+    await expect(policy.enforce(entity)).resolves.toBeUndefined();
   });
 
   it('spec accepts unknown additional fields', async () => {
@@ -147,4 +147,14 @@ describe('UserV1alpha1Policy', () => {
     (entity as any).spec.memberOf[0] = 7;
     await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/);
   });
+
+  it('accepts empty memberOf', async () => {
+    (entity as any).spec.memberOf = [];
+    await expect(policy.enforce(entity)).resolves.toBe(entity);
+  });
+
+  it('rejects null memberOf', async () => {
+    (entity as any).spec.memberOf = null;
+    await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/);
+  });
 });

packages/catalog-model/src/kinds/UserEntityV1alpha1.ts

@@ -50,13 +50,25 @@ export class UserEntityV1alpha1Policy implements EntityPolicy {
               picture: yup.string().min(1).notRequired(),
             })
             .notRequired(),
-          memberOf: yup.array(yup.string()).required(),
+          // Use this manual test because yup .required() requires at least one
+          // element and there is no simple workaround -_-
+          memberOf: yup.array(yup.string()).test({
+            name: 'isDefined',
+            message: 'memberOf must be defined',
+            test: v => Boolean(v),
+          }),
         })
         .required(),
     });
   }
 
-  async enforce(envelope: Entity): Promise<Entity> {
+  async enforce(envelope: Entity): Promise<Entity | undefined> {
+    if (
+      KIND !== envelope.kind ||
+      !API_VERSION.includes(envelope.apiVersion as any)
+    ) {
+      return undefined;
+    }
     return await this.schema.validate(envelope, { strict: true });
   }
 }

packages/catalog-model/src/types.ts

@@ -27,10 +27,11 @@ export type EntityPolicy = {
    * Applies validation or mutation on an entity.
    *
    * @param entity The entity, as validated/mutated so far in the policy tree
-   * @returns The incoming entity, or a mutated version of the same
+   * @returns The incoming entity, or a mutated version of the same, or
+   *          undefined if this processor could not handle the entity
    * @throws An error if the entity should be rejected
    */
-  enforce(entity: Entity): Promise<Entity>;
+  enforce(entity: Entity): Promise<Entity | undefined>;
 };
 
 export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue };

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -66,7 +66,7 @@ scaffolder:
 
 catalog:
   rules:
-    - allow: [Component, API, Group, Template, Location]
+    - allow: [Component, API, Group, User, Template, Location]
   processors:
     github:
       providers:

plugins/catalog-backend/package.json

@@ -23,6 +23,7 @@
     "@backstage/backend-common": "^0.1.1-alpha.23",
     "@backstage/catalog-model": "^0.1.1-alpha.23",
     "@backstage/config": "^0.1.1-alpha.23",
+    "@octokit/graphql": "^4.5.6",
     "@types/express": "^4.17.6",
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -19,6 +19,7 @@ import {
   Entity,
   EntityPolicies,
   EntityPolicy,
+  ENTITY_DEFAULT_NAMESPACE,
   LocationSpec,
 } from '@backstage/catalog-model';
 import { Config, ConfigReader } from '@backstage/config';
@@ -30,6 +31,7 @@ import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
 import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
 import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
 import { FileReaderProcessor } from './processors/FileReaderProcessor';
+import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
 import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
 import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
@@ -85,6 +87,7 @@ export class LocationReaders implements LocationReader {
       new GitlabReaderProcessor(),
       new BitbucketApiReaderProcessor(config),
       new AzureApiReaderProcessor(config),
+      GithubOrgReaderProcessor.fromConfig(config),
       new UrlReaderProcessor(),
       new YamlProcessor(),
       PlaceholderProcessor.default(),
@@ -229,8 +232,15 @@ export class LocationReaders implements LocationReader {
             this.readLocation.bind(this),
           );
         } catch (e) {
-          const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`;
+          const message = `Processor ${
+            processor.constructor.name
+          } threw an error while processing entity ${current.kind}:${
+            current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE
+          }/${current.metadata.name} at ${item.location.type} ${
+            item.location.target
+          }, ${e}`;
           emit(result.generalError(item.location, message));
+          this.logger.warn(message);
         }
       }
     }

plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts

@@ -25,6 +25,10 @@ export class EntityPolicyProcessor implements LocationProcessor {
   }
 
   async processEntity(entity: Entity): Promise<Entity> {
-    return await this.policy.enforce(entity);
+    const output = await this.policy.enforce(entity);
+    if (!output) {
+      throw new Error(`No policy applied to entity`);
+    }
+    return output;
   }
 }

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts

@@ -0,0 +1,170 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { graphql } from '@octokit/graphql';
+import * as results from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+import { getOrganizationTeams, getOrganizationUsers } from './util/github';
+import { buildOrgHierarchy } from './util/org';
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ */
+export class GithubOrgReaderProcessor implements LocationProcessor {
+  static fromConfig(config: Config) {
+    return new GithubOrgReaderProcessor(readConfig(config));
+  }
+
+  constructor(private readonly providers: ProviderConfig[]) {}
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(`${p.target}/`),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`,
+      );
+    }
+
+    const { org } = parseUrl(location.target);
+    const client = !provider.token
+      ? graphql
+      : graphql.defaults({
+          headers: {
+            authorization: `token ${provider.token}`,
+          },
+        });
+
+    const { users } = await getOrganizationUsers(client, org);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(
+      client,
+      org,
+    );
+    buildOrgHierarchy(groups, users, groupMemberUsers);
+
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}
+
+/*
+ * Helpers
+ */
+
+/**
+ * The configuration parameters for a single GitHub API provider.
+ */
+type ProviderConfig = {
+  /**
+   * The prefix of the target that this matches on, e.g. "https://github.com",
+   * with no trailing slash.
+   */
+  target: string;
+
+  /**
+   * The base URL of the API of this provider, e.g. "https://api.github.com",
+   * with no trailing slash.
+   *
+   * May be omitted specifically for GitHub; then it will be deduced.
+   */
+  apiBaseUrl?: string;
+
+  /**
+   * The authorization token to use for requests to this provider.
+   *
+   * If no token is specified, anonymous access is used.
+   */
+  token?: string;
+};
+
+// TODO(freben): Break out common code and config from here and GithubReaderProcessor
+export function readConfig(config: Config): ProviderConfig[] {
+  const providers: ProviderConfig[] = [];
+
+  const providerConfigs =
+    config.getOptionalConfigArray('catalog.processors.githubOrg.providers') ??
+    [];
+
+  // First read all the explicit providers
+  for (const providerConfig of providerConfigs) {
+    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
+    const token = providerConfig.getOptionalString('token');
+
+    if (apiBaseUrl) {
+      apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
+    } else if (target === 'https://github.com') {
+      apiBaseUrl = 'https://api.github.com';
+    }
+
+    if (!apiBaseUrl) {
+      throw new Error(
+        `Provider at ${target} must configure an explicit apiBaseUrl`,
+      );
+    }
+
+    providers.push({ target, apiBaseUrl, token });
+  }
+
+  // If no explicit github.com provider was added, put one in the list as
+  // a convenience
+  if (!providers.some(p => p.target === 'https://github.com')) {
+    providers.push({
+      target: 'https://github.com',
+      apiBaseUrl: 'https://api.github.com',
+    });
+  }
+
+  return providers;
+}
+
+export function parseUrl(urlString: string): { org: string } {
+  const path = new URL(urlString).pathname.substr(1).split('/');
+
+  // /spotify
+  if (path.length === 1) {
+    return { org: path[0] };
+  }
+
+  // /orgs/spotify[/<teams or people>]
+  if (
+    path.length >= 2 &&
+    path.length <= 3 &&
+    path[0] === 'orgs' &&
+    [undefined, 'teams', 'people'].includes(path[2])
+  ) {
+    return { org: path[1] };
+  }
+
+  throw new Error(`Expected a URL pointing to /<org> or /orgs/<org>`);
+}

plugins/catalog-backend/src/ingestion/processors/util/github.ts

@@ -0,0 +1,285 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { graphql } from '@octokit/graphql';
+
+// Graphql types
+
+type QueryResponse = {
+  organization: Organization;
+};
+
+type Organization = {
+  membersWithRole: Connection<User>;
+  team: Team;
+  teams: Connection<Team>;
+};
+
+type PageInfo = {
+  hasNextPage: boolean;
+  endCursor?: string;
+};
+
+type User = {
+  login: string;
+  bio?: string;
+  avatarUrl?: string;
+  email?: string;
+  name?: string;
+};
+
+type Team = {
+  slug: string;
+  combinedSlug: string;
+  description?: string;
+  parentTeam?: Team;
+  members: Connection<User>;
+};
+
+type Connection<T> = {
+  pageInfo: PageInfo;
+  nodes: T[];
+};
+
+/**
+ * Gets all the users out of a GitHub organization.
+ *
+ * Note that the users will not have their memberships filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ */
+export async function getOrganizationUsers(
+  client: typeof graphql,
+  org: string,
+): Promise<{ users: UserEntity[] }> {
+  const users: UserEntity[] = [];
+
+  // There is no user -> teams edge, so we leave the memberships empty for
+  // now and let the team iteration handle it instead
+  let cursor: string | undefined = undefined;
+  const query = `
+      query users($org: String!, $cursor: String) {
+        organization(login: $org) {
+          membersWithRole(first: 100, after: $cursor) {
+            pageInfo { hasNextPage, endCursor }
+            nodes { avatarUrl, bio, email, login, name }
+          }
+        }
+      }`;
+
+  for (let i = 0; i < 100 /* just for sanity */; ++i) {
+    const response: QueryResponse = await client(query, {
+      org,
+      cursor,
+    });
+
+    const connection = response.organization?.membersWithRole;
+    if (!connection) {
+      throw new Error(`Found no organization named ${org}`);
+    }
+
+    for (const user of connection.nodes) {
+      const entity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: user.login,
+          annotations: {
+            'github.com/user-login': user.login,
+          },
+        },
+        spec: {
+          profile: {},
+          memberOf: [],
+        },
+      };
+
+      if (user.bio) entity.metadata.description = user.bio;
+      if (user.name) entity.spec.profile!.displayName = user.name;
+      if (user.email) entity.spec.profile!.email = user.email;
+      if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
+
+      users.push(entity);
+    }
+
+    const { hasNextPage, endCursor } = connection.pageInfo;
+    if (!hasNextPage) {
+      break;
+    } else {
+      cursor = endCursor;
+    }
+  }
+
+  return { users };
+}
+
+/**
+ * Gets all the teams out of a GitHub organization.
+ *
+ * Note that the teams will not have any relations apart from parent filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ */
+export async function getOrganizationTeams(
+  client: typeof graphql,
+  org: string,
+): Promise<{
+  groups: GroupEntity[];
+  groupMemberUsers: Map<string, string[]>;
+}> {
+  const groups: GroupEntity[] = [];
+  const groupMemberUsers = new Map<string, string[]>();
+
+  let cursor: string | undefined = undefined;
+  const query = `
+    query teams($org: String!, $cursor: String) {
+      organization(login: $org) {
+        teams(first: 100, after: $cursor) {
+          pageInfo { hasNextPage, endCursor }
+          nodes {
+            slug
+            combinedSlug
+            parentTeam { slug }
+            members(first: 100, membership: IMMEDIATE) {
+              pageInfo { hasNextPage, endCursor }
+              nodes { login }
+            }
+          }
+        }
+      }
+    }`;
+
+  for (let i = 0; i < 100 /* just for sanity */; ++i) {
+    const response: QueryResponse = await client(query, {
+      org,
+      cursor,
+    });
+
+    const connection = response.organization?.teams;
+    if (!connection) {
+      throw new Error(`Found no organization named ${org}`);
+    }
+
+    for (const team of connection.nodes) {
+      const entity: GroupEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: team.slug,
+          annotations: {
+            'github.com/team-slug': team.combinedSlug,
+          },
+        },
+        spec: {
+          type: 'team',
+          ancestors: [],
+          children: [],
+          descendants: [],
+        },
+      };
+
+      if (team.description) entity.metadata.description = team.description;
+      if (team.parentTeam) entity.spec.parent = team.parentTeam.slug;
+
+      groups.push(entity);
+
+      const memberNames: string[] = [];
+      groupMemberUsers.set(team.slug, memberNames);
+
+      if (!team.members.pageInfo.hasNextPage) {
+        // We got all the members in one go, run the fast path
+        for (const user of team.members.nodes) {
+          memberNames.push(user.login);
+        }
+      } else {
+        // There were more than a hundred immediate members - run the slow
+        // path of fetching them explicitly
+        const { members } = await getTeamMembers(client, org, team.slug);
+        for (const userLogin of members) {
+          memberNames.push(userLogin);
+        }
+      }
+    }
+
+    const { hasNextPage, endCursor } = connection.pageInfo;
+    if (!hasNextPage) {
+      break;
+    } else {
+      cursor = endCursor;
+    }
+  }
+
+  return { groups, groupMemberUsers };
+}
+
+/**
+ * Gets all the users out of a GitHub organization.
+ *
+ * Note that the users will not have their memberships filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ * @param teamSlug The slug of the team to read
+ */
+export async function getTeamMembers(
+  client: typeof graphql,
+  org: string,
+  teamSlug: string,
+): Promise<{ members: string[] }> {
+  const members: string[] = [];
+
+  let cursor: string | undefined = undefined;
+  const query = `
+    query members($org: String!, $teamSlug: String!, $cursor: String) {
+      organization(login: $org) {
+        team(slug: $teamSlug) {
+          members(first: 100, after: $cursor, membership: IMMEDIATE) {
+            pageInfo { hasNextPage, endCursor }
+            nodes { login }
+          }
+        }
+      }
+    }`;
+
+  for (let j = 0; j < 100 /* just for sanity */; ++j) {
+    const response: QueryResponse = await client(query, {
+      org,
+      teamSlug,
+      cursor,
+    });
+
+    const connection = response.organization?.team?.members;
+    if (!connection) {
+      throw new Error(`Found no team named ${teamSlug} in named ${org}`);
+    }
+
+    for (const user of connection.nodes) {
+      members.push(user.login);
+    }
+
+    const { hasNextPage, endCursor } = connection.pageInfo;
+    if (!hasNextPage) {
+      break;
+    } else {
+      cursor = endCursor;
+    }
+  }
+
+  return { members };
+}

plugins/catalog-backend/src/ingestion/processors/util/org.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+
+export function buildOrgHierarchy(
+  groups: GroupEntity[],
+  users: UserEntity[],
+  groupMemberUsers: Map<string, string[]>,
+) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+  const usersByName = new Map(users.map(u => [u.metadata.name, u]));
+
+  //
+  // Make sure that u.memberOf contain all g
+  //
+
+  for (const [groupName, userNames] of groupMemberUsers.entries()) {
+    for (const userName of userNames) {
+      const user = usersByName.get(userName);
+      if (user && !user.spec.memberOf.includes(groupName)) {
+        user.spec.memberOf.push(groupName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.parent.children contain g
+  //
+
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    const parentName = group.spec.parent;
+    if (parentName) {
+      const parent = groupsByName.get(parentName);
+      if (parent && !parent.spec.children.includes(selfName)) {
+        parent.spec.children.push(selfName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.descendants is complete
+  //
+
+  function visitDescendants(current: GroupEntity): string[] {
+    if (current.spec.descendants.length) {
+      return current.spec.descendants;
+    }
+
+    const accumulator = new Set<string>();
+    for (const childName of current.spec.children) {
+      accumulator.add(childName);
+      const child = groupsByName.get(childName);
+      if (child) {
+        for (const d of visitDescendants(child)) {
+          accumulator.add(d);
+        }
+      }
+    }
+
+    const descendants = Array.from(accumulator);
+    current.spec.descendants = descendants;
+    return descendants;
+  }
+
+  for (const group of groups) {
+    visitDescendants(group);
+  }
+
+  //
+  // Make sure that g.ancestors is complete
+  //
+
+  function visitAncestors(current: GroupEntity): string[] {
+    if (current.spec.ancestors.length) {
+      return current.spec.ancestors;
+    }
+
+    let ancestors: string[];
+    const parentName = current.spec.parent;
+    if (!parentName) {
+      ancestors = [];
+    } else {
+      const parent = groupsByName.get(parentName);
+      if (parent) {
+        ancestors = [parentName, ...visitAncestors(parent)];
+      } else {
+        ancestors = [parentName];
+      }
+    }
+
+    current.spec.ancestors = ancestors;
+    return ancestors;
+  }
+
+  for (const group of groups) {
+    visitAncestors(group);
+  }
+}

yarn.lock

@@ -3393,6 +3393,15 @@
     "@octokit/types" "^5.0.0"
     universal-user-agent "^5.0.0"
 
+"@octokit/graphql@^4.5.6":
+  version "4.5.6"
+  resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.6.tgz#708143ba15cf7c1879ed6188266e7f270be805d4"
+  integrity sha512-Rry+unqKTa3svswT2ZAuqenpLrzJd+JTv89LTeVa5UM/5OX8o4KTkPL7/1ABq4f/ZkELb0XEK/2IEoYwykcLXg==
+  dependencies:
+    "@octokit/request" "^5.3.0"
+    "@octokit/types" "^5.0.0"
+    universal-user-agent "^6.0.0"
+
 "@octokit/plugin-enterprise-rest@^6.0.1":
   version "6.0.1"
   resolved "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437"
@@ -22558,6 +22567,11 @@ universal-user-agent@^5.0.0:
   dependencies:
     os-name "^3.1.0"
 
+universal-user-agent@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee"
+  integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==
+
 universalify@^0.1.0:
   version "0.1.2"
   resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"