feat(catalog-backend): implement github org entity ingestion

plugins/catalog-backend/package.json

@@ -23,6 +23,7 @@
     "@backstage/backend-common": "^0.1.1-alpha.23",
     "@backstage/catalog-model": "^0.1.1-alpha.23",
     "@backstage/config": "^0.1.1-alpha.23",
+    "@octokit/graphql": "^4.5.6",
     "@types/express": "^4.17.6",
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -19,6 +19,7 @@ import {
   Entity,
   EntityPolicies,
   EntityPolicy,
+  ENTITY_DEFAULT_NAMESPACE,
   LocationSpec,
 } from '@backstage/catalog-model';
 import { Config, ConfigReader } from '@backstage/config';
@@ -30,6 +31,7 @@ import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
 import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
 import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
 import { FileReaderProcessor } from './processors/FileReaderProcessor';
+import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
 import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
 import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
@@ -85,6 +87,7 @@ export class LocationReaders implements LocationReader {
       new GitlabReaderProcessor(),
       new BitbucketApiReaderProcessor(config),
       new AzureApiReaderProcessor(config),
+      GithubOrgReaderProcessor.fromConfig(config),
       new UrlReaderProcessor(),
       new YamlProcessor(),
       PlaceholderProcessor.default(),
@@ -229,8 +232,15 @@ export class LocationReaders implements LocationReader {
             this.readLocation.bind(this),
           );
         } catch (e) {
-          const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`;
+          const message = `Processor ${
+            processor.constructor.name
+          } threw an error while processing entity ${current.kind}:${
+            current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE
+          }/${current.metadata.name} at ${item.location.type} ${
+            item.location.target
+          }, ${e}`;
           emit(result.generalError(item.location, message));
+          this.logger.warn(message);
         }
       }
     }

plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts

@@ -25,6 +25,10 @@ export class EntityPolicyProcessor implements LocationProcessor {
   }
 
   async processEntity(entity: Entity): Promise<Entity> {
-    return await this.policy.enforce(entity);
+    const output = await this.policy.enforce(entity);
+    if (!output) {
+      throw new Error(`No policy applied to entity`);
+    }
+    return output;
   }
 }

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts

@@ -0,0 +1,170 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { graphql } from '@octokit/graphql';
+import * as results from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+import { getOrganizationTeams, getOrganizationUsers } from './util/github';
+import { buildOrgHierarchy } from './util/org';
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ */
+export class GithubOrgReaderProcessor implements LocationProcessor {
+  static fromConfig(config: Config) {
+    return new GithubOrgReaderProcessor(readConfig(config));
+  }
+
+  constructor(private readonly providers: ProviderConfig[]) {}
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(`${p.target}/`),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`,
+      );
+    }
+
+    const { org } = parseUrl(location.target);
+    const client = !provider.token
+      ? graphql
+      : graphql.defaults({
+          headers: {
+            authorization: `token ${provider.token}`,
+          },
+        });
+
+    const { users } = await getOrganizationUsers(client, org);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(
+      client,
+      org,
+    );
+    buildOrgHierarchy(groups, users, groupMemberUsers);
+
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}
+
+/*
+ * Helpers
+ */
+
+/**
+ * The configuration parameters for a single GitHub API provider.
+ */
+type ProviderConfig = {
+  /**
+   * The prefix of the target that this matches on, e.g. "https://github.com",
+   * with no trailing slash.
+   */
+  target: string;
+
+  /**
+   * The base URL of the API of this provider, e.g. "https://api.github.com",
+   * with no trailing slash.
+   *
+   * May be omitted specifically for GitHub; then it will be deduced.
+   */
+  apiBaseUrl?: string;
+
+  /**
+   * The authorization token to use for requests to this provider.
+   *
+   * If no token is specified, anonymous access is used.
+   */
+  token?: string;
+};
+
+// TODO(freben): Break out common code and config from here and GithubReaderProcessor
+export function readConfig(config: Config): ProviderConfig[] {
+  const providers: ProviderConfig[] = [];
+
+  const providerConfigs =
+    config.getOptionalConfigArray('catalog.processors.githubOrg.providers') ??
+    [];
+
+  // First read all the explicit providers
+  for (const providerConfig of providerConfigs) {
+    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
+    const token = providerConfig.getOptionalString('token');
+
+    if (apiBaseUrl) {
+      apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
+    } else if (target === 'https://github.com') {
+      apiBaseUrl = 'https://api.github.com';
+    }
+
+    if (!apiBaseUrl) {
+      throw new Error(
+        `Provider at ${target} must configure an explicit apiBaseUrl`,
+      );
+    }
+
+    providers.push({ target, apiBaseUrl, token });
+  }
+
+  // If no explicit github.com provider was added, put one in the list as
+  // a convenience
+  if (!providers.some(p => p.target === 'https://github.com')) {
+    providers.push({
+      target: 'https://github.com',
+      apiBaseUrl: 'https://api.github.com',
+    });
+  }
+
+  return providers;
+}
+
+export function parseUrl(urlString: string): { org: string } {
+  const path = new URL(urlString).pathname.substr(1).split('/');
+
+  // /spotify
+  if (path.length === 1) {
+    return { org: path[0] };
+  }
+
+  // /orgs/spotify[/<teams or people>]
+  if (
+    path.length >= 2 &&
+    path.length <= 3 &&
+    path[0] === 'orgs' &&
+    [undefined, 'teams', 'people'].includes(path[2])
+  ) {
+    return { org: path[1] };
+  }
+
+  throw new Error(`Expected a URL pointing to /<org> or /orgs/<org>`);
+}

plugins/catalog-backend/src/ingestion/processors/util/github.ts

@@ -0,0 +1,285 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { graphql } from '@octokit/graphql';
+
+// Graphql types
+
+type QueryResponse = {
+  organization: Organization;
+};
+
+type Organization = {
+  membersWithRole: Connection<User>;
+  team: Team;
+  teams: Connection<Team>;
+};
+
+type PageInfo = {
+  hasNextPage: boolean;
+  endCursor?: string;
+};
+
+type User = {
+  login: string;
+  bio?: string;
+  avatarUrl?: string;
+  email?: string;
+  name?: string;
+};
+
+type Team = {
+  slug: string;
+  combinedSlug: string;
+  description?: string;
+  parentTeam?: Team;
+  members: Connection<User>;
+};
+
+type Connection<T> = {
+  pageInfo: PageInfo;
+  nodes: T[];
+};
+
+/**
+ * Gets all the users out of a GitHub organization.
+ *
+ * Note that the users will not have their memberships filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ */
+export async function getOrganizationUsers(
+  client: typeof graphql,
+  org: string,
+): Promise<{ users: UserEntity[] }> {
+  const users: UserEntity[] = [];
+
+  // There is no user -> teams edge, so we leave the memberships empty for
+  // now and let the team iteration handle it instead
+  let cursor: string | undefined = undefined;
+  const query = `
+      query users($org: String!, $cursor: String) {
+        organization(login: $org) {
+          membersWithRole(first: 100, after: $cursor) {
+            pageInfo { hasNextPage, endCursor }
+            nodes { avatarUrl, bio, email, login, name }
+          }
+        }
+      }`;
+
+  for (let i = 0; i < 100 /* just for sanity */; ++i) {
+    const response: QueryResponse = await client(query, {
+      org,
+      cursor,
+    });
+
+    const connection = response.organization?.membersWithRole;
+    if (!connection) {
+      throw new Error(`Found no organization named ${org}`);
+    }
+
+    for (const user of connection.nodes) {
+      const entity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: user.login,
+          annotations: {
+            'github.com/user-login': user.login,
+          },
+        },
+        spec: {
+          profile: {},
+          memberOf: [],
+        },
+      };
+
+      if (user.bio) entity.metadata.description = user.bio;
+      if (user.name) entity.spec.profile!.displayName = user.name;
+      if (user.email) entity.spec.profile!.email = user.email;
+      if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
+
+      users.push(entity);
+    }
+
+    const { hasNextPage, endCursor } = connection.pageInfo;
+    if (!hasNextPage) {
+      break;
+    } else {
+      cursor = endCursor;
+    }
+  }
+
+  return { users };
+}
+
+/**
+ * Gets all the teams out of a GitHub organization.
+ *
+ * Note that the teams will not have any relations apart from parent filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ */
+export async function getOrganizationTeams(
+  client: typeof graphql,
+  org: string,
+): Promise<{
+  groups: GroupEntity[];
+  groupMemberUsers: Map<string, string[]>;
+}> {
+  const groups: GroupEntity[] = [];
+  const groupMemberUsers = new Map<string, string[]>();
+
+  let cursor: string | undefined = undefined;
+  const query = `
+    query teams($org: String!, $cursor: String) {
+      organization(login: $org) {
+        teams(first: 100, after: $cursor) {
+          pageInfo { hasNextPage, endCursor }
+          nodes {
+            slug
+            combinedSlug
+            parentTeam { slug }
+            members(first: 100, membership: IMMEDIATE) {
+              pageInfo { hasNextPage, endCursor }
+              nodes { login }
+            }
+          }
+        }
+      }
+    }`;
+
+  for (let i = 0; i < 100 /* just for sanity */; ++i) {
+    const response: QueryResponse = await client(query, {
+      org,
+      cursor,
+    });
+
+    const connection = response.organization?.teams;
+    if (!connection) {
+      throw new Error(`Found no organization named ${org}`);
+    }
+
+    for (const team of connection.nodes) {
+      const entity: GroupEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: team.slug,
+          annotations: {
+            'github.com/team-slug': team.combinedSlug,
+          },
+        },
+        spec: {
+          type: 'team',
+          ancestors: [],
+          children: [],
+          descendants: [],
+        },
+      };
+
+      if (team.description) entity.metadata.description = team.description;
+      if (team.parentTeam) entity.spec.parent = team.parentTeam.slug;
+
+      groups.push(entity);
+
+      const memberNames: string[] = [];
+      groupMemberUsers.set(team.slug, memberNames);
+
+      if (!team.members.pageInfo.hasNextPage) {
+        // We got all the members in one go, run the fast path
+        for (const user of team.members.nodes) {
+          memberNames.push(user.login);
+        }
+      } else {
+        // There were more than a hundred immediate members - run the slow
+        // path of fetching them explicitly
+        const { members } = await getTeamMembers(client, org, team.slug);
+        for (const userLogin of members) {
+          memberNames.push(userLogin);
+        }
+      }
+    }
+
+    const { hasNextPage, endCursor } = connection.pageInfo;
+    if (!hasNextPage) {
+      break;
+    } else {
+      cursor = endCursor;
+    }
+  }
+
+  return { groups, groupMemberUsers };
+}
+
+/**
+ * Gets all the users out of a GitHub organization.
+ *
+ * Note that the users will not have their memberships filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ * @param teamSlug The slug of the team to read
+ */
+export async function getTeamMembers(
+  client: typeof graphql,
+  org: string,
+  teamSlug: string,
+): Promise<{ members: string[] }> {
+  const members: string[] = [];
+
+  let cursor: string | undefined = undefined;
+  const query = `
+    query members($org: String!, $teamSlug: String!, $cursor: String) {
+      organization(login: $org) {
+        team(slug: $teamSlug) {
+          members(first: 100, after: $cursor, membership: IMMEDIATE) {
+            pageInfo { hasNextPage, endCursor }
+            nodes { login }
+          }
+        }
+      }
+    }`;
+
+  for (let j = 0; j < 100 /* just for sanity */; ++j) {
+    const response: QueryResponse = await client(query, {
+      org,
+      teamSlug,
+      cursor,
+    });
+
+    const connection = response.organization?.team?.members;
+    if (!connection) {
+      throw new Error(`Found no team named ${teamSlug} in named ${org}`);
+    }
+
+    for (const user of connection.nodes) {
+      members.push(user.login);
+    }
+
+    const { hasNextPage, endCursor } = connection.pageInfo;
+    if (!hasNextPage) {
+      break;
+    } else {
+      cursor = endCursor;
+    }
+  }
+
+  return { members };
+}

plugins/catalog-backend/src/ingestion/processors/util/org.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+
+export function buildOrgHierarchy(
+  groups: GroupEntity[],
+  users: UserEntity[],
+  groupMemberUsers: Map<string, string[]>,
+) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+  const usersByName = new Map(users.map(u => [u.metadata.name, u]));
+
+  //
+  // Make sure that u.memberOf contain all g
+  //
+
+  for (const [groupName, userNames] of groupMemberUsers.entries()) {
+    for (const userName of userNames) {
+      const user = usersByName.get(userName);
+      if (user && !user.spec.memberOf.includes(groupName)) {
+        user.spec.memberOf.push(groupName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.parent.children contain g
+  //
+
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    const parentName = group.spec.parent;
+    if (parentName) {
+      const parent = groupsByName.get(parentName);
+      if (parent && !parent.spec.children.includes(selfName)) {
+        parent.spec.children.push(selfName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.descendants is complete
+  //
+
+  function visitDescendants(current: GroupEntity): string[] {
+    if (current.spec.descendants.length) {
+      return current.spec.descendants;
+    }
+
+    const accumulator = new Set<string>();
+    for (const childName of current.spec.children) {
+      accumulator.add(childName);
+      const child = groupsByName.get(childName);
+      if (child) {
+        for (const d of visitDescendants(child)) {
+          accumulator.add(d);
+        }
+      }
+    }
+
+    const descendants = Array.from(accumulator);
+    current.spec.descendants = descendants;
+    return descendants;
+  }
+
+  for (const group of groups) {
+    visitDescendants(group);
+  }
+
+  //
+  // Make sure that g.ancestors is complete
+  //
+
+  function visitAncestors(current: GroupEntity): string[] {
+    if (current.spec.ancestors.length) {
+      return current.spec.ancestors;
+    }
+
+    let ancestors: string[];
+    const parentName = current.spec.parent;
+    if (!parentName) {
+      ancestors = [];
+    } else {
+      const parent = groupsByName.get(parentName);
+      if (parent) {
+        ancestors = [parentName, ...visitAncestors(parent)];
+      } else {
+        ancestors = [parentName];
+      }
+    }
+
+    current.spec.ancestors = ancestors;
+    return ancestors;
+  }
+
+  for (const group of groups) {
+    visitAncestors(group);
+  }
+}