Merge pull request #2669 from spotify/freben/github-org

feat(catalog-backend): implement github org entity ingestion

plugins/catalog-backend/package.json

@@ -23,6 +23,7 @@
     "@backstage/backend-common": "^0.1.1-alpha.23",
     "@backstage/catalog-model": "^0.1.1-alpha.23",
     "@backstage/config": "^0.1.1-alpha.23",
+    "@octokit/graphql": "^4.5.6",
     "@types/express": "^4.17.6",
     "codeowners-utils": "^1.0.2",
     "core-js": "^3.6.5",

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -19,6 +19,7 @@ import {
   Entity,
   EntityPolicies,
   EntityPolicy,
+  ENTITY_DEFAULT_NAMESPACE,
   LocationSpec,
 } from '@backstage/catalog-model';
 import { Config, ConfigReader } from '@backstage/config';
@@ -30,6 +31,7 @@ import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
 import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
 import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
 import { FileReaderProcessor } from './processors/FileReaderProcessor';
+import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
 import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
 import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
 import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
@@ -86,6 +88,7 @@ export class LocationReaders implements LocationReader {
       new GitlabReaderProcessor(),
       new BitbucketApiReaderProcessor(config),
       new AzureApiReaderProcessor(config),
+      GithubOrgReaderProcessor.fromConfig(config),
       new UrlReaderProcessor(),
       new YamlProcessor(),
       PlaceholderProcessor.default(),
@@ -178,12 +181,14 @@ export class LocationReaders implements LocationReader {
         } catch (e) {
           const message = `Processor ${processor.constructor.name} threw an error while reading location ${item.location.type} ${item.location.target}, ${e}`;
           emit(result.generalError(item.location, message));
+          this.logger.warn(message);
         }
       }
     }
 
     const message = `No processor was able to read location ${item.location.type} ${item.location.target}`;
     emit(result.inputError(item.location, message));
+    this.logger.warn(message);
   }
 
   private async handleData(
@@ -203,6 +208,7 @@ export class LocationReaders implements LocationReader {
         } catch (e) {
           const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`;
           emit(result.generalError(item.location, message));
+          this.logger.warn(message);
         }
       }
     }
@@ -231,8 +237,15 @@ export class LocationReaders implements LocationReader {
             this.readLocation.bind(this),
           );
         } catch (e) {
-          const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`;
+          // Construct the name carefully, if we got validation errors we do
+          // not want to crash here due to missing metadata or so
+          const namespace = !current.metadata
+            ? ''
+            : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
+          const name = !current.metadata ? '' : current.metadata.name;
+          const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
           emit(result.generalError(item.location, message));
+          this.logger.warn(message);
         }
       }
     }
@@ -255,6 +268,7 @@ export class LocationReaders implements LocationReader {
         } catch (e) {
           const message = `Processor ${processor.constructor.name} threw an error while handling another error at ${item.location.type} ${item.location.target}, ${e}`;
           emit(result.generalError(item.location, message));
+          this.logger.warn(message);
         }
       }
     }

plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts

@@ -25,6 +25,10 @@ export class EntityPolicyProcessor implements LocationProcessor {
   }
 
   async processEntity(entity: Entity): Promise<Entity> {
-    return await this.policy.enforce(entity);
+    const output = await this.policy.enforce(entity);
+    if (!output) {
+      throw new Error(`Entity did not match any known schema`);
+    }
+    return output;
   }
 }

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts

@@ -0,0 +1,160 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { graphql } from '@octokit/graphql';
+import * as results from './results';
+import { LocationProcessor, LocationProcessorEmit } from './types';
+import { getOrganizationTeams, getOrganizationUsers } from './util/github';
+import { buildOrgHierarchy } from './util/org';
+
+/**
+ * Extracts teams and users out of a GitHub org.
+ */
+export class GithubOrgReaderProcessor implements LocationProcessor {
+  static fromConfig(config: Config) {
+    return new GithubOrgReaderProcessor(readConfig(config));
+  }
+
+  constructor(private readonly providers: ProviderConfig[]) {}
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: LocationProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(`${p.target}/`),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`,
+      );
+    }
+
+    const { org } = parseUrl(location.target);
+    const client = !provider.token
+      ? graphql
+      : graphql.defaults({
+          headers: {
+            authorization: `token ${provider.token}`,
+          },
+        });
+
+    const { users } = await getOrganizationUsers(client, org);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(
+      client,
+      org,
+    );
+    buildOrgHierarchy(groups, users, groupMemberUsers);
+
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}
+
+/*
+ * Helpers
+ */
+
+/**
+ * The configuration parameters for a single GitHub API provider.
+ */
+type ProviderConfig = {
+  /**
+   * The prefix of the target that this matches on, e.g. "https://github.com",
+   * with no trailing slash.
+   */
+  target: string;
+
+  /**
+   * The base URL of the API of this provider, e.g. "https://api.github.com",
+   * with no trailing slash.
+   *
+   * May be omitted specifically for GitHub; then it will be deduced.
+   */
+  apiBaseUrl?: string;
+
+  /**
+   * The authorization token to use for requests to this provider.
+   *
+   * If no token is specified, anonymous access is used.
+   */
+  token?: string;
+};
+
+// TODO(freben): Break out common code and config from here and GithubReaderProcessor
+export function readConfig(config: Config): ProviderConfig[] {
+  const providers: ProviderConfig[] = [];
+
+  const providerConfigs =
+    config.getOptionalConfigArray('catalog.processors.githubOrg.providers') ??
+    [];
+
+  // First read all the explicit providers
+  for (const providerConfig of providerConfigs) {
+    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
+    const token = providerConfig.getOptionalString('token');
+
+    if (apiBaseUrl) {
+      apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
+    } else if (target === 'https://github.com') {
+      apiBaseUrl = 'https://api.github.com';
+    }
+
+    if (!apiBaseUrl) {
+      throw new Error(
+        `Provider at ${target} must configure an explicit apiBaseUrl`,
+      );
+    }
+
+    providers.push({ target, apiBaseUrl, token });
+  }
+
+  // If no explicit github.com provider was added, put one in the list as
+  // a convenience
+  if (!providers.some(p => p.target === 'https://github.com')) {
+    providers.push({
+      target: 'https://github.com',
+      apiBaseUrl: 'https://api.github.com',
+    });
+  }
+
+  return providers;
+}
+
+export function parseUrl(urlString: string): { org: string } {
+  const path = new URL(urlString).pathname.substr(1).split('/');
+
+  // /spotify
+  if (path.length === 1 && path[0].length) {
+    return { org: decodeURIComponent(path[0]) };
+  }
+
+  throw new Error(`Expected a URL pointing to /<org>`);
+}

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts

@@ -27,7 +27,7 @@ describe('UrlReaderProcessor', () => {
   const mockApiOrigin = 'http://localhost:23000';
   const server = setupServer();
 
-  beforeAll(() => server.listen());
+  beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
   afterEach(() => server.resetHandlers());
   afterAll(() => server.close());
 

plugins/catalog-backend/src/ingestion/processors/util/github.test.ts

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { graphql } from '@octokit/graphql';
+import { graphql as graphqlMsw } from 'msw';
+import { setupServer } from 'msw/node';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  getTeamMembers,
+  QueryResponse,
+} from './github';
+
+describe('github', () => {
+  const server = setupServer();
+  beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
+  afterEach(() => server.resetHandlers());
+  afterAll(() => server.close());
+
+  describe('getOrganizationUsers', () => {
+    it('reads members', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({ name: 'a', description: 'c' }),
+            spec: {
+              profile: { displayName: 'b', email: 'd', picture: 'e' },
+              memberOf: [],
+            },
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(getOrganizationUsers(graphql, 'a')).resolves.toEqual(output);
+    });
+  });
+
+  describe('getOrganizationTeams', () => {
+    it('reads teams', async () => {
+      const input: QueryResponse = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: { pageInfo: { hasNextPage: false }, nodes: [] },
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        groups: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({ name: 'team' }),
+            spec: {
+              type: 'team',
+              parent: 'parent',
+              ancestors: [],
+              children: [],
+              descendants: [],
+            },
+          }),
+        ],
+        groupMemberUsers: new Map([['team', ['user']]]),
+      };
+
+      server.use(
+        graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
+    });
+  });
+
+  describe('getTeamMembers', () => {
+    it('reads team members', async () => {
+      const input: QueryResponse = {
+        organization: {
+          team: {
+            slug: '',
+            combinedSlug: '',
+            members: {
+              pageInfo: { hasNextPage: false },
+              nodes: [{ login: 'user' }],
+            },
+          },
+        },
+      };
+
+      const output = {
+        members: ['user'],
+      };
+
+      server.use(
+        graphqlMsw.query('members', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output);
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/util/github.ts

@@ -0,0 +1,297 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { graphql } from '@octokit/graphql';
+
+// Graphql types
+
+export type QueryResponse = {
+  organization: Organization;
+};
+
+export type Organization = {
+  membersWithRole?: Connection<User>;
+  team?: Team;
+  teams?: Connection<Team>;
+};
+
+export type PageInfo = {
+  hasNextPage: boolean;
+  endCursor?: string;
+};
+
+export type User = {
+  login: string;
+  bio?: string;
+  avatarUrl?: string;
+  email?: string;
+  name?: string;
+};
+
+export type Team = {
+  slug: string;
+  combinedSlug: string;
+  description?: string;
+  parentTeam?: Team;
+  members: Connection<User>;
+};
+
+export type Connection<T> = {
+  pageInfo: PageInfo;
+  nodes: T[];
+};
+
+/**
+ * Gets all the users out of a GitHub organization.
+ *
+ * Note that the users will not have their memberships filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ */
+export async function getOrganizationUsers(
+  client: typeof graphql,
+  org: string,
+): Promise<{ users: UserEntity[] }> {
+  const query = `
+    query users($org: String!, $cursor: String) {
+      organization(login: $org) {
+        membersWithRole(first: 100, after: $cursor) {
+          pageInfo { hasNextPage, endCursor }
+          nodes { avatarUrl, bio, email, login, name }
+        }
+      }
+    }`;
+
+  // There is no user -> teams edge, so we leave the memberships empty for
+  // now and let the team iteration handle it instead
+  const mapper = (user: User) => {
+    const entity: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: {
+        name: user.login,
+        annotations: {
+          'github.com/user-login': user.login,
+        },
+      },
+      spec: {
+        profile: {},
+        memberOf: [],
+      },
+    };
+
+    if (user.bio) entity.metadata.description = user.bio;
+    if (user.name) entity.spec.profile!.displayName = user.name;
+    if (user.email) entity.spec.profile!.email = user.email;
+    if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
+
+    return entity;
+  };
+
+  const users = await queryWithPaging(
+    client,
+    query,
+    r => r.organization?.membersWithRole,
+    mapper,
+    { org },
+  );
+
+  return { users };
+}
+
+/**
+ * Gets all the teams out of a GitHub organization.
+ *
+ * Note that the teams will not have any relations apart from parent filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ */
+export async function getOrganizationTeams(
+  client: typeof graphql,
+  org: string,
+): Promise<{
+  groups: GroupEntity[];
+  groupMemberUsers: Map<string, string[]>;
+}> {
+  const query = `
+    query teams($org: String!, $cursor: String) {
+      organization(login: $org) {
+        teams(first: 100, after: $cursor) {
+          pageInfo { hasNextPage, endCursor }
+          nodes {
+            slug
+            combinedSlug
+            parentTeam { slug }
+            members(first: 100, membership: IMMEDIATE) {
+              pageInfo { hasNextPage }
+              nodes { login }
+            }
+          }
+        }
+      }
+    }`;
+
+  // Gets populated inside the mapper below
+  const groupMemberUsers = new Map<string, string[]>();
+
+  const mapper = async (team: Team) => {
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: team.slug,
+        annotations: {
+          'github.com/team-slug': team.combinedSlug,
+        },
+      },
+      spec: {
+        type: 'team',
+        ancestors: [],
+        children: [],
+        descendants: [],
+      },
+    };
+
+    if (team.description) entity.metadata.description = team.description;
+    if (team.parentTeam) entity.spec.parent = team.parentTeam.slug;
+
+    const memberNames: string[] = [];
+    groupMemberUsers.set(team.slug, memberNames);
+
+    if (!team.members.pageInfo.hasNextPage) {
+      // We got all the members in one go, run the fast path
+      for (const user of team.members.nodes) {
+        memberNames.push(user.login);
+      }
+    } else {
+      // There were more than a hundred immediate members - run the slow
+      // path of fetching them explicitly
+      const { members } = await getTeamMembers(client, org, team.slug);
+      for (const userLogin of members) {
+        memberNames.push(userLogin);
+      }
+    }
+
+    return entity;
+  };
+
+  const groups = await queryWithPaging(
+    client,
+    query,
+    r => r.organization?.teams,
+    mapper,
+    { org },
+  );
+
+  return { groups, groupMemberUsers };
+}
+
+/**
+ * Gets all the users out of a GitHub organization.
+ *
+ * Note that the users will not have their memberships filled in.
+ *
+ * @param client An octokit graphql client
+ * @param org The slug of the org to read
+ * @param teamSlug The slug of the team to read
+ */
+export async function getTeamMembers(
+  client: typeof graphql,
+  org: string,
+  teamSlug: string,
+): Promise<{ members: string[] }> {
+  const query = `
+    query members($org: String!, $teamSlug: String!, $cursor: String) {
+      organization(login: $org) {
+        team(slug: $teamSlug) {
+          members(first: 100, after: $cursor, membership: IMMEDIATE) {
+            pageInfo { hasNextPage, endCursor }
+            nodes { login }
+          }
+        }
+      }
+    }`;
+
+  const members = await queryWithPaging(
+    client,
+    query,
+    r => r.organization?.team?.members,
+    user => user.login,
+    { org, teamSlug },
+  );
+
+  return { members };
+}
+
+//
+// Helpers
+//
+
+/**
+ * Assists in repeatedly executing a query with a paged response.
+ *
+ * Requires that the query accepts a $cursor variable.
+ *
+ * @param client The octokit client
+ * @param query The query to execute
+ * @param connection A function that, given the response, picks out the actual
+ *                   Connection object that's being iterated
+ * @param mapper A function that, given one of the nodes in the Connection,
+ *               returns the model mapped form of it
+ * @param variables The variable values that the query needs, minus the cursor
+ */
+export async function queryWithPaging<
+  GraphqlType,
+  OutputType,
+  Variables extends {},
+  Response = QueryResponse
+>(
+  client: typeof graphql,
+  query: string,
+  connection: (response: Response) => Connection<GraphqlType> | undefined,
+  mapper: (item: GraphqlType) => Promise<OutputType> | OutputType,
+  variables: Variables,
+): Promise<OutputType[]> {
+  const result: OutputType[] = [];
+
+  let cursor: string | undefined = undefined;
+  for (let j = 0; j < 1000 /* just for sanity */; ++j) {
+    const response: Response = await client(query, {
+      ...variables,
+      cursor,
+    });
+
+    const conn = connection(response);
+    if (!conn) {
+      throw new Error(`Found no match for ${JSON.stringify(variables)}`);
+    }
+
+    for (const node of conn.nodes) {
+      result.push(await mapper(node));
+    }
+
+    if (!conn.pageInfo.hasNextPage) {
+      break;
+    } else {
+      cursor = conn.pageInfo.endCursor;
+    }
+  }
+
+  return result;
+}

plugins/catalog-backend/src/ingestion/processors/util/org.test.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { buildOrgHierarchy } from './org';
+
+function u(name: string): UserEntity {
+  return {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: { name },
+    spec: { memberOf: [] },
+  };
+}
+
+function g(
+  name: string,
+  parent: string | undefined,
+  children: string[],
+): GroupEntity {
+  return {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: { name },
+    spec: { type: 'team', parent, children, ancestors: [], descendants: [] },
+  };
+}
+
+describe('buildOrgHierarchy', () => {
+  it('puts users in the respective groups', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', undefined, []);
+    const x = u('x');
+    const y = u('y');
+    const groupMemberUsers: Map<string, string[]> = new Map([
+      ['a', ['x', 'y']],
+      ['b', ['y']],
+    ]);
+    buildOrgHierarchy([a, b], [x, y], groupMemberUsers);
+    expect(x.spec.memberOf).toEqual(['a']);
+    expect(y.spec.memberOf).toEqual(['a', 'b']);
+  });
+
+  it('adds groups to their parent.children', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const d = g('d', 'a', []);
+    buildOrgHierarchy([a, b, c, d], [], new Map());
+    expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd']));
+    expect(b.spec.children).toEqual(expect.arrayContaining(['c']));
+    expect(c.spec.children).toEqual([]);
+    expect(d.spec.children).toEqual([]);
+  });
+
+  it('fills out descendants', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const d = g('d', 'a', []);
+    buildOrgHierarchy([a, b, c, d], [], new Map());
+    expect(a.spec.descendants).toEqual(expect.arrayContaining(['b', 'c', 'd']));
+    expect(b.spec.descendants).toEqual(expect.arrayContaining(['c']));
+    expect(c.spec.descendants).toEqual([]);
+    expect(d.spec.descendants).toEqual([]);
+  });
+
+  it('fills out ancestors', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const d = g('d', 'a', []);
+    buildOrgHierarchy([a, b, c, d], [], new Map());
+    expect(a.spec.ancestors).toEqual([]);
+    expect(b.spec.ancestors).toEqual(expect.arrayContaining(['a']));
+    expect(c.spec.ancestors).toEqual(expect.arrayContaining(['a', 'b']));
+    expect(d.spec.ancestors).toEqual(expect.arrayContaining(['a']));
+  });
+});

plugins/catalog-backend/src/ingestion/processors/util/org.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+
+export function buildOrgHierarchy(
+  groups: GroupEntity[],
+  users: UserEntity[],
+  groupMemberUsers: Map<string, string[]>,
+) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+  const usersByName = new Map(users.map(u => [u.metadata.name, u]));
+
+  //
+  // Make sure that u.memberOf contain all g
+  //
+
+  for (const [groupName, userNames] of groupMemberUsers.entries()) {
+    for (const userName of userNames) {
+      const user = usersByName.get(userName);
+      if (user && !user.spec.memberOf.includes(groupName)) {
+        user.spec.memberOf.push(groupName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.parent.children contain g
+  //
+
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    const parentName = group.spec.parent;
+    if (parentName) {
+      const parent = groupsByName.get(parentName);
+      if (parent && !parent.spec.children.includes(selfName)) {
+        parent.spec.children.push(selfName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.descendants is complete
+  //
+
+  function visitDescendants(current: GroupEntity): string[] {
+    if (current.spec.descendants.length) {
+      return current.spec.descendants;
+    }
+
+    const accumulator = new Set<string>();
+    for (const childName of current.spec.children) {
+      accumulator.add(childName);
+      const child = groupsByName.get(childName);
+      if (child) {
+        for (const d of visitDescendants(child)) {
+          accumulator.add(d);
+        }
+      }
+    }
+
+    const descendants = Array.from(accumulator);
+    current.spec.descendants = descendants;
+    return descendants;
+  }
+
+  for (const group of groups) {
+    visitDescendants(group);
+  }
+
+  //
+  // Make sure that g.ancestors is complete
+  //
+
+  function visitAncestors(current: GroupEntity): string[] {
+    if (current.spec.ancestors.length) {
+      return current.spec.ancestors;
+    }
+
+    let ancestors: string[];
+    const parentName = current.spec.parent;
+    if (!parentName) {
+      ancestors = [];
+    } else {
+      const parent = groupsByName.get(parentName);
+      if (parent) {
+        ancestors = [parentName, ...visitAncestors(parent)];
+      } else {
+        ancestors = [parentName];
+      }
+    }
+
+    current.spec.ancestors = ancestors;
+    return ancestors;
+  }
+
+  for (const group of groups) {
+    visitAncestors(group);
+  }
+}

plugins/catalog-graphql/src/graphql/module.test.ts

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import { createModule } from './module';
 import { execute } from 'graphql';
 import { rest } from 'msw';
@@ -36,9 +37,8 @@ describe('Catalog Module', () => {
     },
   ]);
 
-  beforeAll(() => worker.listen());
+  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
   afterAll(() => worker.close());
-
   afterEach(() => worker.resetHandlers());
 
   describe('Default Entity', () => {

plugins/catalog-graphql/src/service/client.test.ts

@@ -20,9 +20,8 @@ import { setupServer } from 'msw/node';
 describe('Catalog GraphQL Module', () => {
   const worker = setupServer();
 
-  beforeAll(() => worker.listen());
+  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
   afterAll(() => worker.close());
-
   afterEach(() => worker.resetHandlers());
 
   const baseUrl = 'http://localhost:1234';

plugins/catalog/src/api/CatalogClient.test.ts

@@ -25,7 +25,7 @@ const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base';
 const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
 
 describe('CatalogClient', () => {
-  beforeAll(() => server.listen());
+  beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
   afterEach(() => server.resetHandlers());
   afterAll(() => server.close());