Merge pull request #13640 from philips-forks/feature/org_emails

Allow import of github org verified email in User entities

plugins/catalog-backend-module-github/api-report.md

@@ -13,6 +13,8 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GithubCredentialsProvider } from '@backstage/integration';
 import { GithubIntegrationConfig } from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
@@ -21,6 +23,13 @@ import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
 import { TaskRunner } from '@backstage/backend-tasks';
 import { TokenManager } from '@backstage/backend-common';
+import { UserEntity } from '@backstage/catalog-model';
+
+// @public
+export const defaultOrganizationTeamTransformer: TeamTransformer;
+
+// @public
+export const defaultUserTransformer: UserTransformer;
 
 // @public
 export class GithubDiscoveryProcessor implements CatalogProcessor {
@@ -129,6 +138,8 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
     logger: Logger;
     orgs: GithubMultiOrgConfig;
     githubCredentialsProvider?: GithubCredentialsProvider;
+    userTransformer?: UserTransformer;
+    teamTransformer?: TeamTransformer;
   });
   // (undocumented)
   static fromConfig(
@@ -136,6 +147,8 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
     options: {
       logger: Logger;
       githubCredentialsProvider?: GithubCredentialsProvider;
+      userTransformer?: UserTransformer;
+      teamTransformer?: TeamTransformer;
     },
   ): GithubMultiOrgReaderProcessor;
   // (undocumented)
@@ -165,6 +178,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
     gitHubConfig: GithubIntegrationConfig;
     logger: Logger;
     githubCredentialsProvider?: GithubCredentialsProvider;
+    userTransformer?: UserTransformer;
+    teamTransformer?: TeamTransformer;
   });
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;
@@ -188,6 +203,8 @@ export interface GithubOrgEntityProviderOptions {
   logger: Logger;
   orgUrl: string;
   schedule?: 'manual' | TaskRunner;
+  teamTransformer?: TeamTransformer;
+  userTransformer?: UserTransformer;
 }
 
 // @public
@@ -214,4 +231,48 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     emit: CatalogProcessorEmit,
   ): Promise<boolean>;
 }
+
+// @public
+export type GithubTeam = {
+  slug: string;
+  combinedSlug: string;
+  name?: string;
+  description?: string;
+  avatarUrl?: string;
+  editTeamUrl?: string;
+  parentTeam?: GithubTeam;
+  members: GithubUser[];
+};
+
+// @public
+export type GithubUser = {
+  login: string;
+  bio?: string;
+  avatarUrl?: string;
+  email?: string;
+  name?: string;
+  organizationVerifiedDomainEmails?: string[];
+};
+
+// @public
+export type TeamTransformer = (
+  item: GithubTeam,
+  ctx: TransformerContext,
+) => Promise<GroupEntity | undefined>;
+
+// @public
+export interface TransformerContext {
+  // (undocumented)
+  client: typeof graphql;
+  // (undocumented)
+  org: string;
+  // (undocumented)
+  query: string;
+}
+
+// @public
+export type UserTransformer = (
+  item: GithubUser,
+  ctx: TransformerContext,
+) => Promise<UserEntity | undefined>;
 ```

plugins/catalog-backend-module-github/src/index.ts

@@ -15,20 +15,31 @@
  */
 
 /**
- * A Backstage catalog backend module that helps integrate towards GitHub
+ * A Backstage catalog backend module that helps integrate towards Github
  *
  * @packageDocumentation
  */
 
 export { GithubLocationAnalyzer } from './analyzers/GithubLocationAnalyzer';
 export type { GithubLocationAnalyzerOptions } from './analyzers/GithubLocationAnalyzer';
-export type { GithubMultiOrgConfig } from './lib';
 export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
 export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
 export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
 export { GithubEntityProvider } from './providers/GithubEntityProvider';
-export { GithubOrgEntityProvider } from './providers/GithubOrgEntityProvider';
-export type { GithubOrgEntityProviderOptions } from './providers/GithubOrgEntityProvider';
+export type {
+  GithubOrgEntityProvider,
+  GithubOrgEntityProviderOptions,
+} from './providers/GithubOrgEntityProvider';
 export { githubEntityProviderCatalogModule } from './service/GithubEntityProviderCatalogModule';
+export {
+  type GithubMultiOrgConfig,
+  type GithubTeam,
+  type GithubUser,
+  type UserTransformer,
+  defaultUserTransformer,
+  type TeamTransformer,
+  defaultOrganizationTeamTransformer,
+  type TransformerContext,
+} from './lib';
 
 export * from './deprecated';

plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { graphql } from '@octokit/graphql';
+import { GithubTeam, GithubUser } from './github';
+
+/**
+ * Context passed to Transformers
+ *
+ * @public
+ */
+export interface TransformerContext {
+  client: typeof graphql;
+  query: string;
+  org: string;
+}
+
+/**
+ * Transformer for GitHub users to UserEntity
+ *
+ * @public
+ */
+export type UserTransformer = (
+  item: GithubUser,
+  ctx: TransformerContext,
+) => Promise<UserEntity | undefined>;
+
+/**
+ * Transformer for GitHub Team to GroupEntity
+ *
+ * @public
+ */
+export type TeamTransformer = (
+  item: GithubTeam,
+  ctx: TransformerContext,
+) => Promise<GroupEntity | undefined>;
+
+/**
+ * Default transformer for GitHub users to UserEntity
+ *
+ * @public
+ */
+export const defaultUserTransformer: UserTransformer = async (
+  item: GithubUser,
+) => {
+  const entity: UserEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: {
+      name: item.login,
+      annotations: {
+        'github.com/user-login': item.login,
+      },
+    },
+    spec: {
+      profile: {},
+      memberOf: [],
+    },
+  };
+
+  if (item.bio) entity.metadata.description = item.bio;
+  if (item.name) entity.spec.profile!.displayName = item.name;
+  if (item.email) entity.spec.profile!.email = item.email;
+  if (item.avatarUrl) entity.spec.profile!.picture = item.avatarUrl;
+  return entity;
+};
+
+/**
+ * Default transformer for GitHub Team to GroupEntity
+ *
+ * @public
+ */
+export const defaultOrganizationTeamTransformer: TeamTransformer =
+  async team => {
+    const annotations: { [annotationName: string]: string } = {
+      'github.com/team-slug': team.combinedSlug,
+    };
+
+    if (team.editTeamUrl) {
+      annotations['backstage.io/edit-url'] = team.editTeamUrl;
+    }
+
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: team.slug,
+        annotations,
+      },
+      spec: {
+        type: 'team',
+        profile: {},
+        children: [],
+      },
+    };
+
+    if (team.description) {
+      entity.metadata.description = team.description;
+    }
+    if (team.name) {
+      entity.spec.profile!.displayName = team.name;
+    }
+    if (team.avatarUrl) {
+      entity.spec.profile!.picture = team.avatarUrl;
+    }
+    if (team.parentTeam) {
+      entity.spec.parent = team.parentTeam.slug;
+    }
+
+    entity.spec.members = team.members.map(user => user.login);
+
+    return entity;
+  };

plugins/catalog-backend-module-github/src/lib/github.test.ts

@@ -15,15 +15,20 @@
  */
 
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { graphql } from '@octokit/graphql';
 import { graphql as graphqlMsw } from 'msw';
 import { setupServer } from 'msw/node';
+import { TeamTransformer, UserTransformer } from './defaultTransformers';
+
 import {
   getOrganizationTeams,
   getOrganizationUsers,
   getTeamMembers,
   getOrganizationRepositories,
   QueryResponse,
+  GithubUser,
+  GithubTeam,
 } from './github';
 import fetch from 'node-fetch';
 
@@ -35,7 +40,7 @@ describe('github', () => {
   const server = setupServer();
   setupRequestMockHandlers(server);
 
-  describe('getOrganizationUsers', () => {
+  describe('getOrganizationUsers using defaultUserMapper', () => {
     it('reads members', async () => {
       const input: QueryResponse = {
         organization: {
@@ -76,7 +81,120 @@ describe('github', () => {
     });
   });
 
-  describe('getOrganizationTeams', () => {
+  describe('getOrganizationUsers using custom UserTransformer', () => {
+    const customUserTransformer: UserTransformer = async (
+      item: GithubUser,
+      {},
+    ) => {
+      if (item.login === 'aa') {
+        return undefined;
+      }
+
+      return {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: `${item.login}-custom`,
+          annotations: {
+            'github.com/user-login': item.login,
+          },
+        },
+        spec: {
+          profile: {},
+          memberOf: [],
+        },
+      } as UserEntity;
+    };
+
+    it('reads members', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'a-custom',
+            }),
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(
+        getOrganizationUsers(graphql, 'a', 'token', customUserTransformer),
+      ).resolves.toEqual(output);
+    });
+
+    it('reads members if undefined is returned from transformer', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+              {
+                login: 'aa',
+                name: 'bb',
+                bio: 'cc',
+                email: 'dd',
+                avatarUrl: 'ee',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'a-custom',
+            }),
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      const users = await getOrganizationUsers(
+        graphql,
+        'a',
+        'token',
+        customUserTransformer,
+      );
+
+      expect(users.users).toHaveLength(1);
+      expect(users).toEqual(output);
+    });
+  });
+
+  describe('getOrganizationTeams using default TeamTransformer', () => {
     let input: QueryResponse;
 
     beforeEach(() => {
@@ -95,7 +213,7 @@ describe('github', () => {
                 parentTeam: {
                   slug: 'parent',
                   combinedSlug: '',
-                  members: { pageInfo: { hasNextPage: false }, nodes: [] },
+                  members: [],
                 },
                 members: {
                   pageInfo: { hasNextPage: false },
@@ -129,10 +247,10 @@ describe('github', () => {
               },
               parent: 'parent',
               children: [],
+              members: ['user'],
             },
           }),
         ],
-        groupMemberUsers: new Map([['team', ['user']]]),
       };
 
       server.use(
@@ -141,37 +259,192 @@ describe('github', () => {
 
       await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
     });
+  });
 
-    it('applies namespaces', async () => {
+  describe('getOrganizationTeams using custom TeamTransformer', () => {
+    let input: QueryResponse;
+
+    const customTeamTransformer: TeamTransformer = async (
+      item: GithubTeam,
+      {},
+    ) => {
+      if (item.name === 'aa') {
+        return undefined;
+      }
+
+      return {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: `${item.name}-custom`,
+          annotations: {
+            'github.com/team-slug': 'blah/team',
+            'backstage.io/edit-url':
+              'http://example.com/orgs/blah/teams/team/edit',
+          },
+          description: item.description,
+        },
+        spec: {
+          type: 'team',
+          profile: {
+            displayName: `${item.name}-custom`,
+            picture: 'http://example.com/team.jpeg',
+          },
+          parent: 'parent',
+          children: [],
+          members: ['user'],
+        },
+      } as GroupEntity;
+    };
+
+    beforeEach(() => {
+      input = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'Team',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+    });
+
+    it('reads teams', async () => {
       const output = {
         groups: [
           expect.objectContaining({
             metadata: expect.objectContaining({
-              name: 'team',
-              namespace: 'foo',
+              name: 'Team-custom',
               description: 'The one and only team',
+              annotations: {
+                'github.com/team-slug': 'blah/team',
+                'backstage.io/edit-url':
+                  'http://example.com/orgs/blah/teams/team/edit',
+              },
             }),
             spec: {
               type: 'team',
               profile: {
-                displayName: 'Team',
+                displayName: 'Team-custom',
                 picture: 'http://example.com/team.jpeg',
               },
               parent: 'parent',
               children: [],
+              members: ['user'],
             },
           }),
         ],
-        groupMemberUsers: new Map([['foo/team', ['user']]]),
       };
 
       server.use(
         graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
       );
 
-      await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual(
-        output,
+      await expect(
+        getOrganizationTeams(graphql, 'a', customTeamTransformer),
+      ).resolves.toEqual(output);
+    });
+
+    it('reads teams if undefined is returned', async () => {
+      input = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'Team',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'aa',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        groups: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'Team-custom',
+              description: 'The one and only team',
+              annotations: {
+                'github.com/team-slug': 'blah/team',
+                'backstage.io/edit-url':
+                  'http://example.com/orgs/blah/teams/team/edit',
+              },
+            }),
+            spec: {
+              type: 'team',
+              profile: {
+                displayName: 'Team-custom',
+                picture: 'http://example.com/team.jpeg',
+              },
+              parent: 'parent',
+              children: [],
+              members: ['user'],
+            },
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
       );
+
+      const teams = await getOrganizationTeams(
+        graphql,
+        'a',
+        customTeamTransformer,
+      );
+
+      expect(teams.groups).toHaveLength(1);
+      expect(teams).toEqual(output);
     });
   });
 
@@ -191,7 +464,7 @@ describe('github', () => {
       };
 
       const output = {
-        members: ['user'],
+        members: [{ login: 'user' }],
       };
 
       server.use(

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -17,19 +17,30 @@
 import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { GithubCredentialType } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
+import {
+  defaultOrganizationTeamTransformer,
+  defaultUserTransformer,
+  TeamTransformer,
+  TransformerContext,
+  UserTransformer,
+} from './defaultTransformers';
 
 // Graphql types
 
 export type QueryResponse = {
-  organization?: Organization;
-  repositoryOwner?: Organization | User;
+  organization?: OrganizationResponse;
+  repositoryOwner?: RepositoryOwnerResponse;
 };
 
-export type Organization = {
-  membersWithRole?: Connection<User>;
-  team?: Team;
-  teams?: Connection<Team>;
-  repositories?: Connection<Repository>;
+type RepositoryOwnerResponse = {
+  repositories?: Connection<RepositoryResponse>;
+};
+
+export type OrganizationResponse = {
+  membersWithRole?: Connection<GithubUser>;
+  team?: GithubTeamResponse;
+  teams?: Connection<GithubTeamResponse>;
+  repositories?: Connection<RepositoryResponse>;
 };
 
 export type PageInfo = {
@@ -37,27 +48,41 @@ export type PageInfo = {
   endCursor?: string;
 };
 
-export type User = {
+/**
+ * Github User
+ *
+ * @public
+ */
+export type GithubUser = {
   login: string;
   bio?: string;
   avatarUrl?: string;
   email?: string;
   name?: string;
-  repositories?: Connection<Repository>;
+  organizationVerifiedDomainEmails?: string[];
 };
 
-export type Team = {
+/**
+ * Github Team
+ *
+ * @public
+ */
+export type GithubTeam = {
   slug: string;
   combinedSlug: string;
   name?: string;
   description?: string;
   avatarUrl?: string;
   editTeamUrl?: string;
-  parentTeam?: Team;
-  members: Connection<User>;
+  parentTeam?: GithubTeam;
+  members: GithubUser[];
 };
 
-export type Repository = {
+export type GithubTeamResponse = Omit<GithubTeam, 'members'> & {
+  members: Connection<GithubUser>;
+};
+
+export type RepositoryResponse = {
   name: string;
   url: string;
   isArchived: boolean;
@@ -88,7 +113,7 @@ export type Connection<T> = {
 };
 
 /**
- * Gets all the users out of a GitHub organization.
+ * Gets all the users out of a Github organization.
  *
  * Note that the users will not have their memberships filled in.
  *
@@ -99,7 +124,7 @@ export async function getOrganizationUsers(
   client: typeof graphql,
   org: string,
   tokenType: GithubCredentialType,
-  userNamespace?: string,
+  userTransformer: UserTransformer = defaultUserTransformer,
 ): Promise<{ users: UserEntity[] }> {
   const query = `
     query users($org: String!, $email: Boolean!, $cursor: String) {
@@ -111,7 +136,8 @@ export async function getOrganizationUsers(
             bio,
             email @include(if: $email),
             login,
-            name
+            name,
+            organizationVerifiedDomainEmails(login: $org)
           }
         }
       }
@@ -119,44 +145,24 @@ export async function getOrganizationUsers(
 
   // There is no user -> teams edge, so we leave the memberships empty for
   // now and let the team iteration handle it instead
-  const mapper = (user: User) => {
-    const entity: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: {
-        name: user.login,
-        annotations: {
-          'github.com/user-login': user.login,
-        },
-      },
-      spec: {
-        profile: {},
-        memberOf: [],
-      },
-    };
-
-    if (userNamespace) entity.metadata.namespace = userNamespace;
-    if (user.bio) entity.metadata.description = user.bio;
-    if (user.name) entity.spec.profile!.displayName = user.name;
-    if (user.email) entity.spec.profile!.email = user.email;
-    if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
-
-    return entity;
-  };
 
   const users = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.membersWithRole,
-    mapper,
-    { org, email: tokenType === 'token' },
+    userTransformer,
+    {
+      org,
+      email: tokenType === 'token',
+    },
   );
 
   return { users };
 }
 
 /**
- * Gets all the teams out of a GitHub organization.
+ * Gets all the teams out of a Github organization.
  *
  * Note that the teams will not have any relations apart from parent filled in.
  *
@@ -166,10 +172,9 @@ export async function getOrganizationUsers(
 export async function getOrganizationTeams(
   client: typeof graphql,
   org: string,
-  orgNamespace?: string,
+  teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
 ): Promise<{
   groups: GroupEntity[];
-  groupMemberUsers: Map<string, string[]>;
 }> {
   const query = `
     query teams($org: String!, $cursor: String) {
@@ -193,86 +198,51 @@ export async function getOrganizationTeams(
       }
     }`;
 
-  // Gets populated inside the mapper below
-  const groupMemberUsers = new Map<string, string[]>();
+  const materialisedTeams = async (
+    item: GithubTeamResponse,
+    ctx: TransformerContext,
+  ): Promise<GroupEntity | undefined> => {
+    const memberNames: GithubUser[] = [];
 
-  const mapper = async (team: Team) => {
-    const annotations: { [annotationName: string]: string } = {
-      'github.com/team-slug': team.combinedSlug,
-    };
-
-    if (team.editTeamUrl) {
-      annotations['backstage.io/edit-url'] = team.editTeamUrl;
-    }
-
-    const entity: GroupEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Group',
-      metadata: {
-        name: team.slug,
-        annotations,
-      },
-      spec: {
-        type: 'team',
-        profile: {},
-        children: [],
-      },
-    };
-
-    if (orgNamespace) {
-      entity.metadata.namespace = orgNamespace;
-    }
-
-    if (team.description) {
-      entity.metadata.description = team.description;
-    }
-    if (team.name) {
-      entity.spec.profile!.displayName = team.name;
-    }
-    if (team.avatarUrl) {
-      entity.spec.profile!.picture = team.avatarUrl;
-    }
-    if (team.parentTeam) {
-      entity.spec.parent = team.parentTeam.slug;
-    }
-
-    const memberNames: string[] = [];
-    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;
-    groupMemberUsers.set(groupKey, memberNames);
-
-    if (!team.members.pageInfo.hasNextPage) {
+    if (!item.members.pageInfo.hasNextPage) {
       // We got all the members in one go, run the fast path
-      for (const user of team.members.nodes) {
-        memberNames.push(user.login);
+      for (const user of item.members.nodes) {
+        memberNames.push(user);
       }
     } else {
       // There were more than a hundred immediate members - run the slow
       // path of fetching them explicitly
-      const { members } = await getTeamMembers(client, org, team.slug);
+      const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
       for (const userLogin of members) {
         memberNames.push(userLogin);
       }
     }
 
-    return entity;
+    const team: GithubTeam = {
+      ...item,
+      members: memberNames,
+    };
+
+    return await teamTransformer(team, ctx);
   };
 
   const groups = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.teams,
-    mapper,
+    materialisedTeams,
     { org },
   );
 
-  return { groups, groupMemberUsers };
+  return { groups };
 }
 
 export async function getOrganizationRepositories(
   client: typeof graphql,
   org: string,
   catalogPath: string,
-): Promise<{ repositories: Repository[] }> {
+): Promise<{ repositories: RepositoryResponse[] }> {
   let relativeCatalogPathRef: string;
   // We must strip the leading slash or the query for objects does not work
   if (catalogPath.startsWith('/')) {
@@ -321,8 +291,9 @@ export async function getOrganizationRepositories(
   const repositories = await queryWithPaging(
     client,
     query,
+    org,
     r => r.repositoryOwner?.repositories,
-    x => x,
+    async x => x,
     { org, catalogPathRef },
   );
 
@@ -330,7 +301,7 @@ export async function getOrganizationRepositories(
 }
 
 /**
- * Gets all the users out of a GitHub organization.
+ * Gets all the users out of a Github organization.
  *
  * Note that the users will not have their memberships filled in.
  *
@@ -342,7 +313,7 @@ export async function getTeamMembers(
   client: typeof graphql,
   org: string,
   teamSlug: string,
-): Promise<{ members: string[] }> {
+): Promise<{ members: GithubUser[] }> {
   const query = `
     query members($org: String!, $teamSlug: String!, $cursor: String) {
       organization(login: $org) {
@@ -358,8 +329,9 @@ export async function getTeamMembers(
   const members = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.team?.members,
-    user => user.login,
+    async user => user,
     { org, teamSlug },
   );
 
@@ -379,7 +351,7 @@ export async function getTeamMembers(
  * @param query - The query to execute
  * @param connection - A function that, given the response, picks out the actual
  *                   Connection object that's being iterated
- * @param mapper - A function that, given one of the nodes in the Connection,
+ * @param transformer - A function that, given one of the nodes in the Connection,
  *               returns the model mapped form of it
  * @param variables - The variable values that the query needs, minus the cursor
  */
@@ -391,8 +363,12 @@ export async function queryWithPaging<
 >(
   client: typeof graphql,
   query: string,
+  org: string,
   connection: (response: Response) => Connection<GraphqlType> | undefined,
-  mapper: (item: GraphqlType) => Promise<OutputType> | OutputType,
+  transformer: (
+    item: GraphqlType,
+    ctx: TransformerContext,
+  ) => Promise<OutputType | undefined>,
   variables: Variables,
 ): Promise<OutputType[]> {
   const result: OutputType[] = [];
@@ -410,7 +386,15 @@ export async function queryWithPaging<
     }
 
     for (const node of conn.nodes) {
-      result.push(await mapper(node));
+      const transformedNode = await transformer(node, {
+        client,
+        query,
+        org,
+      });
+
+      if (transformedNode) {
+        result.push(transformedNode);
+      }
     }
 
     if (!conn.pageInfo.hasNextPage) {

plugins/catalog-backend-module-github/src/lib/index.ts

@@ -20,6 +20,15 @@ export {
   getOrganizationRepositories,
   getOrganizationTeams,
   getOrganizationUsers,
+  type GithubUser,
+  type GithubTeam,
 } from './github';
+export {
+  type UserTransformer,
+  defaultUserTransformer,
+  type TeamTransformer,
+  defaultOrganizationTeamTransformer,
+  type TransformerContext,
+} from './defaultTransformers';
 export { assignGroupsToUsers, buildOrgHierarchy } from './org';
 export { parseGithubOrgUrl } from './util';

plugins/catalog-backend-module-github/src/lib/org.test.ts

@@ -68,13 +68,47 @@ describe('buildOrgHierarchy', () => {
 describe('assignGroupsToUsers', () => {
   it('should assign groups to users', () => {
     const users: UserEntity[] = [u('u1'), u('u2')];
-    const groupMemberUsers = new Map<string, string[]>([
-      ['g1', ['u1', 'u2']],
-      ['g2', ['u2']],
-      ['g3', ['u3']],
-    ]);
 
-    assignGroupsToUsers(users, groupMemberUsers);
+    const groups: GroupEntity[] = [
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g1',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u1', 'u2'],
+        },
+      },
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g2',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u2'],
+        },
+      },
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g3',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u3'],
+        },
+      },
+    ];
+
+    assignGroupsToUsers(users, groups);
 
     expect(users[0].spec.memberOf).toEqual(['g1']);
     expect(users[1].spec.memberOf).toEqual(['g1', 'g2']);

plugins/catalog-backend-module-github/src/lib/org.ts

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import {
+  DEFAULT_NAMESPACE,
+  GroupEntity,
+  UserEntity,
+} from '@backstage/catalog-model';
 
 export function buildOrgHierarchy(groups: GroupEntity[]) {
   const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
@@ -52,8 +56,19 @@ export function buildOrgHierarchy(groups: GroupEntity[]) {
 // Ensure that users have their direct group memberships.
 export function assignGroupsToUsers(
   users: UserEntity[],
-  groupMemberUsers: Map<string, string[]>,
+  groups: GroupEntity[],
 ) {
+  const groupMemberUsers = new Map(
+    groups.map(group => {
+      const groupKey =
+        group.metadata.namespace &&
+        group.metadata.namespace !== DEFAULT_NAMESPACE
+          ? `${group.metadata.namespace}/${group.metadata.name}`
+          : group.metadata.name;
+      return [groupKey, group.spec.members || []];
+    }),
+  );
+
   const usersByName = new Map(users.map(u => [u.metadata.name, u]));
   for (const [groupName, userNames] of groupMemberUsers.entries()) {
     for (const userName of userNames) {

plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { GroupEntity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import {
   DefaultGithubCredentialsProvider,
@@ -32,11 +33,15 @@ import {
 import { graphql } from '@octokit/graphql';
 import { Logger } from 'winston';
 import {
+  assignGroupsToUsers,
   buildOrgHierarchy,
+  defaultOrganizationTeamTransformer,
   getOrganizationTeams,
   getOrganizationUsers,
   GithubMultiOrgConfig,
   readGithubMultiOrgConfig,
+  TeamTransformer,
+  UserTransformer,
 } from '../lib';
 
 /**
@@ -57,6 +62,8 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
     options: {
       logger: Logger;
       githubCredentialsProvider?: GithubCredentialsProvider;
+      userTransformer?: UserTransformer;
+      teamTransformer?: TeamTransformer;
     },
   ) {
     const c = config.getOptionalConfig('catalog.processors.githubMultiOrg');
@@ -69,12 +76,16 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
     });
   }
 
-  constructor(options: {
-    integrations: ScmIntegrationRegistry;
-    logger: Logger;
-    orgs: GithubMultiOrgConfig;
-    githubCredentialsProvider?: GithubCredentialsProvider;
-  }) {
+  constructor(
+    private options: {
+      integrations: ScmIntegrationRegistry;
+      logger: Logger;
+      orgs: GithubMultiOrgConfig;
+      githubCredentialsProvider?: GithubCredentialsProvider;
+      userTransformer?: UserTransformer;
+      teamTransformer?: TeamTransformer;
+    },
+  ) {
     this.integrations = options.integrations;
     this.logger = options.logger;
     this.orgs = options.orgs;
@@ -130,12 +141,22 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
           client,
           orgConfig.name,
           tokenType,
-          orgConfig.userNamespace,
+          this.options.userTransformer,
         );
-        const { groups, groupMemberUsers } = await getOrganizationTeams(
+        const { groups } = await getOrganizationTeams(
           client,
           orgConfig.name,
-          orgConfig.groupNamespace,
+          async (team, ctx): Promise<GroupEntity | undefined> => {
+            const result = this.options.teamTransformer
+              ? await this.options.teamTransformer(team, ctx)
+              : await defaultOrganizationTeamTransformer(team, ctx);
+
+            if (result) {
+              result.metadata.namespace = orgConfig.groupNamespace;
+            }
+
+            return result;
+          },
         );
 
         const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
@@ -151,15 +172,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
             allUsersMap.set(prefix + u.metadata.name, u);
           }
         });
-
-        for (const [groupName, userNames] of groupMemberUsers.entries()) {
-          for (const userName of userNames) {
-            const user = allUsersMap.get(prefix + userName);
-            if (user && !user.spec.memberOf.includes(groupName)) {
-              user.spec.memberOf.push(groupName);
-            }
-          }
-        }
+        assignGroupsToUsers(users, groups);
         buildOrgHierarchy(groups);
 
         for (const group of groups) {

plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts

@@ -101,17 +101,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     this.logger.info('Reading GitHub users and groups');
 
     const { users } = await getOrganizationUsers(client, org, tokenType);
-    const { groups, groupMemberUsers } = await getOrganizationTeams(
-      client,
-      org,
-    );
+    const { groups } = await getOrganizationTeams(client, org);
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
     this.logger.debug(
       `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,
     );
 
-    assignGroupsToUsers(users, groupMemberUsers);
+    assignGroupsToUsers(users, groups);
     buildOrgHierarchy(groups);
 
     // Done!

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts

@@ -37,7 +37,7 @@ import {
   readProviderConfigs,
   GithubEntityProviderConfig,
 } from './GithubEntityProviderConfig';
-import { getOrganizationRepositories, Repository } from '../lib/github';
+import { getOrganizationRepositories, RepositoryResponse } from '../lib/github';
 import { satisfiesTopicFilter } from '../lib/util';
 
 /**
@@ -175,7 +175,7 @@ export class GithubEntityProvider implements EntityProvider {
   }
 
   // go to the server and get all of the repositories
-  private async findCatalogFiles(): Promise<Repository[]> {
+  private async findCatalogFiles(): Promise<RepositoryResponse[]> {
     const organization = this.config.organization;
     const host = this.integration.host;
     const catalogPath = this.config.catalogPath;
@@ -208,7 +208,7 @@ export class GithubEntityProvider implements EntityProvider {
     return repositories;
   }
 
-  private matchesFilters(repositories: Repository[]) {
+  private matchesFilters(repositories: RepositoryResponse[]) {
     const repositoryFilter = this.config.filters?.repository;
     const topicFilters = this.config.filters?.topic;
 
@@ -226,7 +226,7 @@ export class GithubEntityProvider implements EntityProvider {
     return matchingRepositories;
   }
 
-  private createLocationUrl(repository: Repository): string {
+  private createLocationUrl(repository: RepositoryResponse): string {
     const branch =
       this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
     const catalogFile = this.config.catalogPath.startsWith('/')

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts

@@ -163,6 +163,7 @@ describe('GithubOrgEntityProvider', () => {
                   picture: 'http://example.com/team.jpeg',
                 },
                 type: 'team',
+                members: ['a'],
               },
             },
             locationKey: 'github-org-provider:my-id',

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts

@@ -43,6 +43,7 @@ import {
   getOrganizationUsers,
   parseGithubOrgUrl,
 } from '../lib';
+import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers';
 
 /**
  * Options for {@link GithubOrgEntityProvider}.
@@ -88,6 +89,16 @@ export interface GithubOrgEntityProviderOptions {
    * Optionally supply a custom credentials provider, replacing the default one.
    */
   githubCredentialsProvider?: GithubCredentialsProvider;
+
+  /**
+   * Optionally include a user transformer for transforming from GitHub users to User Entities
+   */
+  userTransformer?: UserTransformer;
+
+  /**
+   * Optionally include a team transformer for transforming from GitHub teams to Group Entities
+   */
+  teamTransformer?: TeamTransformer;
 }
 
 // TODO: Consider supporting an (optional) webhook that reacts on org changes
@@ -123,6 +134,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
       githubCredentialsProvider:
         options.githubCredentialsProvider ||
         DefaultGithubCredentialsProvider.fromIntegrations(integrations),
+      userTransformer: options.userTransformer,
+      teamTransformer: options.teamTransformer,
     });
 
     provider.schedule(options.schedule);
@@ -137,6 +150,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
       gitHubConfig: GithubIntegrationConfig;
       logger: Logger;
       githubCredentialsProvider?: GithubCredentialsProvider;
+      userTransformer?: UserTransformer;
+      teamTransformer?: TeamTransformer;
     },
   ) {
     this.credentialsProvider =
@@ -177,12 +192,19 @@ export class GithubOrgEntityProvider implements EntityProvider {
     });
 
     const { org } = parseGithubOrgUrl(this.options.orgUrl);
-    const { users } = await getOrganizationUsers(client, org, tokenType);
-    const { groups, groupMemberUsers } = await getOrganizationTeams(
+    const { users } = await getOrganizationUsers(
       client,
       org,
+      tokenType,
+      this.options.userTransformer,
     );
-    assignGroupsToUsers(users, groupMemberUsers);
+    const { groups } = await getOrganizationTeams(
+      client,
+      org,
+      this.options.teamTransformer,
+    );
+
+    assignGroupsToUsers(users, groups);
     buildOrgHierarchy(groups);
 
     const { markCommitComplete } = markReadComplete({ users, groups });