feat: Enable User and Team transformers in GitHub Provider

Introduces the ability to use custom transformers to generate entities from GitHub users and teams. Meaning you can now transform them however you like as they are being imported.

Signed-off-by: Scott Guymer <scott.guymer@philips.com>

plugins/catalog-backend-module-github/api-report.md

@@ -13,6 +13,8 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GithubCredentialsProvider } from '@backstage/integration';
 import { GithubIntegrationConfig } from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
@@ -21,6 +23,13 @@ import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
 import { TaskRunner } from '@backstage/backend-tasks';
 import { TokenManager } from '@backstage/backend-common';
+import { UserEntity } from '@backstage/catalog-model';
+
+// @public
+export const defaultOrganizationTeamTransformer: TeamTransformer;
+
+// @public
+export const defaultUserTransformer: UserTransformer;
 
 // @public
 export class GithubDiscoveryProcessor implements CatalogProcessor {
@@ -165,6 +174,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
     gitHubConfig: GithubIntegrationConfig;
     logger: Logger;
     githubCredentialsProvider?: GithubCredentialsProvider;
+    userTransformer?: UserTransformer;
+    teamTransformer?: TeamTransformer;
   });
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;
@@ -188,6 +199,8 @@ export interface GithubOrgEntityProviderOptions {
   logger: Logger;
   orgUrl: string;
   schedule?: 'manual' | TaskRunner;
+  teamTransformer?: TeamTransformer;
+  userTransformer?: UserTransformer;
 }
 
 // @public
@@ -214,4 +227,48 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     emit: CatalogProcessorEmit,
   ): Promise<boolean>;
 }
+
+// @public
+export type GithubTeam = {
+  slug: string;
+  combinedSlug: string;
+  name?: string;
+  description?: string;
+  avatarUrl?: string;
+  editTeamUrl?: string;
+  parentTeam?: GithubTeam;
+  members: GithubUser[];
+};
+
+// @public
+export type GithubUser = {
+  login: string;
+  bio?: string;
+  avatarUrl?: string;
+  email?: string;
+  name?: string;
+  organizationVerifiedDomainEmails?: string[];
+};
+
+// @public
+export type TeamTransformer = (
+  item: GithubTeam,
+  ctx: TransformerContext,
+) => Promise<GroupEntity | undefined>;
+
+// @public
+export interface TransformerContext {
+  // (undocumented)
+  client: typeof graphql;
+  // (undocumented)
+  org: string;
+  // (undocumented)
+  query: string;
+}
+
+// @public
+export type UserTransformer = (
+  item: GithubUser,
+  ctx: TransformerContext,
+) => Promise<UserEntity | undefined>;
 ```

plugins/catalog-backend-module-github/src/index.ts

@@ -15,20 +15,31 @@
  */
 
 /**
- * A Backstage catalog backend module that helps integrate towards GitHub
+ * A Backstage catalog backend module that helps integrate towards Github
  *
  * @packageDocumentation
  */
 
 export { GithubLocationAnalyzer } from './analyzers/GithubLocationAnalyzer';
 export type { GithubLocationAnalyzerOptions } from './analyzers/GithubLocationAnalyzer';
-export type { GithubMultiOrgConfig } from './lib';
 export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
 export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
 export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
 export { GithubEntityProvider } from './providers/GithubEntityProvider';
-export { GithubOrgEntityProvider } from './providers/GithubOrgEntityProvider';
-export type { GithubOrgEntityProviderOptions } from './providers/GithubOrgEntityProvider';
+export type {
+  GithubOrgEntityProvider,
+  GithubOrgEntityProviderOptions,
+} from './providers/GithubOrgEntityProvider';
 export { githubEntityProviderCatalogModule } from './service/GithubEntityProviderCatalogModule';
+export {
+  type GithubMultiOrgConfig,
+  type GithubTeam,
+  type GithubUser,
+  type UserTransformer,
+  defaultUserTransformer,
+  type TeamTransformer,
+  defaultOrganizationTeamTransformer,
+  type TransformerContext,
+} from './lib';
 
 export * from './deprecated';

plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { graphql } from '@octokit/graphql';
+import { GithubTeam, GithubUser } from './github';
+
+/**
+ * Context passed to Transformers
+ *
+ * @public
+ */
+export interface TransformerContext {
+  client: typeof graphql;
+  query: string;
+  org: string;
+}
+
+/**
+ * Transformer for GitHub users to UserEntity
+ *
+ * @public
+ */
+export type UserTransformer = (
+  item: GithubUser,
+  ctx: TransformerContext,
+) => Promise<UserEntity | undefined>;
+
+/**
+ * Transformer for GitHub Team to GroupEntity
+ *
+ * @public
+ */
+export type TeamTransformer = (
+  item: GithubTeam,
+  ctx: TransformerContext,
+) => Promise<GroupEntity | undefined>;
+
+/**
+ * Default transformer for GitHub users to UserEntity
+ *
+ * @public
+ */
+export const defaultUserTransformer: UserTransformer = async (
+  item: GithubUser,
+) => {
+  const entity: UserEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: {
+      name: item.login,
+      annotations: {
+        'github.com/user-login': item.login,
+      },
+    },
+    spec: {
+      profile: {},
+      memberOf: [],
+    },
+  };
+
+  if (item.bio) entity.metadata.description = item.bio;
+  if (item.name) entity.spec.profile!.displayName = item.name;
+  if (item.email) entity.spec.profile!.email = item.email;
+  if (item.avatarUrl) entity.spec.profile!.picture = item.avatarUrl;
+  return entity;
+};
+
+/**
+ * Default transformer for GitHub Team to GroupEntity
+ *
+ * @public
+ */
+export const defaultOrganizationTeamTransformer: TeamTransformer =
+  async team => {
+    const annotations: { [annotationName: string]: string } = {
+      'github.com/team-slug': team.combinedSlug,
+    };
+
+    if (team.editTeamUrl) {
+      annotations['backstage.io/edit-url'] = team.editTeamUrl;
+    }
+
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: team.slug,
+        annotations,
+      },
+      spec: {
+        type: 'team',
+        profile: {},
+        children: [],
+      },
+    };
+
+    if (team.description) {
+      entity.metadata.description = team.description;
+    }
+    if (team.name) {
+      entity.spec.profile!.displayName = team.name;
+    }
+    if (team.avatarUrl) {
+      entity.spec.profile!.picture = team.avatarUrl;
+    }
+    if (team.parentTeam) {
+      entity.spec.parent = team.parentTeam.slug;
+    }
+
+    entity.spec.members = team.members.map(user => user.login);
+
+    return entity;
+  };

plugins/catalog-backend-module-github/src/lib/github.test.ts

@@ -15,15 +15,20 @@
  */
 
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { graphql } from '@octokit/graphql';
 import { graphql as graphqlMsw } from 'msw';
 import { setupServer } from 'msw/node';
+import { TeamTransformer, UserTransformer } from './defaultTransformers';
+
 import {
   getOrganizationTeams,
   getOrganizationUsers,
   getTeamMembers,
   getOrganizationRepositories,
   QueryResponse,
+  GithubUser,
+  GithubTeam,
 } from './github';
 import fetch from 'node-fetch';
 
@@ -35,7 +40,7 @@ describe('github', () => {
   const server = setupServer();
   setupRequestMockHandlers(server);
 
-  describe('getOrganizationUsers', () => {
+  describe('getOrganizationUsers using defaultUserMapper', () => {
     it('reads members', async () => {
       const input: QueryResponse = {
         organization: {
@@ -76,7 +81,120 @@ describe('github', () => {
     });
   });
 
-  describe('getOrganizationTeams', () => {
+  describe('getOrganizationUsers using custom UserTransformer', () => {
+    const customUserTransformer: UserTransformer = async (
+      item: GithubUser,
+      {},
+    ) => {
+      if (item.login === 'aa') {
+        return undefined;
+      }
+
+      return {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: `${item.login}-custom`,
+          annotations: {
+            'github.com/user-login': item.login,
+          },
+        },
+        spec: {
+          profile: {},
+          memberOf: [],
+        },
+      } as UserEntity;
+    };
+
+    it('reads members', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'a-custom',
+            }),
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(
+        getOrganizationUsers(graphql, 'a', 'token', customUserTransformer),
+      ).resolves.toEqual(output);
+    });
+
+    it('reads members if undefined is returned from transformer', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+              {
+                login: 'aa',
+                name: 'bb',
+                bio: 'cc',
+                email: 'dd',
+                avatarUrl: 'ee',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'a-custom',
+            }),
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      const users = await getOrganizationUsers(
+        graphql,
+        'a',
+        'token',
+        customUserTransformer,
+      );
+
+      expect(users.users).toHaveLength(1);
+      expect(users).toEqual(output);
+    });
+  });
+
+  describe('getOrganizationTeams using default TeamTransformer', () => {
     let input: QueryResponse;
 
     beforeEach(() => {
@@ -95,7 +213,7 @@ describe('github', () => {
                 parentTeam: {
                   slug: 'parent',
                   combinedSlug: '',
-                  members: { pageInfo: { hasNextPage: false }, nodes: [] },
+                  members: [],
                 },
                 members: {
                   pageInfo: { hasNextPage: false },
@@ -129,10 +247,10 @@ describe('github', () => {
               },
               parent: 'parent',
               children: [],
+              members: ['user'],
             },
           }),
         ],
-        groupMemberUsers: new Map([['team', ['user']]]),
       };
 
       server.use(
@@ -141,37 +259,192 @@ describe('github', () => {
 
       await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
     });
+  });
 
-    it('applies namespaces', async () => {
+  describe('getOrganizationTeams using custom TeamTransformer', () => {
+    let input: QueryResponse;
+
+    const customTeamTransformer: TeamTransformer = async (
+      item: GithubTeam,
+      {},
+    ) => {
+      if (item.name === 'aa') {
+        return undefined;
+      }
+
+      return {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: `${item.name}-custom`,
+          annotations: {
+            'github.com/team-slug': 'blah/team',
+            'backstage.io/edit-url':
+              'http://example.com/orgs/blah/teams/team/edit',
+          },
+          description: item.description,
+        },
+        spec: {
+          type: 'team',
+          profile: {
+            displayName: `${item.name}-custom`,
+            picture: 'http://example.com/team.jpeg',
+          },
+          parent: 'parent',
+          children: [],
+          members: ['user'],
+        },
+      } as GroupEntity;
+    };
+
+    beforeEach(() => {
+      input = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'Team',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+    });
+
+    it('reads teams', async () => {
       const output = {
         groups: [
           expect.objectContaining({
             metadata: expect.objectContaining({
-              name: 'team',
-              namespace: 'foo',
+              name: 'Team-custom',
               description: 'The one and only team',
+              annotations: {
+                'github.com/team-slug': 'blah/team',
+                'backstage.io/edit-url':
+                  'http://example.com/orgs/blah/teams/team/edit',
+              },
             }),
             spec: {
               type: 'team',
               profile: {
-                displayName: 'Team',
+                displayName: 'Team-custom',
                 picture: 'http://example.com/team.jpeg',
               },
               parent: 'parent',
               children: [],
+              members: ['user'],
             },
           }),
         ],
-        groupMemberUsers: new Map([['foo/team', ['user']]]),
       };
 
       server.use(
         graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
       );
 
-      await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual(
-        output,
+      await expect(
+        getOrganizationTeams(graphql, 'a', customTeamTransformer),
+      ).resolves.toEqual(output);
+    });
+
+    it('reads teams if undefined is returned', async () => {
+      input = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'Team',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'aa',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        groups: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'Team-custom',
+              description: 'The one and only team',
+              annotations: {
+                'github.com/team-slug': 'blah/team',
+                'backstage.io/edit-url':
+                  'http://example.com/orgs/blah/teams/team/edit',
+              },
+            }),
+            spec: {
+              type: 'team',
+              profile: {
+                displayName: 'Team-custom',
+                picture: 'http://example.com/team.jpeg',
+              },
+              parent: 'parent',
+              children: [],
+              members: ['user'],
+            },
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
       );
+
+      const teams = await getOrganizationTeams(
+        graphql,
+        'a',
+        customTeamTransformer,
+      );
+
+      expect(teams.groups).toHaveLength(1);
+      expect(teams).toEqual(output);
     });
   });
 
@@ -191,7 +464,7 @@ describe('github', () => {
       };
 
       const output = {
-        members: ['user'],
+        members: [{ login: 'user' }],
       };
 
       server.use(

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -17,19 +17,30 @@
 import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { GithubCredentialType } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
+import {
+  defaultOrganizationTeamTransformer,
+  defaultUserTransformer,
+  TeamTransformer,
+  TransformerContext,
+  UserTransformer,
+} from './defaultTransformers';
 
 // Graphql types
 
 export type QueryResponse = {
-  organization?: Organization;
-  repositoryOwner?: Organization | User;
+  organization?: OrganizationResponse;
+  repositoryOwner?: RepositoryOwnerResponse;
 };
 
-export type Organization = {
-  membersWithRole?: Connection<User>;
-  team?: Team;
-  teams?: Connection<Team>;
-  repositories?: Connection<Repository>;
+type RepositoryOwnerResponse = {
+  repositories?: Connection<RepositoryResponse>;
+};
+
+export type OrganizationResponse = {
+  membersWithRole?: Connection<GithubUser>;
+  team?: GithubTeamResponse;
+  teams?: Connection<GithubTeamResponse>;
+  repositories?: Connection<RepositoryResponse>;
 };
 
 export type PageInfo = {
@@ -37,27 +48,41 @@ export type PageInfo = {
   endCursor?: string;
 };
 
-export type User = {
+/**
+ * Github User
+ *
+ * @public
+ */
+export type GithubUser = {
   login: string;
   bio?: string;
   avatarUrl?: string;
   email?: string;
   name?: string;
-  repositories?: Connection<Repository>;
+  organizationVerifiedDomainEmails?: string[];
 };
 
-export type Team = {
+/**
+ * Github Team
+ *
+ * @public
+ */
+export type GithubTeam = {
   slug: string;
   combinedSlug: string;
   name?: string;
   description?: string;
   avatarUrl?: string;
   editTeamUrl?: string;
-  parentTeam?: Team;
-  members: Connection<User>;
+  parentTeam?: GithubTeam;
+  members: GithubUser[];
 };
 
-export type Repository = {
+export type GithubTeamResponse = Omit<GithubTeam, 'members'> & {
+  members: Connection<GithubUser>;
+};
+
+export type RepositoryResponse = {
   name: string;
   url: string;
   isArchived: boolean;
@@ -88,7 +113,7 @@ export type Connection<T> = {
 };
 
 /**
- * Gets all the users out of a GitHub organization.
+ * Gets all the users out of a Github organization.
  *
  * Note that the users will not have their memberships filled in.
  *
@@ -99,7 +124,7 @@ export async function getOrganizationUsers(
   client: typeof graphql,
   org: string,
   tokenType: GithubCredentialType,
-  userNamespace?: string,
+  userTransformer: UserTransformer = defaultUserTransformer,
 ): Promise<{ users: UserEntity[] }> {
   const query = `
     query users($org: String!, $email: Boolean!, $cursor: String) {
@@ -111,7 +136,8 @@ export async function getOrganizationUsers(
             bio,
             email @include(if: $email),
             login,
-            name
+            name,
+            organizationVerifiedDomainEmails(login: $org)
           }
         }
       }
@@ -119,44 +145,24 @@ export async function getOrganizationUsers(
 
   // There is no user -> teams edge, so we leave the memberships empty for
   // now and let the team iteration handle it instead
-  const mapper = (user: User) => {
-    const entity: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: {
-        name: user.login,
-        annotations: {
-          'github.com/user-login': user.login,
-        },
-      },
-      spec: {
-        profile: {},
-        memberOf: [],
-      },
-    };
-
-    if (userNamespace) entity.metadata.namespace = userNamespace;
-    if (user.bio) entity.metadata.description = user.bio;
-    if (user.name) entity.spec.profile!.displayName = user.name;
-    if (user.email) entity.spec.profile!.email = user.email;
-    if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
-
-    return entity;
-  };
 
   const users = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.membersWithRole,
-    mapper,
-    { org, email: tokenType === 'token' },
+    userTransformer,
+    {
+      org,
+      email: tokenType === 'token',
+    },
   );
 
   return { users };
 }
 
 /**
- * Gets all the teams out of a GitHub organization.
+ * Gets all the teams out of a Github organization.
  *
  * Note that the teams will not have any relations apart from parent filled in.
  *
@@ -166,10 +172,9 @@ export async function getOrganizationUsers(
 export async function getOrganizationTeams(
   client: typeof graphql,
   org: string,
-  orgNamespace?: string,
+  teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
 ): Promise<{
   groups: GroupEntity[];
-  groupMemberUsers: Map<string, string[]>;
 }> {
   const query = `
     query teams($org: String!, $cursor: String) {
@@ -193,86 +198,51 @@ export async function getOrganizationTeams(
       }
     }`;
 
-  // Gets populated inside the mapper below
-  const groupMemberUsers = new Map<string, string[]>();
+  const materialisedTeams = async (
+    item: GithubTeamResponse,
+    ctx: TransformerContext,
+  ): Promise<GroupEntity | undefined> => {
+    const memberNames: GithubUser[] = [];
 
-  const mapper = async (team: Team) => {
-    const annotations: { [annotationName: string]: string } = {
-      'github.com/team-slug': team.combinedSlug,
-    };
-
-    if (team.editTeamUrl) {
-      annotations['backstage.io/edit-url'] = team.editTeamUrl;
-    }
-
-    const entity: GroupEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Group',
-      metadata: {
-        name: team.slug,
-        annotations,
-      },
-      spec: {
-        type: 'team',
-        profile: {},
-        children: [],
-      },
-    };
-
-    if (orgNamespace) {
-      entity.metadata.namespace = orgNamespace;
-    }
-
-    if (team.description) {
-      entity.metadata.description = team.description;
-    }
-    if (team.name) {
-      entity.spec.profile!.displayName = team.name;
-    }
-    if (team.avatarUrl) {
-      entity.spec.profile!.picture = team.avatarUrl;
-    }
-    if (team.parentTeam) {
-      entity.spec.parent = team.parentTeam.slug;
-    }
-
-    const memberNames: string[] = [];
-    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;
-    groupMemberUsers.set(groupKey, memberNames);
-
-    if (!team.members.pageInfo.hasNextPage) {
+    if (!item.members.pageInfo.hasNextPage) {
       // We got all the members in one go, run the fast path
-      for (const user of team.members.nodes) {
-        memberNames.push(user.login);
+      for (const user of item.members.nodes) {
+        memberNames.push(user);
       }
     } else {
       // There were more than a hundred immediate members - run the slow
       // path of fetching them explicitly
-      const { members } = await getTeamMembers(client, org, team.slug);
+      const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
       for (const userLogin of members) {
         memberNames.push(userLogin);
       }
     }
 
-    return entity;
+    const team: GithubTeam = {
+      ...item,
+      members: memberNames,
+    };
+
+    return await teamTransformer(team, ctx);
   };
 
   const groups = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.teams,
-    mapper,
+    materialisedTeams,
     { org },
   );
 
-  return { groups, groupMemberUsers };
+  return { groups };
 }
 
 export async function getOrganizationRepositories(
   client: typeof graphql,
   org: string,
   catalogPath: string,
-): Promise<{ repositories: Repository[] }> {
+): Promise<{ repositories: RepositoryResponse[] }> {
   let relativeCatalogPathRef: string;
   // We must strip the leading slash or the query for objects does not work
   if (catalogPath.startsWith('/')) {
@@ -321,8 +291,9 @@ export async function getOrganizationRepositories(
   const repositories = await queryWithPaging(
     client,
     query,
+    org,
     r => r.repositoryOwner?.repositories,
-    x => x,
+    async x => x,
     { org, catalogPathRef },
   );
 
@@ -330,7 +301,7 @@ export async function getOrganizationRepositories(
 }
 
 /**
- * Gets all the users out of a GitHub organization.
+ * Gets all the users out of a Github organization.
  *
  * Note that the users will not have their memberships filled in.
  *
@@ -342,7 +313,7 @@ export async function getTeamMembers(
   client: typeof graphql,
   org: string,
   teamSlug: string,
-): Promise<{ members: string[] }> {
+): Promise<{ members: GithubUser[] }> {
   const query = `
     query members($org: String!, $teamSlug: String!, $cursor: String) {
       organization(login: $org) {
@@ -358,8 +329,9 @@ export async function getTeamMembers(
   const members = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.team?.members,
-    user => user.login,
+    async user => user,
     { org, teamSlug },
   );
 
@@ -379,7 +351,7 @@ export async function getTeamMembers(
  * @param query - The query to execute
  * @param connection - A function that, given the response, picks out the actual
  *                   Connection object that's being iterated
- * @param mapper - A function that, given one of the nodes in the Connection,
+ * @param transformer - A function that, given one of the nodes in the Connection,
  *               returns the model mapped form of it
  * @param variables - The variable values that the query needs, minus the cursor
  */
@@ -391,8 +363,12 @@ export async function queryWithPaging<
 >(
   client: typeof graphql,
   query: string,
+  org: string,
   connection: (response: Response) => Connection<GraphqlType> | undefined,
-  mapper: (item: GraphqlType) => Promise<OutputType> | OutputType,
+  transformer: (
+    item: GraphqlType,
+    ctx: TransformerContext,
+  ) => Promise<OutputType | undefined>,
   variables: Variables,
 ): Promise<OutputType[]> {
   const result: OutputType[] = [];
@@ -410,7 +386,15 @@ export async function queryWithPaging<
     }
 
     for (const node of conn.nodes) {
-      result.push(await mapper(node));
+      const transformedNode = await transformer(node, {
+        client,
+        query,
+        org,
+      });
+
+      if (transformedNode) {
+        result.push(transformedNode);
+      }
     }
 
     if (!conn.pageInfo.hasNextPage) {

plugins/catalog-backend-module-github/src/lib/index.ts

@@ -20,6 +20,15 @@ export {
   getOrganizationRepositories,
   getOrganizationTeams,
   getOrganizationUsers,
+  type GithubUser,
+  type GithubTeam,
 } from './github';
+export {
+  type UserTransformer,
+  defaultUserTransformer,
+  type TeamTransformer,
+  defaultOrganizationTeamTransformer,
+  type TransformerContext,
+} from './defaultTransformers';
 export { assignGroupsToUsers, buildOrgHierarchy } from './org';
 export { parseGithubOrgUrl } from './util';

plugins/catalog-backend-module-github/src/lib/org.test.ts

@@ -68,13 +68,47 @@ describe('buildOrgHierarchy', () => {
 describe('assignGroupsToUsers', () => {
   it('should assign groups to users', () => {
     const users: UserEntity[] = [u('u1'), u('u2')];
-    const groupMemberUsers = new Map<string, string[]>([
-      ['g1', ['u1', 'u2']],
-      ['g2', ['u2']],
-      ['g3', ['u3']],
-    ]);
 
-    assignGroupsToUsers(users, groupMemberUsers);
+    const groups: GroupEntity[] = [
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g1',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u1', 'u2'],
+        },
+      },
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g2',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u2'],
+        },
+      },
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g3',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u3'],
+        },
+      },
+    ];
+
+    assignGroupsToUsers(users, groups);
 
     expect(users[0].spec.memberOf).toEqual(['g1']);
     expect(users[1].spec.memberOf).toEqual(['g1', 'g2']);

plugins/catalog-backend-module-github/src/lib/org.ts

@@ -52,8 +52,17 @@ export function buildOrgHierarchy(groups: GroupEntity[]) {
 // Ensure that users have their direct group memberships.
 export function assignGroupsToUsers(
   users: UserEntity[],
-  groupMemberUsers: Map<string, string[]>,
+  groups: GroupEntity[],
 ) {
+  const groupMemberUsers = new Map(
+    groups.map(group => {
+      const groupKey = group.metadata.namespace
+        ? `${group.metadata.namespace}/${group.metadata.name}`
+        : group.metadata.name;
+      return [groupKey, group.spec.members || []];
+    }),
+  );
+
   const usersByName = new Map(users.map(u => [u.metadata.name, u]));
   for (const [groupName, userNames] of groupMemberUsers.entries()) {
     for (const userName of userNames) {

plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { GroupEntity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import {
   DefaultGithubCredentialsProvider,
@@ -32,7 +33,9 @@ import {
 import { graphql } from '@octokit/graphql';
 import { Logger } from 'winston';
 import {
+  assignGroupsToUsers,
   buildOrgHierarchy,
+  defaultOrganizationTeamTransformer,
   getOrganizationTeams,
   getOrganizationUsers,
   GithubMultiOrgConfig,
@@ -130,12 +133,19 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
           client,
           orgConfig.name,
           tokenType,
-          orgConfig.userNamespace,
         );
-        const { groups, groupMemberUsers } = await getOrganizationTeams(
+        const { groups } = await getOrganizationTeams(
           client,
           orgConfig.name,
-          orgConfig.groupNamespace,
+          async (team, ctx): Promise<GroupEntity | undefined> => {
+            const result = await defaultOrganizationTeamTransformer(team, ctx);
+
+            if (result) {
+              result.metadata.namespace = orgConfig.groupNamespace;
+            }
+
+            return result;
+          },
         );
 
         const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
@@ -151,15 +161,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
             allUsersMap.set(prefix + u.metadata.name, u);
           }
         });
-
-        for (const [groupName, userNames] of groupMemberUsers.entries()) {
-          for (const userName of userNames) {
-            const user = allUsersMap.get(prefix + userName);
-            if (user && !user.spec.memberOf.includes(groupName)) {
-              user.spec.memberOf.push(groupName);
-            }
-          }
-        }
+        assignGroupsToUsers(users, groups);
         buildOrgHierarchy(groups);
 
         for (const group of groups) {

plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts

@@ -101,17 +101,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     this.logger.info('Reading GitHub users and groups');
 
     const { users } = await getOrganizationUsers(client, org, tokenType);
-    const { groups, groupMemberUsers } = await getOrganizationTeams(
-      client,
-      org,
-    );
+    const { groups } = await getOrganizationTeams(client, org);
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
     this.logger.debug(
       `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,
     );
 
-    assignGroupsToUsers(users, groupMemberUsers);
+    assignGroupsToUsers(users, groups);
     buildOrgHierarchy(groups);
 
     // Done!

plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts

@@ -43,6 +43,7 @@ import {
   getOrganizationUsers,
   parseGithubOrgUrl,
 } from '../lib';
+import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers';
 
 /**
  * Options for {@link GithubOrgEntityProvider}.
@@ -88,6 +89,16 @@ export interface GithubOrgEntityProviderOptions {
    * Optionally supply a custom credentials provider, replacing the default one.
    */
   githubCredentialsProvider?: GithubCredentialsProvider;
+
+  /**
+   * Optionally include a user transformer for transforming from GitHub users to User Entities
+   */
+  userTransformer?: UserTransformer;
+
+  /**
+   * Optionally include a user transformer for transforming from GitHub users to User Entities
+   */
+  teamTransformer?: TeamTransformer;
 }
 
 // TODO: Consider supporting an (optional) webhook that reacts on org changes
@@ -123,6 +134,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
       githubCredentialsProvider:
         options.githubCredentialsProvider ||
         DefaultGithubCredentialsProvider.fromIntegrations(integrations),
+      userTransformer: options.userTransformer,
+      teamTransformer: options.teamTransformer,
     });
 
     provider.schedule(options.schedule);
@@ -137,6 +150,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
       gitHubConfig: GithubIntegrationConfig;
       logger: Logger;
       githubCredentialsProvider?: GithubCredentialsProvider;
+      userTransformer?: UserTransformer;
+      teamTransformer?: TeamTransformer;
     },
   ) {
     this.credentialsProvider =
@@ -177,12 +192,19 @@ export class GithubOrgEntityProvider implements EntityProvider {
     });
 
     const { org } = parseGithubOrgUrl(this.options.orgUrl);
-    const { users } = await getOrganizationUsers(client, org, tokenType);
-    const { groups, groupMemberUsers } = await getOrganizationTeams(
+    const { users } = await getOrganizationUsers(
       client,
       org,
+      tokenType,
+      this.options.userTransformer,
     );
-    assignGroupsToUsers(users, groupMemberUsers);
+    const { groups } = await getOrganizationTeams(
+      client,
+      org,
+      this.options.teamTransformer,
+    );
+
+    assignGroupsToUsers(users, groups);
     buildOrgHierarchy(groups);
 
     const { markCommitComplete } = markReadComplete({ users, groups });

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts

@@ -37,7 +37,7 @@ import {
   readProviderConfigs,
   GithubEntityProviderConfig,
 } from './GithubEntityProviderConfig';
-import { getOrganizationRepositories, Repository } from '../lib/github';
+import { getOrganizationRepositories, RepositoryResponse } from '../lib/github';
 import { satisfiesTopicFilter } from '../lib/util';
 
 /**
@@ -175,7 +175,7 @@ export class GithubEntityProvider implements EntityProvider {
   }
 
   // go to the server and get all of the repositories
-  private async findCatalogFiles(): Promise<Repository[]> {
+  private async findCatalogFiles(): Promise<RepositoryResponse[]> {
     const organization = this.config.organization;
     const host = this.integration.host;
     const catalogPath = this.config.catalogPath;
@@ -208,7 +208,7 @@ export class GithubEntityProvider implements EntityProvider {
     return repositories;
   }
 
-  private matchesFilters(repositories: Repository[]) {
+  private matchesFilters(repositories: RepositoryResponse[]) {
     const repositoryFilter = this.config.filters?.repository;
     const topicFilters = this.config.filters?.topic;
 
@@ -226,7 +226,7 @@ export class GithubEntityProvider implements EntityProvider {
     return matchingRepositories;
   }
 
-  private createLocationUrl(repository: Repository): string {
+  private createLocationUrl(repository: RepositoryResponse): string {
     const branch =
       this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
     const catalogFile = this.config.catalogPath.startsWith('/')

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts

@@ -163,6 +163,7 @@ describe('GithubOrgEntityProvider', () => {
                   picture: 'http://example.com/team.jpeg',
                 },
                 type: 'team',
+                members: ['a'],
               },
             },
             locationKey: 'github-org-provider:my-id',