feat: Enable User and Team transformers in GitHub Provider

Introduces the ability to use custom transformers to generate entities from GitHub users and teams. Meaning you can now transform them however you like as they are being imported.

Signed-off-by: Scott Guymer <scott.guymer@philips.com>

.changeset/proud-queens-warn.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-github': patch
+---
+
+- Make it possible to inject custom user and team transformers when configuring the `GithubOrgEntityProvider`

docs/integrations/github/org.md

@@ -97,6 +97,66 @@ that must be approved first before the changes are applied.**
 
 ![email](../../assets/integrations/github/email.png)
 
+### Custom Transformers
+
+You can inject your own transformation logic to help map from GH API responses
+into backstage entities. You can do this on the user and team requests to
+enable you to do further processing or updates to the entities.
+
+To enable this you pass a function into the `GitHubOrgEntityProvider`. You can
+pass a `UserTransformer`, `TeamTransformer` or both. The function is invoked
+for each item (user or team) that is returned from the API. You can either
+return an Entity (User or Group) or `undefined` if you do not want to import
+that item.
+
+There is also a `defaultUserTransformer` and `defaultOrganizationTeamTransformer`.
+You could use these and simply decorate the response from the default
+transformation if you only need to change a few properties.
+
+### Resolving GitHub users via organization email
+
+When you authenticate users you should resolve them to an entity within the
+catalog. Often the authentication you use could be a corporate SSO system that
+provides you with email as a key. To enable you to find and resolve GitHub users
+it's useful to also import the private domain verified emails into the User
+entity in backstage.
+
+The integration attempts to return `organizationVerifiedDomainEmails` from the
+GitHub API and makes this available as part of the object passed to
+`UserTransformer`. The GitHub API will only return emails that use a domain
+that's a verified domain for your GitHub Org. It also relies on the user having
+configured such an email in their own account. The API will only return these
+values when using GitHub App authentication and with the correct app permission
+allowing access to emails.
+
+You can decorate the `defaultUserTransformer` to replace the org email in the
+returned identity.
+
+```typescript
+async (user, ctx): Promise<UserEntity | undefined> => {
+  const entity = await defaultUserTransformer(user, ctx);
+
+  if (entity && user.organizationVerifiedDomainEmails) {
+    entity.spec.profile!.email = user.organizationVerifiedDomainEmails[0] || '';
+  }
+
+  return entity;
+},
+```
+
+Once you have imported the emails you can resolve users in your sign-in in
+resolver using the catalog entity search via email
+
+```typescript
+// packages/backend/src/plugins/auth.ts
+ctx.signInWithCatalogUser({
+  filter: {
+    kind: ['User'],
+    'spec.profile.email': email as string,
+  },
+});
+```
+
 ## Using a Processor instead of a Provider
 
 An alternative to using the Provider for ingesting organizational entities is to

plugins/catalog-backend-module-github/api-report.md

@@ -13,6 +13,8 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GithubCredentialsProvider } from '@backstage/integration';
 import { GithubIntegrationConfig } from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { GroupEntity } from '@backstage/catalog-model';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
@@ -21,6 +23,13 @@ import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
 import { TaskRunner } from '@backstage/backend-tasks';
 import { TokenManager } from '@backstage/backend-common';
+import { UserEntity } from '@backstage/catalog-model';
+
+// @public
+export const defaultOrganizationTeamTransformer: TeamTransformer;
+
+// @public
+export const defaultUserTransformer: UserTransformer;
 
 // @public
 export class GithubDiscoveryProcessor implements CatalogProcessor {
@@ -165,6 +174,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
     gitHubConfig: GithubIntegrationConfig;
     logger: Logger;
     githubCredentialsProvider?: GithubCredentialsProvider;
+    userTransformer?: UserTransformer;
+    teamTransformer?: TeamTransformer;
   });
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;
@@ -188,6 +199,8 @@ export interface GithubOrgEntityProviderOptions {
   logger: Logger;
   orgUrl: string;
   schedule?: 'manual' | TaskRunner;
+  teamTransformer?: TeamTransformer;
+  userTransformer?: UserTransformer;
 }
 
 // @public
@@ -214,4 +227,48 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     emit: CatalogProcessorEmit,
   ): Promise<boolean>;
 }
+
+// @public
+export type GithubTeam = {
+  slug: string;
+  combinedSlug: string;
+  name?: string;
+  description?: string;
+  avatarUrl?: string;
+  editTeamUrl?: string;
+  parentTeam?: GithubTeam;
+  members: GithubUser[];
+};
+
+// @public
+export type GithubUser = {
+  login: string;
+  bio?: string;
+  avatarUrl?: string;
+  email?: string;
+  name?: string;
+  organizationVerifiedDomainEmails?: string[];
+};
+
+// @public
+export type TeamTransformer = (
+  item: GithubTeam,
+  ctx: TransformerContext,
+) => Promise<GroupEntity | undefined>;
+
+// @public
+export interface TransformerContext {
+  // (undocumented)
+  client: typeof graphql;
+  // (undocumented)
+  org: string;
+  // (undocumented)
+  query: string;
+}
+
+// @public
+export type UserTransformer = (
+  item: GithubUser,
+  ctx: TransformerContext,
+) => Promise<UserEntity | undefined>;
 ```

plugins/catalog-backend-module-github/src/index.ts

@@ -15,20 +15,31 @@
  */
 
 /**
- * A Backstage catalog backend module that helps integrate towards GitHub
+ * A Backstage catalog backend module that helps integrate towards Github
  *
  * @packageDocumentation
  */
 
 export { GithubLocationAnalyzer } from './analyzers/GithubLocationAnalyzer';
 export type { GithubLocationAnalyzerOptions } from './analyzers/GithubLocationAnalyzer';
-export type { GithubMultiOrgConfig } from './lib';
 export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
 export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
 export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
 export { GithubEntityProvider } from './providers/GithubEntityProvider';
-export { GithubOrgEntityProvider } from './providers/GithubOrgEntityProvider';
-export type { GithubOrgEntityProviderOptions } from './providers/GithubOrgEntityProvider';
+export type {
+  GithubOrgEntityProvider,
+  GithubOrgEntityProviderOptions,
+} from './providers/GithubOrgEntityProvider';
 export { githubEntityProviderCatalogModule } from './service/GithubEntityProviderCatalogModule';
+export {
+  type GithubMultiOrgConfig,
+  type GithubTeam,
+  type GithubUser,
+  type UserTransformer,
+  defaultUserTransformer,
+  type TeamTransformer,
+  defaultOrganizationTeamTransformer,
+  type TransformerContext,
+} from './lib';
 
 export * from './deprecated';

plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { graphql } from '@octokit/graphql';
+import { GithubTeam, GithubUser } from './github';
+
+/**
+ * Context passed to Transformers
+ *
+ * @public
+ */
+export interface TransformerContext {
+  client: typeof graphql;
+  query: string;
+  org: string;
+}
+
+/**
+ * Transformer for GitHub users to UserEntity
+ *
+ * @public
+ */
+export type UserTransformer = (
+  item: GithubUser,
+  ctx: TransformerContext,
+) => Promise<UserEntity | undefined>;
+
+/**
+ * Transformer for GitHub Team to GroupEntity
+ *
+ * @public
+ */
+export type TeamTransformer = (
+  item: GithubTeam,
+  ctx: TransformerContext,
+) => Promise<GroupEntity | undefined>;
+
+/**
+ * Default transformer for GitHub users to UserEntity
+ *
+ * @public
+ */
+export const defaultUserTransformer: UserTransformer = async (
+  item: GithubUser,
+) => {
+  const entity: UserEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: {
+      name: item.login,
+      annotations: {
+        'github.com/user-login': item.login,
+      },
+    },
+    spec: {
+      profile: {},
+      memberOf: [],
+    },
+  };
+
+  if (item.bio) entity.metadata.description = item.bio;
+  if (item.name) entity.spec.profile!.displayName = item.name;
+  if (item.email) entity.spec.profile!.email = item.email;
+  if (item.avatarUrl) entity.spec.profile!.picture = item.avatarUrl;
+  return entity;
+};
+
+/**
+ * Default transformer for GitHub Team to GroupEntity
+ *
+ * @public
+ */
+export const defaultOrganizationTeamTransformer: TeamTransformer =
+  async team => {
+    const annotations: { [annotationName: string]: string } = {
+      'github.com/team-slug': team.combinedSlug,
+    };
+
+    if (team.editTeamUrl) {
+      annotations['backstage.io/edit-url'] = team.editTeamUrl;
+    }
+
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: team.slug,
+        annotations,
+      },
+      spec: {
+        type: 'team',
+        profile: {},
+        children: [],
+      },
+    };
+
+    if (team.description) {
+      entity.metadata.description = team.description;
+    }
+    if (team.name) {
+      entity.spec.profile!.displayName = team.name;
+    }
+    if (team.avatarUrl) {
+      entity.spec.profile!.picture = team.avatarUrl;
+    }
+    if (team.parentTeam) {
+      entity.spec.parent = team.parentTeam.slug;
+    }
+
+    entity.spec.members = team.members.map(user => user.login);
+
+    return entity;
+  };

plugins/catalog-backend-module-github/src/lib/github.test.ts

@@ -15,15 +15,20 @@
  */
 
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { graphql } from '@octokit/graphql';
 import { graphql as graphqlMsw } from 'msw';
 import { setupServer } from 'msw/node';
+import { TeamTransformer, UserTransformer } from './defaultTransformers';
+
 import {
   getOrganizationTeams,
   getOrganizationUsers,
   getTeamMembers,
   getOrganizationRepositories,
   QueryResponse,
+  GithubUser,
+  GithubTeam,
 } from './github';
 import fetch from 'node-fetch';
 
@@ -35,7 +40,7 @@ describe('github', () => {
   const server = setupServer();
   setupRequestMockHandlers(server);
 
-  describe('getOrganizationUsers', () => {
+  describe('getOrganizationUsers using defaultUserMapper', () => {
     it('reads members', async () => {
       const input: QueryResponse = {
         organization: {
@@ -76,7 +81,120 @@ describe('github', () => {
     });
   });
 
-  describe('getOrganizationTeams', () => {
+  describe('getOrganizationUsers using custom UserTransformer', () => {
+    const customUserTransformer: UserTransformer = async (
+      item: GithubUser,
+      {},
+    ) => {
+      if (item.login === 'aa') {
+        return undefined;
+      }
+
+      return {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: `${item.login}-custom`,
+          annotations: {
+            'github.com/user-login': item.login,
+          },
+        },
+        spec: {
+          profile: {},
+          memberOf: [],
+        },
+      } as UserEntity;
+    };
+
+    it('reads members', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'a-custom',
+            }),
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(
+        getOrganizationUsers(graphql, 'a', 'token', customUserTransformer),
+      ).resolves.toEqual(output);
+    });
+
+    it('reads members if undefined is returned from transformer', async () => {
+      const input: QueryResponse = {
+        organization: {
+          membersWithRole: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                login: 'a',
+                name: 'b',
+                bio: 'c',
+                email: 'd',
+                avatarUrl: 'e',
+              },
+              {
+                login: 'aa',
+                name: 'bb',
+                bio: 'cc',
+                email: 'dd',
+                avatarUrl: 'ee',
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        users: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'a-custom',
+            }),
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      const users = await getOrganizationUsers(
+        graphql,
+        'a',
+        'token',
+        customUserTransformer,
+      );
+
+      expect(users.users).toHaveLength(1);
+      expect(users).toEqual(output);
+    });
+  });
+
+  describe('getOrganizationTeams using default TeamTransformer', () => {
     let input: QueryResponse;
 
     beforeEach(() => {
@@ -95,7 +213,7 @@ describe('github', () => {
                 parentTeam: {
                   slug: 'parent',
                   combinedSlug: '',
-                  members: { pageInfo: { hasNextPage: false }, nodes: [] },
+                  members: [],
                 },
                 members: {
                   pageInfo: { hasNextPage: false },
@@ -129,10 +247,10 @@ describe('github', () => {
               },
               parent: 'parent',
               children: [],
+              members: ['user'],
             },
           }),
         ],
-        groupMemberUsers: new Map([['team', ['user']]]),
       };
 
       server.use(
@@ -141,37 +259,192 @@ describe('github', () => {
 
       await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
     });
+  });
 
-    it('applies namespaces', async () => {
+  describe('getOrganizationTeams using custom TeamTransformer', () => {
+    let input: QueryResponse;
+
+    const customTeamTransformer: TeamTransformer = async (
+      item: GithubTeam,
+      {},
+    ) => {
+      if (item.name === 'aa') {
+        return undefined;
+      }
+
+      return {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: `${item.name}-custom`,
+          annotations: {
+            'github.com/team-slug': 'blah/team',
+            'backstage.io/edit-url':
+              'http://example.com/orgs/blah/teams/team/edit',
+          },
+          description: item.description,
+        },
+        spec: {
+          type: 'team',
+          profile: {
+            displayName: `${item.name}-custom`,
+            picture: 'http://example.com/team.jpeg',
+          },
+          parent: 'parent',
+          children: [],
+          members: ['user'],
+        },
+      } as GroupEntity;
+    };
+
+    beforeEach(() => {
+      input = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'Team',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+    });
+
+    it('reads teams', async () => {
       const output = {
         groups: [
           expect.objectContaining({
             metadata: expect.objectContaining({
-              name: 'team',
-              namespace: 'foo',
+              name: 'Team-custom',
               description: 'The one and only team',
+              annotations: {
+                'github.com/team-slug': 'blah/team',
+                'backstage.io/edit-url':
+                  'http://example.com/orgs/blah/teams/team/edit',
+              },
             }),
             spec: {
               type: 'team',
               profile: {
-                displayName: 'Team',
+                displayName: 'Team-custom',
                 picture: 'http://example.com/team.jpeg',
               },
               parent: 'parent',
               children: [],
+              members: ['user'],
             },
           }),
         ],
-        groupMemberUsers: new Map([['foo/team', ['user']]]),
       };
 
       server.use(
         graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
       );
 
-      await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual(
-        output,
+      await expect(
+        getOrganizationTeams(graphql, 'a', customTeamTransformer),
+      ).resolves.toEqual(output);
+    });
+
+    it('reads teams if undefined is returned', async () => {
+      input = {
+        organization: {
+          teams: {
+            pageInfo: { hasNextPage: false },
+            nodes: [
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'Team',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+              {
+                slug: 'team',
+                combinedSlug: 'blah/team',
+                name: 'aa',
+                description: 'The one and only team',
+                avatarUrl: 'http://example.com/team.jpeg',
+                editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit',
+                parentTeam: {
+                  slug: 'parent',
+                  combinedSlug: '',
+                  members: [],
+                },
+                members: {
+                  pageInfo: { hasNextPage: false },
+                  nodes: [{ login: 'user' }],
+                },
+              },
+            ],
+          },
+        },
+      };
+
+      const output = {
+        groups: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'Team-custom',
+              description: 'The one and only team',
+              annotations: {
+                'github.com/team-slug': 'blah/team',
+                'backstage.io/edit-url':
+                  'http://example.com/orgs/blah/teams/team/edit',
+              },
+            }),
+            spec: {
+              type: 'team',
+              profile: {
+                displayName: 'Team-custom',
+                picture: 'http://example.com/team.jpeg',
+              },
+              parent: 'parent',
+              children: [],
+              members: ['user'],
+            },
+          }),
+        ],
+      };
+
+      server.use(
+        graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
       );
+
+      const teams = await getOrganizationTeams(
+        graphql,
+        'a',
+        customTeamTransformer,
+      );
+
+      expect(teams.groups).toHaveLength(1);
+      expect(teams).toEqual(output);
     });
   });
 
@@ -191,7 +464,7 @@ describe('github', () => {
       };
 
       const output = {
-        members: ['user'],
+        members: [{ login: 'user' }],
       };
 
       server.use(

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -17,19 +17,30 @@
 import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { GithubCredentialType } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
+import {
+  defaultOrganizationTeamTransformer,
+  defaultUserTransformer,
+  TeamTransformer,
+  TransformerContext,
+  UserTransformer,
+} from './defaultTransformers';
 
 // Graphql types
 
 export type QueryResponse = {
-  organization?: Organization;
-  repositoryOwner?: Organization | User;
+  organization?: OrganizationResponse;
+  repositoryOwner?: RepositoryOwnerResponse;
 };
 
-export type Organization = {
-  membersWithRole?: Connection<User>;
-  team?: Team;
-  teams?: Connection<Team>;
-  repositories?: Connection<Repository>;
+type RepositoryOwnerResponse = {
+  repositories?: Connection<RepositoryResponse>;
+};
+
+export type OrganizationResponse = {
+  membersWithRole?: Connection<GithubUser>;
+  team?: GithubTeamResponse;
+  teams?: Connection<GithubTeamResponse>;
+  repositories?: Connection<RepositoryResponse>;
 };
 
 export type PageInfo = {
@@ -37,27 +48,41 @@ export type PageInfo = {
   endCursor?: string;
 };
 
-export type User = {
+/**
+ * Github User
+ *
+ * @public
+ */
+export type GithubUser = {
   login: string;
   bio?: string;
   avatarUrl?: string;
   email?: string;
   name?: string;
-  repositories?: Connection<Repository>;
+  organizationVerifiedDomainEmails?: string[];
 };
 
-export type Team = {
+/**
+ * Github Team
+ *
+ * @public
+ */
+export type GithubTeam = {
   slug: string;
   combinedSlug: string;
   name?: string;
   description?: string;
   avatarUrl?: string;
   editTeamUrl?: string;
-  parentTeam?: Team;
-  members: Connection<User>;
+  parentTeam?: GithubTeam;
+  members: GithubUser[];
 };
 
-export type Repository = {
+export type GithubTeamResponse = Omit<GithubTeam, 'members'> & {
+  members: Connection<GithubUser>;
+};
+
+export type RepositoryResponse = {
   name: string;
   url: string;
   isArchived: boolean;
@@ -88,7 +113,7 @@ export type Connection<T> = {
 };
 
 /**
- * Gets all the users out of a GitHub organization.
+ * Gets all the users out of a Github organization.
  *
  * Note that the users will not have their memberships filled in.
  *
@@ -99,7 +124,7 @@ export async function getOrganizationUsers(
   client: typeof graphql,
   org: string,
   tokenType: GithubCredentialType,
-  userNamespace?: string,
+  userTransformer: UserTransformer = defaultUserTransformer,
 ): Promise<{ users: UserEntity[] }> {
   const query = `
     query users($org: String!, $email: Boolean!, $cursor: String) {
@@ -111,7 +136,8 @@ export async function getOrganizationUsers(
             bio,
             email @include(if: $email),
             login,
-            name
+            name,
+            organizationVerifiedDomainEmails(login: $org)
           }
         }
       }
@@ -119,44 +145,24 @@ export async function getOrganizationUsers(
 
   // There is no user -> teams edge, so we leave the memberships empty for
   // now and let the team iteration handle it instead
-  const mapper = (user: User) => {
-    const entity: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: {
-        name: user.login,
-        annotations: {
-          'github.com/user-login': user.login,
-        },
-      },
-      spec: {
-        profile: {},
-        memberOf: [],
-      },
-    };
-
-    if (userNamespace) entity.metadata.namespace = userNamespace;
-    if (user.bio) entity.metadata.description = user.bio;
-    if (user.name) entity.spec.profile!.displayName = user.name;
-    if (user.email) entity.spec.profile!.email = user.email;
-    if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
-
-    return entity;
-  };
 
   const users = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.membersWithRole,
-    mapper,
-    { org, email: tokenType === 'token' },
+    userTransformer,
+    {
+      org,
+      email: tokenType === 'token',
+    },
   );
 
   return { users };
 }
 
 /**
- * Gets all the teams out of a GitHub organization.
+ * Gets all the teams out of a Github organization.
  *
  * Note that the teams will not have any relations apart from parent filled in.
  *
@@ -166,10 +172,9 @@ export async function getOrganizationUsers(
 export async function getOrganizationTeams(
   client: typeof graphql,
   org: string,
-  orgNamespace?: string,
+  teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
 ): Promise<{
   groups: GroupEntity[];
-  groupMemberUsers: Map<string, string[]>;
 }> {
   const query = `
     query teams($org: String!, $cursor: String) {
@@ -193,86 +198,51 @@ export async function getOrganizationTeams(
       }
     }`;
 
-  // Gets populated inside the mapper below
-  const groupMemberUsers = new Map<string, string[]>();
+  const materialisedTeams = async (
+    item: GithubTeamResponse,
+    ctx: TransformerContext,
+  ): Promise<GroupEntity | undefined> => {
+    const memberNames: GithubUser[] = [];
 
-  const mapper = async (team: Team) => {
-    const annotations: { [annotationName: string]: string } = {
-      'github.com/team-slug': team.combinedSlug,
-    };
-
-    if (team.editTeamUrl) {
-      annotations['backstage.io/edit-url'] = team.editTeamUrl;
-    }
-
-    const entity: GroupEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Group',
-      metadata: {
-        name: team.slug,
-        annotations,
-      },
-      spec: {
-        type: 'team',
-        profile: {},
-        children: [],
-      },
-    };
-
-    if (orgNamespace) {
-      entity.metadata.namespace = orgNamespace;
-    }
-
-    if (team.description) {
-      entity.metadata.description = team.description;
-    }
-    if (team.name) {
-      entity.spec.profile!.displayName = team.name;
-    }
-    if (team.avatarUrl) {
-      entity.spec.profile!.picture = team.avatarUrl;
-    }
-    if (team.parentTeam) {
-      entity.spec.parent = team.parentTeam.slug;
-    }
-
-    const memberNames: string[] = [];
-    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;
-    groupMemberUsers.set(groupKey, memberNames);
-
-    if (!team.members.pageInfo.hasNextPage) {
+    if (!item.members.pageInfo.hasNextPage) {
       // We got all the members in one go, run the fast path
-      for (const user of team.members.nodes) {
-        memberNames.push(user.login);
+      for (const user of item.members.nodes) {
+        memberNames.push(user);
       }
     } else {
       // There were more than a hundred immediate members - run the slow
       // path of fetching them explicitly
-      const { members } = await getTeamMembers(client, org, team.slug);
+      const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
       for (const userLogin of members) {
         memberNames.push(userLogin);
       }
     }
 
-    return entity;
+    const team: GithubTeam = {
+      ...item,
+      members: memberNames,
+    };
+
+    return await teamTransformer(team, ctx);
   };
 
   const groups = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.teams,
-    mapper,
+    materialisedTeams,
     { org },
   );
 
-  return { groups, groupMemberUsers };
+  return { groups };
 }
 
 export async function getOrganizationRepositories(
   client: typeof graphql,
   org: string,
   catalogPath: string,
-): Promise<{ repositories: Repository[] }> {
+): Promise<{ repositories: RepositoryResponse[] }> {
   let relativeCatalogPathRef: string;
   // We must strip the leading slash or the query for objects does not work
   if (catalogPath.startsWith('/')) {
@@ -321,8 +291,9 @@ export async function getOrganizationRepositories(
   const repositories = await queryWithPaging(
     client,
     query,
+    org,
     r => r.repositoryOwner?.repositories,
-    x => x,
+    async x => x,
     { org, catalogPathRef },
   );
 
@@ -330,7 +301,7 @@ export async function getOrganizationRepositories(
 }
 
 /**
- * Gets all the users out of a GitHub organization.
+ * Gets all the users out of a Github organization.
  *
  * Note that the users will not have their memberships filled in.
  *
@@ -342,7 +313,7 @@ export async function getTeamMembers(
   client: typeof graphql,
   org: string,
   teamSlug: string,
-): Promise<{ members: string[] }> {
+): Promise<{ members: GithubUser[] }> {
   const query = `
     query members($org: String!, $teamSlug: String!, $cursor: String) {
       organization(login: $org) {
@@ -358,8 +329,9 @@ export async function getTeamMembers(
   const members = await queryWithPaging(
     client,
     query,
+    org,
     r => r.organization?.team?.members,
-    user => user.login,
+    async user => user,
     { org, teamSlug },
   );
 
@@ -379,7 +351,7 @@ export async function getTeamMembers(
  * @param query - The query to execute
  * @param connection - A function that, given the response, picks out the actual
  *                   Connection object that's being iterated
- * @param mapper - A function that, given one of the nodes in the Connection,
+ * @param transformer - A function that, given one of the nodes in the Connection,
  *               returns the model mapped form of it
  * @param variables - The variable values that the query needs, minus the cursor
  */
@@ -391,8 +363,12 @@ export async function queryWithPaging<
 >(
   client: typeof graphql,
   query: string,
+  org: string,
   connection: (response: Response) => Connection<GraphqlType> | undefined,
-  mapper: (item: GraphqlType) => Promise<OutputType> | OutputType,
+  transformer: (
+    item: GraphqlType,
+    ctx: TransformerContext,
+  ) => Promise<OutputType | undefined>,
   variables: Variables,
 ): Promise<OutputType[]> {
   const result: OutputType[] = [];
@@ -410,7 +386,15 @@ export async function queryWithPaging<
     }
 
     for (const node of conn.nodes) {
-      result.push(await mapper(node));
+      const transformedNode = await transformer(node, {
+        client,
+        query,
+        org,
+      });
+
+      if (transformedNode) {
+        result.push(transformedNode);
+      }
     }
 
     if (!conn.pageInfo.hasNextPage) {

plugins/catalog-backend-module-github/src/lib/index.ts

@@ -20,6 +20,15 @@ export {
   getOrganizationRepositories,
   getOrganizationTeams,
   getOrganizationUsers,
+  type GithubUser,
+  type GithubTeam,
 } from './github';
+export {
+  type UserTransformer,
+  defaultUserTransformer,
+  type TeamTransformer,
+  defaultOrganizationTeamTransformer,
+  type TransformerContext,
+} from './defaultTransformers';
 export { assignGroupsToUsers, buildOrgHierarchy } from './org';
 export { parseGithubOrgUrl } from './util';

plugins/catalog-backend-module-github/src/lib/org.test.ts

@@ -68,13 +68,47 @@ describe('buildOrgHierarchy', () => {
 describe('assignGroupsToUsers', () => {
   it('should assign groups to users', () => {
     const users: UserEntity[] = [u('u1'), u('u2')];
-    const groupMemberUsers = new Map<string, string[]>([
-      ['g1', ['u1', 'u2']],
-      ['g2', ['u2']],
-      ['g3', ['u3']],
-    ]);
 
-    assignGroupsToUsers(users, groupMemberUsers);
+    const groups: GroupEntity[] = [
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g1',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u1', 'u2'],
+        },
+      },
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g2',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u2'],
+        },
+      },
+      {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'g3',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: ['u3'],
+        },
+      },
+    ];
+
+    assignGroupsToUsers(users, groups);
 
     expect(users[0].spec.memberOf).toEqual(['g1']);
     expect(users[1].spec.memberOf).toEqual(['g1', 'g2']);

plugins/catalog-backend-module-github/src/lib/org.ts

@@ -52,8 +52,17 @@ export function buildOrgHierarchy(groups: GroupEntity[]) {
 // Ensure that users have their direct group memberships.
 export function assignGroupsToUsers(
   users: UserEntity[],
-  groupMemberUsers: Map<string, string[]>,
+  groups: GroupEntity[],
 ) {
+  const groupMemberUsers = new Map(
+    groups.map(group => {
+      const groupKey = group.metadata.namespace
+        ? `${group.metadata.namespace}/${group.metadata.name}`
+        : group.metadata.name;
+      return [groupKey, group.spec.members || []];
+    }),
+  );
+
   const usersByName = new Map(users.map(u => [u.metadata.name, u]));
   for (const [groupName, userNames] of groupMemberUsers.entries()) {
     for (const userName of userNames) {

plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { GroupEntity } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import {
   DefaultGithubCredentialsProvider,
@@ -32,7 +33,9 @@ import {
 import { graphql } from '@octokit/graphql';
 import { Logger } from 'winston';
 import {
+  assignGroupsToUsers,
   buildOrgHierarchy,
+  defaultOrganizationTeamTransformer,
   getOrganizationTeams,
   getOrganizationUsers,
   GithubMultiOrgConfig,
@@ -130,12 +133,19 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
           client,
           orgConfig.name,
           tokenType,
-          orgConfig.userNamespace,
         );
-        const { groups, groupMemberUsers } = await getOrganizationTeams(
+        const { groups } = await getOrganizationTeams(
           client,
           orgConfig.name,
-          orgConfig.groupNamespace,
+          async (team, ctx): Promise<GroupEntity | undefined> => {
+            const result = await defaultOrganizationTeamTransformer(team, ctx);
+
+            if (result) {
+              result.metadata.namespace = orgConfig.groupNamespace;
+            }
+
+            return result;
+          },
         );
 
         const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
@@ -151,15 +161,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
             allUsersMap.set(prefix + u.metadata.name, u);
           }
         });
-
-        for (const [groupName, userNames] of groupMemberUsers.entries()) {
-          for (const userName of userNames) {
-            const user = allUsersMap.get(prefix + userName);
-            if (user && !user.spec.memberOf.includes(groupName)) {
-              user.spec.memberOf.push(groupName);
-            }
-          }
-        }
+        assignGroupsToUsers(users, groups);
         buildOrgHierarchy(groups);
 
         for (const group of groups) {

plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts

@@ -101,17 +101,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     this.logger.info('Reading GitHub users and groups');
 
     const { users } = await getOrganizationUsers(client, org, tokenType);
-    const { groups, groupMemberUsers } = await getOrganizationTeams(
-      client,
-      org,
-    );
+    const { groups } = await getOrganizationTeams(client, org);
 
     const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
     this.logger.debug(
       `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,
     );
 
-    assignGroupsToUsers(users, groupMemberUsers);
+    assignGroupsToUsers(users, groups);
     buildOrgHierarchy(groups);
 
     // Done!

plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts

@@ -43,6 +43,7 @@ import {
   getOrganizationUsers,
   parseGithubOrgUrl,
 } from '../lib';
+import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers';
 
 /**
  * Options for {@link GithubOrgEntityProvider}.
@@ -88,6 +89,16 @@ export interface GithubOrgEntityProviderOptions {
    * Optionally supply a custom credentials provider, replacing the default one.
    */
   githubCredentialsProvider?: GithubCredentialsProvider;
+
+  /**
+   * Optionally include a user transformer for transforming from GitHub users to User Entities
+   */
+  userTransformer?: UserTransformer;
+
+  /**
+   * Optionally include a user transformer for transforming from GitHub users to User Entities
+   */
+  teamTransformer?: TeamTransformer;
 }
 
 // TODO: Consider supporting an (optional) webhook that reacts on org changes
@@ -123,6 +134,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
       githubCredentialsProvider:
         options.githubCredentialsProvider ||
         DefaultGithubCredentialsProvider.fromIntegrations(integrations),
+      userTransformer: options.userTransformer,
+      teamTransformer: options.teamTransformer,
     });
 
     provider.schedule(options.schedule);
@@ -137,6 +150,8 @@ export class GithubOrgEntityProvider implements EntityProvider {
       gitHubConfig: GithubIntegrationConfig;
       logger: Logger;
       githubCredentialsProvider?: GithubCredentialsProvider;
+      userTransformer?: UserTransformer;
+      teamTransformer?: TeamTransformer;
     },
   ) {
     this.credentialsProvider =
@@ -177,12 +192,19 @@ export class GithubOrgEntityProvider implements EntityProvider {
     });
 
     const { org } = parseGithubOrgUrl(this.options.orgUrl);
-    const { users } = await getOrganizationUsers(client, org, tokenType);
-    const { groups, groupMemberUsers } = await getOrganizationTeams(
+    const { users } = await getOrganizationUsers(
       client,
       org,
+      tokenType,
+      this.options.userTransformer,
     );
-    assignGroupsToUsers(users, groupMemberUsers);
+    const { groups } = await getOrganizationTeams(
+      client,
+      org,
+      this.options.teamTransformer,
+    );
+
+    assignGroupsToUsers(users, groups);
     buildOrgHierarchy(groups);
 
     const { markCommitComplete } = markReadComplete({ users, groups });

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts

@@ -37,7 +37,7 @@ import {
   readProviderConfigs,
   GithubEntityProviderConfig,
 } from './GithubEntityProviderConfig';
-import { getOrganizationRepositories, Repository } from '../lib/github';
+import { getOrganizationRepositories, RepositoryResponse } from '../lib/github';
 import { satisfiesTopicFilter } from '../lib/util';
 
 /**
@@ -175,7 +175,7 @@ export class GithubEntityProvider implements EntityProvider {
   }
 
   // go to the server and get all of the repositories
-  private async findCatalogFiles(): Promise<Repository[]> {
+  private async findCatalogFiles(): Promise<RepositoryResponse[]> {
     const organization = this.config.organization;
     const host = this.integration.host;
     const catalogPath = this.config.catalogPath;
@@ -208,7 +208,7 @@ export class GithubEntityProvider implements EntityProvider {
     return repositories;
   }
 
-  private matchesFilters(repositories: Repository[]) {
+  private matchesFilters(repositories: RepositoryResponse[]) {
     const repositoryFilter = this.config.filters?.repository;
     const topicFilters = this.config.filters?.topic;
 
@@ -226,7 +226,7 @@ export class GithubEntityProvider implements EntityProvider {
     return matchingRepositories;
   }
 
-  private createLocationUrl(repository: Repository): string {
+  private createLocationUrl(repository: RepositoryResponse): string {
     const branch =
       this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
     const catalogFile = this.config.catalogPath.startsWith('/')

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts

@@ -163,6 +163,7 @@ describe('GithubOrgEntityProvider', () => {
                   picture: 'http://example.com/team.jpeg',
                 },
                 type: 'team',
+                members: ['a'],
               },
             },
             locationKey: 'github-org-provider:my-id',