Merge pull request #14870 from angeliski/add-github-org-events-handler

Implements EventSubscriber in GithubOrgEntityProvider

plugins/catalog-backend-module-github/api-report.md

@@ -178,7 +178,9 @@ export class GitHubOrgEntityProvider extends GithubOrgEntityProvider {
 }
 
 // @public
-export class GithubOrgEntityProvider implements EntityProvider {
+export class GithubOrgEntityProvider
+  implements EntityProvider, EventSubscriber
+{
   constructor(options: {
     id: string;
     orgUrl: string;
@@ -197,7 +199,11 @@ export class GithubOrgEntityProvider implements EntityProvider {
   ): GithubOrgEntityProvider;
   // (undocumented)
   getProviderName(): string;
+  // (undocumented)
+  onEvent(params: EventParams): Promise<void>;
   read(options?: { logger?: Logger }): Promise<void>;
+  // (undocumented)
+  supportsEventTopics(): string[];
 }
 
 // @public @deprecated (undocumented)

plugins/catalog-backend-module-github/src/lib/github.test.ts

@@ -29,6 +29,9 @@ import {
   QueryResponse,
   GithubUser,
   GithubTeam,
+  createAddEntitiesOperation,
+  createRemoveEntitiesOperation,
+  createReplaceEntitiesOperation,
 } from './github';
 import fetch from 'node-fetch';
 
@@ -557,4 +560,107 @@ describe('github', () => {
       ).resolves.toEqual(output);
     });
   });
+
+  describe('createAddEntitiesOperation', () => {
+    it('create a function to add deferred entities to a delta operation', () => {
+      const operation = createAddEntitiesOperation('my-id', 'host');
+
+      const userEntity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+            'github.com/user-login': 'githubuser',
+          },
+        },
+        spec: {
+          memberOf: ['new-team'],
+        },
+      };
+      expect(operation('org', [userEntity])).toEqual({
+        added: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: userEntity,
+          },
+        ],
+        removed: [],
+      });
+    });
+  });
+
+  describe('createRemoveEntitiesOperation', () => {
+    it('create a function to remove deferred entities to a delta operation', () => {
+      const operation = createRemoveEntitiesOperation('my-id', 'host');
+
+      const userEntity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+            'github.com/user-login': 'githubuser',
+          },
+        },
+        spec: {
+          memberOf: ['new-team'],
+        },
+      };
+      expect(operation('org', [userEntity])).toEqual({
+        removed: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: userEntity,
+          },
+        ],
+        added: [],
+      });
+    });
+  });
+  describe('createReplaceEntitiesOperation', () => {
+    it('create a function to replace deferred entities to a delta operation', () => {
+      const operation = createReplaceEntitiesOperation('my-id', 'host');
+
+      const userEntity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+            'github.com/user-login': 'githubuser',
+          },
+        },
+        spec: {
+          memberOf: ['new-team'],
+        },
+      };
+      expect(operation('org', [userEntity])).toEqual({
+        removed: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: userEntity,
+          },
+        ],
+        added: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: userEntity,
+          },
+        ],
+      });
+    });
+  });
 });

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model';
 import { GithubCredentialType } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
 import {
@@ -24,6 +24,9 @@ import {
   TransformerContext,
   UserTransformer,
 } from './defaultTransformers';
+import { withLocations } from '../providers/GithubOrgEntityProvider';
+
+import { DeferredEntity } from '@backstage/plugin-catalog-backend';
 
 // Graphql types
 
@@ -191,7 +194,14 @@ export async function getOrganizationTeams(
             parentTeam { slug }
             members(first: 100, membership: IMMEDIATE) {
               pageInfo { hasNextPage }
-              nodes { login }
+              nodes { 
+                avatarUrl,
+                bio,
+                email,
+                login,
+                name,
+                organizationVerifiedDomainEmails(login: $org)
+               }
             }
           }
         }
@@ -238,6 +248,164 @@ export async function getOrganizationTeams(
   return { groups };
 }
 
+export async function getOrganizationTeamsFromUsers(
+  client: typeof graphql,
+  org: string,
+  userLogins: string[],
+  teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
+): Promise<{
+  groups: GroupEntity[];
+}> {
+  const query = `
+   query teams($org: String!, $cursor: String, $userLogins: [String!] = "") {
+  organization(login: $org) {
+    teams(first: 100, after: $cursor, userLogins: $userLogins) {
+      pageInfo {
+        hasNextPage
+        endCursor
+      }
+      nodes {
+        slug
+        combinedSlug
+        name
+        description
+        avatarUrl
+        editTeamUrl
+        parentTeam {
+          slug
+        }
+        members(first: 100, membership: IMMEDIATE) {
+          pageInfo {
+            hasNextPage
+          }
+          nodes {
+            avatarUrl,
+            bio,
+            email,
+            login,
+            name,
+            organizationVerifiedDomainEmails(login: $org)
+          }
+        }
+      }
+    }
+  }
+}`;
+
+  const materialisedTeams = async (
+    item: GithubTeamResponse,
+    ctx: TransformerContext,
+  ): Promise<GroupEntity | undefined> => {
+    const memberNames: GithubUser[] = [];
+
+    if (!item.members.pageInfo.hasNextPage) {
+      // We got all the members in one go, run the fast path
+      for (const user of item.members.nodes) {
+        memberNames.push(user);
+      }
+    } else {
+      // There were more than a hundred immediate members - run the slow
+      // path of fetching them explicitly
+      const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
+      for (const userLogin of members) {
+        memberNames.push(userLogin);
+      }
+    }
+
+    const team: GithubTeam = {
+      ...item,
+      members: memberNames,
+    };
+
+    return await teamTransformer(team, ctx);
+  };
+
+  const groups = await queryWithPaging(
+    client,
+    query,
+    org,
+    r => r.organization?.teams,
+    materialisedTeams,
+    { org, userLogins },
+  );
+
+  return { groups };
+}
+
+export async function getOrganizationTeam(
+  client: typeof graphql,
+  org: string,
+  teamSlug: string,
+  teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
+): Promise<{
+  group: GroupEntity;
+}> {
+  const query = `
+  query teams($org: String!, $teamSlug: String!) {
+      organization(login: $org) {
+        team(slug:$teamSlug) {
+            slug
+            combinedSlug
+            name
+            description
+            avatarUrl
+            editTeamUrl
+            parentTeam { slug }
+            members(first: 100, membership: IMMEDIATE) {
+              pageInfo { hasNextPage }
+              nodes { login }
+            }
+        }
+      }
+    }`;
+
+  const materialisedTeam = async (
+    item: GithubTeamResponse,
+    ctx: TransformerContext,
+  ): Promise<GroupEntity | undefined> => {
+    const memberNames: GithubUser[] = [];
+
+    if (!item.members.pageInfo.hasNextPage) {
+      // We got all the members in one go, run the fast path
+      for (const user of item.members.nodes) {
+        memberNames.push(user);
+      }
+    } else {
+      // There were more than a hundred immediate members - run the slow
+      // path of fetching them explicitly
+      const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
+      for (const userLogin of members) {
+        memberNames.push(userLogin);
+      }
+    }
+
+    const team: GithubTeam = {
+      ...item,
+      members: memberNames,
+    };
+
+    return await teamTransformer(team, ctx);
+  };
+
+  const response: QueryResponse = await client(query, {
+    org,
+    teamSlug,
+  });
+
+  if (!response.organization?.team)
+    throw new Error(`Found no match for group ${teamSlug}`);
+
+  const group = await materialisedTeam(response.organization?.team, {
+    query,
+    client,
+    org,
+  });
+
+  if (!group) throw new Error(`Can't transform for group ${teamSlug}`);
+
+  return { group };
+}
+
 export async function getOrganizationRepositories(
   client: typeof graphql,
   org: string,
@@ -349,6 +517,7 @@ export async function getTeamMembers(
  *
  * @param client - The octokit client
  * @param query - The query to execute
+ * @param org - The slug of the org to read
  * @param connection - A function that, given the response, picks out the actual
  *                   Connection object that's being iterated
  * @param transformer - A function that, given one of the nodes in the Connection,
@@ -406,3 +575,39 @@ export async function queryWithPaging<
 
   return result;
 }
+
+export type DeferredEntitiesBuilder = (
+  org: string,
+  entities: Entity[],
+) => { added: DeferredEntity[]; removed: DeferredEntity[] };
+
+export const createAddEntitiesOperation =
+  (id: string, host: string) => (org: string, entities: Entity[]) => ({
+    removed: [],
+    added: entities.map(entity => ({
+      locationKey: `github-org-provider:${id}`,
+      entity: withLocations(`https://${host}`, org, entity),
+    })),
+  });
+
+export const createRemoveEntitiesOperation =
+  (id: string, host: string) => (org: string, entities: Entity[]) => ({
+    added: [],
+    removed: entities.map(entity => ({
+      locationKey: `github-org-provider:${id}`,
+      entity: withLocations(`https://${host}`, org, entity),
+    })),
+  });
+
+export const createReplaceEntitiesOperation =
+  (id: string, host: string) => (org: string, entities: Entity[]) => {
+    const entitiesToReplace = entities.map(entity => ({
+      locationKey: `github-org-provider:${id}`,
+      entity: withLocations(`https://${host}`, org, entity),
+    }));
+
+    return {
+      removed: entitiesToReplace,
+      added: entitiesToReplace,
+    };
+  };

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts

@@ -28,22 +28,43 @@ import {
   ScmIntegrations,
   SingleInstanceGithubCredentialsProvider,
 } from '@backstage/integration';
+import { EventParams } from '@backstage/plugin-events-node';
+import { EventSubscriber } from '@backstage/plugin-events-node';
 import {
   EntityProvider,
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-backend';
 import { graphql } from '@octokit/graphql';
+import {
+  OrganizationEvent,
+  OrganizationMemberAddedEvent,
+  OrganizationMemberRemovedEvent,
+  TeamEvent,
+  TeamEditedEvent,
+  MembershipEvent,
+} from '@octokit/webhooks-types';
 import { merge } from 'lodash';
 import * as uuid from 'uuid';
 import { Logger } from 'winston';
 import {
   assignGroupsToUsers,
   buildOrgHierarchy,
+  defaultOrganizationTeamTransformer,
+  defaultUserTransformer,
   getOrganizationTeams,
   getOrganizationUsers,
+  GithubTeam,
   parseGithubOrgUrl,
 } from '../lib';
-import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers';
+import { TeamTransformer, UserTransformer } from '../lib';
+import {
+  createAddEntitiesOperation,
+  createRemoveEntitiesOperation,
+  createReplaceEntitiesOperation,
+  DeferredEntitiesBuilder,
+  getOrganizationTeam,
+  getOrganizationTeamsFromUsers,
+} from '../lib/github';
 
 /**
  * Options for {@link GithubOrgEntityProvider}.
@@ -101,13 +122,14 @@ export interface GithubOrgEntityProviderOptions {
   teamTransformer?: TeamTransformer;
 }
 
-// TODO: Consider supporting an (optional) webhook that reacts on org changes
 /**
  * Ingests org data (users and groups) from GitHub.
  *
  * @public
  */
-export class GithubOrgEntityProvider implements EntityProvider {
+export class GithubOrgEntityProvider
+  implements EntityProvider, EventSubscriber
+{
   private readonly credentialsProvider: GithubCredentialsProvider;
   private connection?: EntityProviderConnection;
   private scheduleFn?: () => Promise<void>;
@@ -224,6 +246,306 @@ export class GithubOrgEntityProvider implements EntityProvider {
     markCommitComplete();
   }
 
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
+  async onEvent(params: EventParams): Promise<void> {
+    const { logger } = this.options;
+    logger.debug(`Received event from ${params.topic}`);
+
+    const addEntitiesOperation = createAddEntitiesOperation(
+      this.options.id,
+      this.options.gitHubConfig.host,
+    );
+    const removeEntitiesOperation = createRemoveEntitiesOperation(
+      this.options.id,
+      this.options.gitHubConfig.host,
+    );
+
+    const replaceEntitiesOperation = createReplaceEntitiesOperation(
+      this.options.id,
+      this.options.gitHubConfig.host,
+    );
+
+    // handle change users in the org
+    // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization
+    if (params.topic.includes('organization')) {
+      const orgEvent = params.eventPayload as OrganizationEvent;
+
+      if (
+        orgEvent.action === 'member_added' ||
+        orgEvent.action === 'member_removed'
+      ) {
+        const createDeltaOperation =
+          orgEvent.action === 'member_added'
+            ? addEntitiesOperation
+            : removeEntitiesOperation;
+        await this.onMemberChangeInOrganization(orgEvent, createDeltaOperation);
+      }
+    }
+
+    // handle change teams in the org
+    // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team
+    if (params.topic.includes('team')) {
+      const teamEvent = params.eventPayload as TeamEvent;
+      if (teamEvent.action === 'created' || teamEvent.action === 'deleted') {
+        const createDeltaOperation =
+          teamEvent.action === 'created'
+            ? addEntitiesOperation
+            : removeEntitiesOperation;
+        await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation);
+      } else if (teamEvent.action === 'edited') {
+        await this.onTeamEditedInOrganization(
+          teamEvent,
+          replaceEntitiesOperation,
+        );
+      }
+    }
+
+    // handle change membership in the org
+    // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership
+    if (params.topic.includes('membership')) {
+      const membershipEvent = params.eventPayload as MembershipEvent;
+      this.onMembershipChangedInOrganization(
+        membershipEvent,
+        replaceEntitiesOperation,
+      );
+    }
+
+    return;
+  }
+
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
+  supportsEventTopics(): string[] {
+    return ['github.organization', 'github.team', 'github.membership'];
+  }
+
+  private async onTeamEditedInOrganization(
+    event: TeamEditedEvent,
+    createDeltaOperation: DeferredEntitiesBuilder,
+  ) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const teamSlug = event.team.slug;
+    const { headers, type: tokenType } =
+      await this.credentialsProvider.getCredentials({
+        url: this.options.orgUrl,
+      });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const { org } = parseGithubOrgUrl(this.options.orgUrl);
+    const { group } = await getOrganizationTeam(
+      client,
+      org,
+      teamSlug,
+      this.options.teamTransformer,
+    );
+
+    const { users } = await getOrganizationUsers(
+      client,
+      org,
+      tokenType,
+      this.options.userTransformer,
+    );
+
+    const usersFromChangedGroup = group.spec.members || [];
+    const usersToRebuild = users.filter(u =>
+      usersFromChangedGroup.includes(u.metadata.name),
+    );
+
+    const { groups } = await getOrganizationTeamsFromUsers(
+      client,
+      org,
+      usersToRebuild.map(u => u.metadata.name),
+      this.options.teamTransformer,
+    );
+
+    assignGroupsToUsers(usersToRebuild, groups);
+    buildOrgHierarchy(groups);
+
+    const oldName = event.changes.name?.from || '';
+    const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-');
+
+    const { removed } = createDeltaOperation(org, [
+      {
+        ...group,
+        metadata: {
+          name: oldSlug,
+        },
+      },
+    ]);
+    const { added } = createDeltaOperation(org, [...usersToRebuild, ...groups]);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private async onMembershipChangedInOrganization(
+    event: MembershipEvent,
+    createDeltaOperation: DeferredEntitiesBuilder,
+  ) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    // The docs are saying I will receive the slug for the removed event,
+    // but the types don't reflect that,
+    // so I will just check to be sure the slug is there
+    // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership
+    if (!('slug' in event.team)) {
+      return;
+    }
+
+    const teamSlug = event.team.slug;
+    const userLogin = event.member.login;
+    const { headers, type: tokenType } =
+      await this.credentialsProvider.getCredentials({
+        url: this.options.orgUrl,
+      });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const { org } = parseGithubOrgUrl(this.options.orgUrl);
+    const { group } = await getOrganizationTeam(
+      client,
+      org,
+      teamSlug,
+      this.options.teamTransformer,
+    );
+
+    const { users } = await getOrganizationUsers(
+      client,
+      org,
+      tokenType,
+      this.options.userTransformer,
+    );
+
+    const usersToRebuild = users.filter(u => u.metadata.name === userLogin);
+
+    const { groups } = await getOrganizationTeamsFromUsers(
+      client,
+      org,
+      [userLogin],
+      this.options.teamTransformer,
+    );
+
+    // we include group because the removed event need to update the old group too
+    if (!groups.some(g => g.metadata.name === group.metadata.name)) {
+      groups.push(group);
+    }
+
+    assignGroupsToUsers(usersToRebuild, groups);
+    buildOrgHierarchy(groups);
+
+    const { added, removed } = createDeltaOperation(org, [
+      ...usersToRebuild,
+      ...groups,
+    ]);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private async onTeamChangeInOrganization(
+    event: TeamEvent,
+    createDeltaOperation: DeferredEntitiesBuilder,
+  ) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const organizationTeamTransformer =
+      this.options.teamTransformer || defaultOrganizationTeamTransformer;
+    const { name, html_url: url, description, slug } = event.team;
+    const org = event.organization.login;
+    const { headers } = await this.credentialsProvider.getCredentials({
+      url: this.options.orgUrl,
+    });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const group = (await organizationTeamTransformer(
+      {
+        name,
+        slug,
+        editTeamUrl: `${url}/edit`,
+        combinedSlug: `${org}/${slug}`,
+        description: description || undefined,
+        parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam,
+        // entity will be removed
+        members: [],
+      },
+      {
+        org,
+        client,
+        query: '',
+      },
+    )) as Entity;
+
+    const { added, removed } = createDeltaOperation(org, [group]);
+
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private async onMemberChangeInOrganization(
+    event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent,
+    createDeltaOperation: DeferredEntitiesBuilder,
+  ) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const userTransformer =
+      this.options.userTransformer || defaultUserTransformer;
+    const { name, avatar_url: avatarUrl, email, login } = event.membership.user;
+    const org = event.organization.login;
+    const { headers } = await this.credentialsProvider.getCredentials({
+      url: this.options.orgUrl,
+    });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const user = (await userTransformer(
+      {
+        name,
+        avatarUrl,
+        login,
+        email: email || undefined,
+        // we don't have this information in the event, so the refresh will handle that for us
+        organizationVerifiedDomainEmails: [],
+      },
+      {
+        org,
+        client,
+        query: '',
+      },
+    )) as Entity;
+
+    const { added, removed } = createDeltaOperation(org, [user]);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
   private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) {
     if (!schedule || schedule === 'manual') {
       return;