feat(events,catalog/github): handle github organization,team and membership events

Signed-off-by: Rogerio Angeliski <angeliski@hotmail.com>

docs/integrations/github/org.md

@@ -17,7 +17,7 @@ entities that mirror your org setup.
 > provide authentication. See the
 > [GitHub auth provider](../../auth/github/provider.md) for that.
 
-## Installation
+## Installation without Events Support
 
 This guide will use the Entity Provider method. If you for some reason prefer
 the Processor method (not recommended), it is described separately below.
@@ -60,6 +60,54 @@ schedule it:
 +  );
 ```
 
+## Installation with Events Support
+
+Please follow the installation instructions at
+
+- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md
+- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md
+
+Additionally, you need to decide how you want to receive events from external sources like
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+Set up your provider
+
+```diff
+// packages/backend/src/plugins/catalogEventBasedProviders.ts
++import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
+ import { EntityProvider } from '@backstage/plugin-catalog-node';
+ import { EventSubscriber } from '@backstage/plugin-events-node';
+ import { PluginEnvironment } from '../types';
+ export default async function createCatalogEventBasedProviders(
+-  _: PluginEnvironment,
++  env: PluginEnvironment,
+ ): Promise<Array<EntityProvider & EventSubscriber>> {
+   const providers: Array<
+     (EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
+   > = [];
+-  // add your event-based entity providers here
++  providers.push(
++    GithubOrgEntityProvider.fromConfig(env.config, {
++      id: 'production',
++      orgUrl: 'https://github.com/backstage',
++      logger: env.logger,
++      schedule: env.scheduler.createScheduledTaskRunner({
++        frequency: { minutes: 60 },
++        timeout: { minutes: 15 },
++      }),
++    }),
++  );
+   return providers.flat();
+ }
+```
+
+You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks).
+The webhook will need to be configured to forward `organizatio` and `team` events.
+
+**Please note that the event `team` with action `edited` and `removed_from_repository` will not be handled for this provider.**
+
 ## Configuration
 
 As mentioned above, you also must have some configuration in your app-config

plugins/catalog-backend-module-github/api-report.md

@@ -177,7 +177,9 @@ export class GitHubOrgEntityProvider extends GithubOrgEntityProvider {
 }
 
 // @public
-export class GithubOrgEntityProvider implements EntityProvider {
+export class GithubOrgEntityProvider
+  implements EntityProvider, EventSubscriber
+{
   constructor(options: {
     id: string;
     orgUrl: string;
@@ -196,7 +198,11 @@ export class GithubOrgEntityProvider implements EntityProvider {
   ): GithubOrgEntityProvider;
   // (undocumented)
   getProviderName(): string;
+  // (undocumented)
+  onEvent(params: EventParams): Promise<void>;
   read(options?: { logger?: Logger }): Promise<void>;
+  // (undocumented)
+  supportsEventTopics(): string[];
 }
 
 // @public @deprecated (undocumented)

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts

@@ -22,6 +22,7 @@ import {
 } from '@backstage/integration';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { graphql } from '@octokit/graphql';
+import { EventParams } from '@backstage/plugin-events-node';
 import {
   GithubOrgEntityProvider,
   withLocations,
@@ -237,4 +238,362 @@ describe('GithubOrgEntityProvider', () => {
       });
     });
   });
+
+  describe('receiving events from github', () => {
+    afterEach(() => jest.resetAllMocks());
+
+    it('should apply delta added on receive a new member in the organization', async () => {
+      const entityProviderConnection: EntityProviderConnection = {
+        applyMutation: jest.fn(),
+        refresh: jest.fn(),
+      };
+
+      const logger = getVoidLogger();
+      const gitHubConfig: GithubIntegrationConfig = {
+        host: 'github.com',
+      };
+
+      const mockGetCredentials = jest.fn().mockReturnValue({
+        headers: { token: 'blah' },
+        type: 'app',
+      });
+
+      const githubCredentialsProvider: GithubCredentialsProvider = {
+        getCredentials: mockGetCredentials,
+      };
+
+      const entityProvider = new GithubOrgEntityProvider({
+        id: 'my-id',
+        githubCredentialsProvider,
+        orgUrl: 'https://github.com/backstage',
+        gitHubConfig,
+        logger,
+      });
+
+      entityProvider.connect(entityProviderConnection);
+
+      const expectedEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+            'github.com/user-login': 'githubuser',
+          },
+        },
+        spec: {
+          memberOf: [],
+          profile: {
+            displayName: 'githubuser',
+            email: 'user1@test.com',
+            picture: 'https://avatars.githubusercontent.com/u/83820368',
+          },
+        },
+      };
+
+      const event: EventParams = {
+        topic: 'github.organization',
+        eventPayload: {
+          action: 'member_added',
+          membership: {
+            user: {
+              name: 'githubuser',
+              login: 'githubuser',
+              avatar_url: 'https://avatars.githubusercontent.com/u/83820368',
+              email: 'user1@test.com',
+            },
+          },
+          organization: {
+            login: 'test-org',
+          },
+        },
+      };
+
+      await entityProvider.onEvent(event);
+
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+        type: 'delta',
+        added: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: expectedEntity,
+          },
+        ],
+        removed: [],
+      });
+    });
+
+    it('should apply delta removed on receive a removed member in the organization', async () => {
+      const entityProviderConnection: EntityProviderConnection = {
+        applyMutation: jest.fn(),
+        refresh: jest.fn(),
+      };
+
+      const logger = getVoidLogger();
+      const gitHubConfig: GithubIntegrationConfig = {
+        host: 'github.com',
+      };
+
+      const mockGetCredentials = jest.fn().mockReturnValue({
+        headers: { token: 'blah' },
+        type: 'app',
+      });
+
+      const githubCredentialsProvider: GithubCredentialsProvider = {
+        getCredentials: mockGetCredentials,
+      };
+
+      const entityProvider = new GithubOrgEntityProvider({
+        id: 'my-id',
+        githubCredentialsProvider,
+        orgUrl: 'https://github.com/test-org',
+        gitHubConfig,
+        logger,
+      });
+
+      entityProvider.connect(entityProviderConnection);
+
+      const expectedEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+            'github.com/user-login': 'githubuser',
+          },
+        },
+        spec: {
+          memberOf: [],
+          profile: {
+            displayName: 'githubuser',
+            email: 'user1@test.com',
+            picture: 'https://avatars.githubusercontent.com/u/83820368',
+          },
+        },
+      };
+
+      const event: EventParams = {
+        topic: 'github.organization',
+        eventPayload: {
+          action: 'member_removed',
+          membership: {
+            user: {
+              name: 'githubuser',
+              login: 'githubuser',
+              avatar_url: 'https://avatars.githubusercontent.com/u/83820368',
+              email: 'user1@test.com',
+            },
+          },
+          organization: {
+            login: 'test-org',
+          },
+        },
+      };
+
+      await entityProvider.onEvent(event);
+
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+        type: 'delta',
+        removed: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: expectedEntity,
+          },
+        ],
+        added: [],
+      });
+    });
+
+    it('should apply delta added on receive a created team', async () => {
+      const entityProviderConnection: EntityProviderConnection = {
+        applyMutation: jest.fn(),
+        refresh: jest.fn(),
+      };
+
+      const logger = getVoidLogger();
+      const gitHubConfig: GithubIntegrationConfig = {
+        host: 'github.com',
+      };
+
+      const mockGetCredentials = jest.fn().mockReturnValue({
+        headers: { token: 'blah' },
+        type: 'app',
+      });
+
+      const githubCredentialsProvider: GithubCredentialsProvider = {
+        getCredentials: mockGetCredentials,
+      };
+
+      const entityProvider = new GithubOrgEntityProvider({
+        id: 'my-id',
+        githubCredentialsProvider,
+        orgUrl: 'https://github.com/test-org',
+        gitHubConfig,
+        logger,
+      });
+
+      entityProvider.connect(entityProviderConnection);
+
+      const expectedEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'new-team',
+          description: 'description from the new team',
+          annotations: {
+            'backstage.io/edit-url':
+              'https://github.com/orgs/test-org/teams/new-team/edit',
+            'backstage.io/managed-by-location':
+              'url:https://github.com/orgs/test-org/teams/new-team',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/orgs/test-org/teams/new-team',
+            'github.com/team-slug': 'test-org/new-team',
+          },
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: [],
+          parent: 'father-team',
+          profile: {
+            displayName: 'New Team',
+          },
+        },
+      };
+
+      const event: EventParams = {
+        topic: 'github.team',
+        eventPayload: {
+          action: 'created',
+          team: {
+            name: 'New Team',
+            slug: 'new-team',
+            description: 'description from the new team',
+            html_url: 'https://github.com/orgs/test-org/teams/new-team',
+            parent: {
+              slug: 'father-team',
+            },
+          },
+          organization: {
+            login: 'test-org',
+          },
+        },
+      };
+
+      await entityProvider.onEvent(event);
+
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+        type: 'delta',
+        added: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: expectedEntity,
+          },
+        ],
+        removed: [],
+      });
+    });
+
+    it('should apply delta removed on receive a deleted team', async () => {
+      const entityProviderConnection: EntityProviderConnection = {
+        applyMutation: jest.fn(),
+        refresh: jest.fn(),
+      };
+
+      const logger = getVoidLogger();
+      const gitHubConfig: GithubIntegrationConfig = {
+        host: 'github.com',
+      };
+
+      const mockGetCredentials = jest.fn().mockReturnValue({
+        headers: { token: 'blah' },
+        type: 'app',
+      });
+
+      const githubCredentialsProvider: GithubCredentialsProvider = {
+        getCredentials: mockGetCredentials,
+      };
+
+      const entityProvider = new GithubOrgEntityProvider({
+        id: 'my-id',
+        githubCredentialsProvider,
+        orgUrl: 'https://github.com/test-org',
+        gitHubConfig,
+        logger,
+      });
+
+      entityProvider.connect(entityProviderConnection);
+
+      const expectedEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'new-team',
+          description: 'description from the new team',
+          annotations: {
+            'backstage.io/edit-url':
+              'https://github.com/orgs/test-org/teams/new-team/edit',
+            'backstage.io/managed-by-location':
+              'url:https://github.com/orgs/test-org/teams/new-team',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/orgs/test-org/teams/new-team',
+            'github.com/team-slug': 'test-org/new-team',
+          },
+        },
+        spec: {
+          type: 'team',
+          children: [],
+          members: [],
+          parent: 'father-team',
+          profile: {
+            displayName: 'New Team',
+          },
+        },
+      };
+
+      const event: EventParams = {
+        topic: 'github.team',
+        eventPayload: {
+          action: 'deleted',
+          team: {
+            name: 'New Team',
+            slug: 'new-team',
+            description: 'description from the new team',
+            html_url: 'https://github.com/orgs/test-org/teams/new-team',
+            parent: {
+              slug: 'father-team',
+            },
+          },
+          organization: {
+            login: 'test-org',
+          },
+        },
+      };
+
+      await entityProvider.onEvent(event);
+
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+      expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+        type: 'delta',
+        removed: [
+          {
+            locationKey: 'github-org-provider:my-id',
+            entity: expectedEntity,
+          },
+        ],
+        added: [],
+      });
+    });
+  });
 });

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts

@@ -28,23 +28,39 @@ import {
   ScmIntegrations,
   SingleInstanceGithubCredentialsProvider,
 } from '@backstage/integration';
+import { EventParams } from '@backstage/plugin-events-node';
+import { EventSubscriber } from '@backstage/plugin-events-node';
 import {
+  DeferredEntity,
   EntityProvider,
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-backend';
 import { graphql } from '@octokit/graphql';
+import {
+  OrganizationEvent,
+  OrganizationMemberAddedEvent,
+  OrganizationMemberRemovedEvent,
+  TeamEvent,
+} from '@octokit/webhooks-types';
 import { merge } from 'lodash';
 import * as uuid from 'uuid';
 import { Logger } from 'winston';
 import {
   assignGroupsToUsers,
   buildOrgHierarchy,
+  defaultOrganizationTeamTransformer,
+  defaultUserTransformer,
   getOrganizationTeams,
   getOrganizationUsers,
+  GithubTeam,
   parseGithubOrgUrl,
 } from '../lib';
 import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers';
 
+type DeltaOperationFactory = (
+  org: string,
+  entities: Entity[],
+) => { added: DeferredEntity[]; removed: DeferredEntity[] };
 /**
  * Options for {@link GithubOrgEntityProvider}.
  *
@@ -107,7 +123,9 @@ export interface GithubOrgEntityProviderOptions {
  *
  * @public
  */
-export class GithubOrgEntityProvider implements EntityProvider {
+export class GithubOrgEntityProvider
+  implements EntityProvider, EventSubscriber
+{
   private readonly credentialsProvider: GithubCredentialsProvider;
   private connection?: EntityProviderConnection;
   private scheduleFn?: () => Promise<void>;
@@ -224,6 +242,158 @@ export class GithubOrgEntityProvider implements EntityProvider {
     markCommitComplete();
   }
 
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
+  async onEvent(params: EventParams): Promise<void> {
+    const { logger } = this.options;
+    logger.debug(`Received event from ${params.topic}`);
+
+    const addEntities: DeltaOperationFactory = (org, entities) => ({
+      removed: [],
+      added: entities.map(entity => ({
+        locationKey: `github-org-provider:${this.options.id}`,
+        entity: withLocations(
+          `https://${this.options.gitHubConfig.host}`,
+          org,
+          entity,
+        ),
+      })),
+    });
+
+    const removeEntities: DeltaOperationFactory = (org, entities) => ({
+      added: [],
+      removed: entities.map(entity => ({
+        locationKey: `github-org-provider:${this.options.id}`,
+        entity: withLocations(
+          `https://${this.options.gitHubConfig.host}`,
+          org,
+          entity,
+        ),
+      })),
+    });
+
+    // handle change users in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization
+    if (params.topic.includes('organization')) {
+      const orgEvent = params.eventPayload as OrganizationEvent;
+
+      if (orgEvent.action === 'member_added') {
+        await this.onMemberChangeInOrganization(orgEvent, addEntities);
+      } else if (orgEvent.action === 'member_removed') {
+        await this.onMemberChangeInOrganization(orgEvent, removeEntities);
+      }
+    }
+
+    // handle change teams in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team
+    // we don't handle team.edited because isn't possible now to correlate this process when the team name changes
+    // so the full refresh will be responsible for that operation
+    if (params.topic.includes('team')) {
+      const teamEvent = params.eventPayload as TeamEvent;
+
+      if (teamEvent.action === 'created') {
+        await this.onTeamDeltaChangeInOrganization(teamEvent, addEntities);
+      } else if (teamEvent.action === 'deleted') {
+        await this.onTeamDeltaChangeInOrganization(teamEvent, removeEntities);
+      }
+    }
+
+    return;
+  }
+
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
+  supportsEventTopics(): string[] {
+    return ['github.organization', 'github.team'];
+  }
+
+  private async onTeamDeltaChangeInOrganization(
+    event: TeamEvent,
+    createDeltaOperation: DeltaOperationFactory,
+  ) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const organizationTeamTransformer =
+      this.options.teamTransformer || defaultOrganizationTeamTransformer;
+    const { name, html_url: url, description, slug } = event.team;
+    const org = event.organization.login;
+    const { headers } = await this.credentialsProvider.getCredentials({
+      url: this.options.orgUrl,
+    });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const group = (await organizationTeamTransformer(
+      {
+        name,
+        slug,
+        editTeamUrl: `${url}/edit`,
+        combinedSlug: `${org}/${slug}`,
+        description: description || undefined,
+        parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam,
+        // entity will be removed
+        members: [],
+      },
+      {
+        org,
+        client,
+        query: '',
+      },
+    )) as Entity;
+
+    const { added, removed } = createDeltaOperation(org, [group]);
+
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
+  private async onMemberChangeInOrganization(
+    event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent,
+    createDeltaOperation: DeltaOperationFactory,
+  ) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const userTransformer =
+      this.options.userTransformer || defaultUserTransformer;
+    const { name, avatar_url: avatarUrl, email, login } = event.membership.user;
+    const org = event.organization.login;
+    const { headers } = await this.credentialsProvider.getCredentials({
+      url: this.options.orgUrl,
+    });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const user = (await userTransformer(
+      {
+        name,
+        avatarUrl,
+        login,
+        email: email || undefined,
+        // we don't have this information in the event, so the refresh will handle that for us
+        organizationVerifiedDomainEmails: [],
+      },
+      {
+        org,
+        client,
+        query: '',
+      },
+    )) as Entity;
+
+    const { added, removed } = createDeltaOperation(org, [user]);
+    await this.connection.applyMutation({
+      type: 'delta',
+      removed,
+      added,
+    });
+  }
+
   private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) {
     if (!schedule || schedule === 'manual') {
       return;