From b9dc05b91e8bcefb723ea89c708e93ca3dfe9681 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Fri, 25 Nov 2022 17:44:02 +0000 Subject: [PATCH] feat(events,catalog/github): handle github organization,team and membership events Signed-off-by: Rogerio Angeliski --- docs/integrations/github/org.md | 50 ++- .../api-report.md | 8 +- .../providers/GithubOrgEntityProvider.test.ts | 359 ++++++++++++++++++ .../src/providers/GithubOrgEntityProvider.ts | 172 ++++++++- 4 files changed, 586 insertions(+), 3 deletions(-) diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 9e3c450adc..ada6c31421 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -17,7 +17,7 @@ entities that mirror your org setup. > provide authentication. See the > [GitHub auth provider](../../auth/github/provider.md) for that. -## Installation +## Installation without Events Support This guide will use the Entity Provider method. If you for some reason prefer the Processor method (not recommended), it is described separately below. @@ -60,6 +60,54 @@ schedule it: + ); ``` +## Installation with Events Support + +Please follow the installation instructions at + +- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md +- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```diff +// packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + import { EntityProvider } from '@backstage/plugin-catalog-node'; + import { EventSubscriber } from '@backstage/plugin-events-node'; + import { PluginEnvironment } from '../types'; + export default async function createCatalogEventBasedProviders( +- _: PluginEnvironment, ++ env: PluginEnvironment, + ): Promise> { + const providers: Array< + (EntityProvider & EventSubscriber) | Array + > = []; +- // add your event-based entity providers here ++ providers.push( ++ GithubOrgEntityProvider.fromConfig(env.config, { ++ id: 'production', ++ orgUrl: 'https://github.com/backstage', ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 60 }, ++ timeout: { minutes: 15 }, ++ }), ++ }), ++ ); + return providers.flat(); + } +``` + +You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). +The webhook will need to be configured to forward `organizatio` and `team` events. + +**Please note that the event `team` with action `edited` and `removed_from_repository` will not be handled for this provider.** + ## Configuration As mentioned above, you also must have some configuration in your app-config diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 7a85d685a3..590b7d2d6b 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -177,7 +177,9 @@ export class GitHubOrgEntityProvider extends GithubOrgEntityProvider { } // @public -export class GithubOrgEntityProvider implements EntityProvider { +export class GithubOrgEntityProvider + implements EntityProvider, EventSubscriber +{ constructor(options: { id: string; orgUrl: string; @@ -196,7 +198,11 @@ export class GithubOrgEntityProvider implements EntityProvider { ): GithubOrgEntityProvider; // (undocumented) getProviderName(): string; + // (undocumented) + onEvent(params: EventParams): Promise; read(options?: { logger?: Logger }): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @public @deprecated (undocumented) diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index 0b8765b93e..2f6376cb09 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -22,6 +22,7 @@ import { } from '@backstage/integration'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; +import { EventParams } from '@backstage/plugin-events-node'; import { GithubOrgEntityProvider, withLocations, @@ -237,4 +238,362 @@ describe('GithubOrgEntityProvider', () => { }); }); }); + + describe('receiving events from github', () => { + afterEach(() => jest.resetAllMocks()); + + it('should apply delta added on receive a new member in the organization', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + profile: { + displayName: 'githubuser', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }; + + const event: EventParams = { + topic: 'github.organization', + eventPayload: { + action: 'member_added', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + removed: [], + }); + }); + + it('should apply delta removed on receive a removed member in the organization', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/test-org', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + profile: { + displayName: 'githubuser', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }; + + const event: EventParams = { + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + added: [], + }); + }); + + it('should apply delta added on receive a created team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/test-org', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }; + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'created', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + removed: [], + }); + }); + + it('should apply delta removed on receive a deleted team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/test-org', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }; + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'deleted', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + added: [], + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 210df46363..5de931286d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -28,23 +28,39 @@ import { ScmIntegrations, SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; +import { EventParams } from '@backstage/plugin-events-node'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; +import { + OrganizationEvent, + OrganizationMemberAddedEvent, + OrganizationMemberRemovedEvent, + TeamEvent, +} from '@octokit/webhooks-types'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { assignGroupsToUsers, buildOrgHierarchy, + defaultOrganizationTeamTransformer, + defaultUserTransformer, getOrganizationTeams, getOrganizationUsers, + GithubTeam, parseGithubOrgUrl, } from '../lib'; import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers'; +type DeltaOperationFactory = ( + org: string, + entities: Entity[], +) => { added: DeferredEntity[]; removed: DeferredEntity[] }; /** * Options for {@link GithubOrgEntityProvider}. * @@ -107,7 +123,9 @@ export interface GithubOrgEntityProviderOptions { * * @public */ -export class GithubOrgEntityProvider implements EntityProvider { +export class GithubOrgEntityProvider + implements EntityProvider, EventSubscriber +{ private readonly credentialsProvider: GithubCredentialsProvider; private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; @@ -224,6 +242,158 @@ export class GithubOrgEntityProvider implements EntityProvider { markCommitComplete(); } + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + const { logger } = this.options; + logger.debug(`Received event from ${params.topic}`); + + const addEntities: DeltaOperationFactory = (org, entities) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + const removeEntities: DeltaOperationFactory = (org, entities) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + // handle change users in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization + if (params.topic.includes('organization')) { + const orgEvent = params.eventPayload as OrganizationEvent; + + if (orgEvent.action === 'member_added') { + await this.onMemberChangeInOrganization(orgEvent, addEntities); + } else if (orgEvent.action === 'member_removed') { + await this.onMemberChangeInOrganization(orgEvent, removeEntities); + } + } + + // handle change teams in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team + // we don't handle team.edited because isn't possible now to correlate this process when the team name changes + // so the full refresh will be responsible for that operation + if (params.topic.includes('team')) { + const teamEvent = params.eventPayload as TeamEvent; + + if (teamEvent.action === 'created') { + await this.onTeamDeltaChangeInOrganization(teamEvent, addEntities); + } else if (teamEvent.action === 'deleted') { + await this.onTeamDeltaChangeInOrganization(teamEvent, removeEntities); + } + } + + return; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return ['github.organization', 'github.team']; + } + + private async onTeamDeltaChangeInOrganization( + event: TeamEvent, + createDeltaOperation: DeltaOperationFactory, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const organizationTeamTransformer = + this.options.teamTransformer || defaultOrganizationTeamTransformer; + const { name, html_url: url, description, slug } = event.team; + const org = event.organization.login; + const { headers } = await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const group = (await organizationTeamTransformer( + { + name, + slug, + editTeamUrl: `${url}/edit`, + combinedSlug: `${org}/${slug}`, + description: description || undefined, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const { added, removed } = createDeltaOperation(org, [group]); + + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMemberChangeInOrganization( + event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, + createDeltaOperation: DeltaOperationFactory, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.membership.user; + const org = event.organization.login; + const { headers } = await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email || undefined, + // we don't have this information in the event, so the refresh will handle that for us + organizationVerifiedDomainEmails: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const { added, removed } = createDeltaOperation(org, [user]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return;