diff --git a/.changeset/good-foxes-fail.md b/.changeset/good-foxes-fail.md new file mode 100644 index 0000000000..5d31161e64 --- /dev/null +++ b/.changeset/good-foxes-fail.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added support for event based updates in the `GithubOrgEntityProvider`! +Based on webhook events from GitHub the affected `User` or `Group` entity will be refreshed. +This includes adding new entities, refreshing existing ones, and removing obsolete ones. + +Please find more information at +https://backstage.io/docs/integrations/github/org#installation-with-events-support diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 9e3c450adc..0aa9997bae 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -17,7 +17,7 @@ entities that mirror your org setup. > provide authentication. See the > [GitHub auth provider](../../auth/github/provider.md) for that. -## Installation +## Installation without Events Support This guide will use the Entity Provider method. If you for some reason prefer the Processor method (not recommended), it is described separately below. @@ -60,6 +60,52 @@ schedule it: + ); ``` +## Installation with Events Support + +Please follow the installation instructions at + +- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md +- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```diff +// packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + import { EntityProvider } from '@backstage/plugin-catalog-node'; + import { EventSubscriber } from '@backstage/plugin-events-node'; + import { PluginEnvironment } from '../types'; + export default async function createCatalogEventBasedProviders( +- _: PluginEnvironment, ++ env: PluginEnvironment, + ): Promise> { + const providers: Array< + (EntityProvider & EventSubscriber) | Array + > = []; +- // add your event-based entity providers here ++ providers.push( ++ GithubOrgEntityProvider.fromConfig(env.config, { ++ id: 'production', ++ orgUrl: 'https://github.com/backstage', ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 60 }, ++ timeout: { minutes: 15 }, ++ }), ++ }), ++ ); + return providers.flat(); + } +``` + +You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). +The webhook will need to be configured to forward `organization`,`team` and `membership` events. + ## Configuration As mentioned above, you also must have some configuration in your app-config diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 8ccfdb09ee..78ffd5b018 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -178,7 +178,9 @@ export class GitHubOrgEntityProvider extends GithubOrgEntityProvider { } // @public -export class GithubOrgEntityProvider implements EntityProvider { +export class GithubOrgEntityProvider + implements EntityProvider, EventSubscriber +{ constructor(options: { id: string; orgUrl: string; @@ -197,7 +199,11 @@ export class GithubOrgEntityProvider implements EntityProvider { ): GithubOrgEntityProvider; // (undocumented) getProviderName(): string; + // (undocumented) + onEvent(params: EventParams): Promise; read(options?: { logger?: Logger }): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @public @deprecated (undocumented) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index d07d2f6af8..387dc8a2ee 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -29,6 +29,9 @@ import { QueryResponse, GithubUser, GithubTeam, + createAddEntitiesOperation, + createRemoveEntitiesOperation, + createReplaceEntitiesOperation, } from './github'; import fetch from 'node-fetch'; @@ -557,4 +560,107 @@ describe('github', () => { ).resolves.toEqual(output); }); }); + + describe('createAddEntitiesOperation', () => { + it('create a function to add deferred entities to a delta operation', () => { + const operation = createAddEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + removed: [], + }); + }); + }); + + describe('createRemoveEntitiesOperation', () => { + it('create a function to remove deferred entities to a delta operation', () => { + const operation = createRemoveEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [], + }); + }); + }); + describe('createReplaceEntitiesOperation', () => { + it('create a function to replace deferred entities to a delta operation', () => { + const operation = createReplaceEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 4d583d0730..f719e3d506 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model'; import { GithubCredentialType } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { @@ -24,6 +24,9 @@ import { TransformerContext, UserTransformer, } from './defaultTransformers'; +import { withLocations } from '../providers/GithubOrgEntityProvider'; + +import { DeferredEntity } from '@backstage/plugin-catalog-backend'; // Graphql types @@ -191,7 +194,14 @@ export async function getOrganizationTeams( parentTeam { slug } members(first: 100, membership: IMMEDIATE) { pageInfo { hasNextPage } - nodes { login } + nodes { + avatarUrl, + bio, + email, + login, + name, + organizationVerifiedDomainEmails(login: $org) + } } } } @@ -238,6 +248,164 @@ export async function getOrganizationTeams( return { groups }; } +export async function getOrganizationTeamsFromUsers( + client: typeof graphql, + org: string, + userLogins: string[], + teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, +): Promise<{ + groups: GroupEntity[]; +}> { + const query = ` + query teams($org: String!, $cursor: String, $userLogins: [String!] = "") { + organization(login: $org) { + teams(first: 100, after: $cursor, userLogins: $userLogins) { + pageInfo { + hasNextPage + endCursor + } + nodes { + slug + combinedSlug + name + description + avatarUrl + editTeamUrl + parentTeam { + slug + } + members(first: 100, membership: IMMEDIATE) { + pageInfo { + hasNextPage + } + nodes { + avatarUrl, + bio, + email, + login, + name, + organizationVerifiedDomainEmails(login: $org) + } + } + } + } + } +}`; + + const materialisedTeams = async ( + item: GithubTeamResponse, + ctx: TransformerContext, + ): Promise => { + const memberNames: GithubUser[] = []; + + if (!item.members.pageInfo.hasNextPage) { + // We got all the members in one go, run the fast path + for (const user of item.members.nodes) { + memberNames.push(user); + } + } else { + // There were more than a hundred immediate members - run the slow + // path of fetching them explicitly + const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug); + for (const userLogin of members) { + memberNames.push(userLogin); + } + } + + const team: GithubTeam = { + ...item, + members: memberNames, + }; + + return await teamTransformer(team, ctx); + }; + + const groups = await queryWithPaging( + client, + query, + org, + r => r.organization?.teams, + materialisedTeams, + { org, userLogins }, + ); + + return { groups }; +} + +export async function getOrganizationTeam( + client: typeof graphql, + org: string, + teamSlug: string, + teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, +): Promise<{ + group: GroupEntity; +}> { + const query = ` + query teams($org: String!, $teamSlug: String!) { + organization(login: $org) { + team(slug:$teamSlug) { + slug + combinedSlug + name + description + avatarUrl + editTeamUrl + parentTeam { slug } + members(first: 100, membership: IMMEDIATE) { + pageInfo { hasNextPage } + nodes { login } + } + } + } + }`; + + const materialisedTeam = async ( + item: GithubTeamResponse, + ctx: TransformerContext, + ): Promise => { + const memberNames: GithubUser[] = []; + + if (!item.members.pageInfo.hasNextPage) { + // We got all the members in one go, run the fast path + for (const user of item.members.nodes) { + memberNames.push(user); + } + } else { + // There were more than a hundred immediate members - run the slow + // path of fetching them explicitly + const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug); + for (const userLogin of members) { + memberNames.push(userLogin); + } + } + + const team: GithubTeam = { + ...item, + members: memberNames, + }; + + return await teamTransformer(team, ctx); + }; + + const response: QueryResponse = await client(query, { + org, + teamSlug, + }); + + if (!response.organization?.team) + throw new Error(`Found no match for group ${teamSlug}`); + + const group = await materialisedTeam(response.organization?.team, { + query, + client, + org, + }); + + if (!group) throw new Error(`Can't transform for group ${teamSlug}`); + + return { group }; +} + export async function getOrganizationRepositories( client: typeof graphql, org: string, @@ -349,6 +517,7 @@ export async function getTeamMembers( * * @param client - The octokit client * @param query - The query to execute + * @param org - The slug of the org to read * @param connection - A function that, given the response, picks out the actual * Connection object that's being iterated * @param transformer - A function that, given one of the nodes in the Connection, @@ -406,3 +575,39 @@ export async function queryWithPaging< return result; } + +export type DeferredEntitiesBuilder = ( + org: string, + entities: Entity[], +) => { added: DeferredEntity[]; removed: DeferredEntity[] }; + +export const createAddEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })), + }); + +export const createRemoveEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })), + }); + +export const createReplaceEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index 0b8765b93e..c24cd573e0 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -22,6 +22,7 @@ import { } from '@backstage/integration'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; +import { EventParams } from '@backstage/plugin-events-node'; import { GithubOrgEntityProvider, withLocations, @@ -237,4 +238,1079 @@ describe('GithubOrgEntityProvider', () => { }); }); }); + + describe('receiving events from github', () => { + afterEach(() => jest.resetAllMocks()); + + it('should apply delta added on receive a new member in the organization', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + profile: { + displayName: 'githubuser', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }; + + const event: EventParams = { + topic: 'github.organization', + eventPayload: { + action: 'member_added', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + removed: [], + }); + }); + + it('should apply delta removed on receive a removed member in the organization', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + profile: { + displayName: 'githubuser', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }; + + const event: EventParams = { + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + added: [], + }); + }); + + it('should apply delta added on receive a created team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }; + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'created', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + removed: [], + }); + }); + + it('should apply delta removed on receive a deleted team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }; + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'deleted', + team: { + databaseId: 1, + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + added: [], + }); + }); + + it('should apply delta on receive a edited team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'githubuser' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'githubuser', + name: 'githubuser', + bio: 'githubuser', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'githubuser' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + entityProvider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'edited', + changes: { + name: { + from: 'mygroup with spaces', + }, + }, + team: { + node_id: 'xpto', + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + description: 'c', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + memberOf: ['team'], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: ['team'], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a', 'githubuser'], + }, + }, + }, + ], + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/mygroup-with-spaces', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/mygroup-with-spaces', + }, + name: 'mygroup-with-spaces', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a', 'githubuser'], + }, + }, + }, + ], + type: 'delta', + }); + }); + + it('should apply delta on receive a membership added', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'githubuser' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'githubuser', + name: 'githubuser', + bio: 'githubuser', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'githubuser' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + entityProvider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.membership', + eventPayload: { + action: 'added', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + member: { + login: 'githubuser', + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: ['team'], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a', 'githubuser'], + }, + }, + }, + ], + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: ['team'], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a', 'githubuser'], + }, + }, + }, + ], + type: 'delta', + }); + }); + + it('should apply delta on receive a membership removed', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'githubuser', + name: 'githubuser', + bio: 'githubuser', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + entityProvider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.membership', + eventPayload: { + action: 'removed', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + member: { + login: 'githubuser', + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: [], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a'], + }, + }, + }, + ], + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: [], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a'], + }, + }, + }, + ], + type: 'delta', + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 210df46363..46ad089625 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -28,22 +28,43 @@ import { ScmIntegrations, SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; +import { EventParams } from '@backstage/plugin-events-node'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; +import { + OrganizationEvent, + OrganizationMemberAddedEvent, + OrganizationMemberRemovedEvent, + TeamEvent, + TeamEditedEvent, + MembershipEvent, +} from '@octokit/webhooks-types'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { assignGroupsToUsers, buildOrgHierarchy, + defaultOrganizationTeamTransformer, + defaultUserTransformer, getOrganizationTeams, getOrganizationUsers, + GithubTeam, parseGithubOrgUrl, } from '../lib'; -import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers'; +import { TeamTransformer, UserTransformer } from '../lib'; +import { + createAddEntitiesOperation, + createRemoveEntitiesOperation, + createReplaceEntitiesOperation, + DeferredEntitiesBuilder, + getOrganizationTeam, + getOrganizationTeamsFromUsers, +} from '../lib/github'; /** * Options for {@link GithubOrgEntityProvider}. @@ -101,13 +122,14 @@ export interface GithubOrgEntityProviderOptions { teamTransformer?: TeamTransformer; } -// TODO: Consider supporting an (optional) webhook that reacts on org changes /** * Ingests org data (users and groups) from GitHub. * * @public */ -export class GithubOrgEntityProvider implements EntityProvider { +export class GithubOrgEntityProvider + implements EntityProvider, EventSubscriber +{ private readonly credentialsProvider: GithubCredentialsProvider; private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; @@ -224,6 +246,306 @@ export class GithubOrgEntityProvider implements EntityProvider { markCommitComplete(); } + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + const { logger } = this.options; + logger.debug(`Received event from ${params.topic}`); + + const addEntitiesOperation = createAddEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); + const removeEntitiesOperation = createRemoveEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); + + const replaceEntitiesOperation = createReplaceEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); + + // handle change users in the org + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization + if (params.topic.includes('organization')) { + const orgEvent = params.eventPayload as OrganizationEvent; + + if ( + orgEvent.action === 'member_added' || + orgEvent.action === 'member_removed' + ) { + const createDeltaOperation = + orgEvent.action === 'member_added' + ? addEntitiesOperation + : removeEntitiesOperation; + await this.onMemberChangeInOrganization(orgEvent, createDeltaOperation); + } + } + + // handle change teams in the org + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team + if (params.topic.includes('team')) { + const teamEvent = params.eventPayload as TeamEvent; + if (teamEvent.action === 'created' || teamEvent.action === 'deleted') { + const createDeltaOperation = + teamEvent.action === 'created' + ? addEntitiesOperation + : removeEntitiesOperation; + await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation); + } else if (teamEvent.action === 'edited') { + await this.onTeamEditedInOrganization( + teamEvent, + replaceEntitiesOperation, + ); + } + } + + // handle change membership in the org + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (params.topic.includes('membership')) { + const membershipEvent = params.eventPayload as MembershipEvent; + this.onMembershipChangedInOrganization( + membershipEvent, + replaceEntitiesOperation, + ); + } + + return; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return ['github.organization', 'github.team', 'github.membership']; + } + + private async onTeamEditedInOrganization( + event: TeamEditedEvent, + createDeltaOperation: DeferredEntitiesBuilder, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const teamSlug = event.team.slug; + const { headers, type: tokenType } = + await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGithubOrgUrl(this.options.orgUrl); + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.options.teamTransformer, + ); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const usersFromChangedGroup = group.spec.members || []; + const usersToRebuild = users.filter(u => + usersFromChangedGroup.includes(u.metadata.name), + ); + + const { groups } = await getOrganizationTeamsFromUsers( + client, + org, + usersToRebuild.map(u => u.metadata.name), + this.options.teamTransformer, + ); + + assignGroupsToUsers(usersToRebuild, groups); + buildOrgHierarchy(groups); + + const oldName = event.changes.name?.from || ''; + const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-'); + + const { removed } = createDeltaOperation(org, [ + { + ...group, + metadata: { + name: oldSlug, + }, + }, + ]); + const { added } = createDeltaOperation(org, [...usersToRebuild, ...groups]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMembershipChangedInOrganization( + event: MembershipEvent, + createDeltaOperation: DeferredEntitiesBuilder, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + // The docs are saying I will receive the slug for the removed event, + // but the types don't reflect that, + // so I will just check to be sure the slug is there + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (!('slug' in event.team)) { + return; + } + + const teamSlug = event.team.slug; + const userLogin = event.member.login; + const { headers, type: tokenType } = + await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGithubOrgUrl(this.options.orgUrl); + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.options.teamTransformer, + ); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const usersToRebuild = users.filter(u => u.metadata.name === userLogin); + + const { groups } = await getOrganizationTeamsFromUsers( + client, + org, + [userLogin], + this.options.teamTransformer, + ); + + // we include group because the removed event need to update the old group too + if (!groups.some(g => g.metadata.name === group.metadata.name)) { + groups.push(group); + } + + assignGroupsToUsers(usersToRebuild, groups); + buildOrgHierarchy(groups); + + const { added, removed } = createDeltaOperation(org, [ + ...usersToRebuild, + ...groups, + ]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onTeamChangeInOrganization( + event: TeamEvent, + createDeltaOperation: DeferredEntitiesBuilder, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const organizationTeamTransformer = + this.options.teamTransformer || defaultOrganizationTeamTransformer; + const { name, html_url: url, description, slug } = event.team; + const org = event.organization.login; + const { headers } = await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const group = (await organizationTeamTransformer( + { + name, + slug, + editTeamUrl: `${url}/edit`, + combinedSlug: `${org}/${slug}`, + description: description || undefined, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const { added, removed } = createDeltaOperation(org, [group]); + + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMemberChangeInOrganization( + event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, + createDeltaOperation: DeferredEntitiesBuilder, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.membership.user; + const org = event.organization.login; + const { headers } = await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email || undefined, + // we don't have this information in the event, so the refresh will handle that for us + organizationVerifiedDomainEmails: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const { added, removed } = createDeltaOperation(org, [user]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return;