From 970678adbe2d7b40cf8321741d6a7262ddd8f1e9 Mon Sep 17 00:00:00 2001 From: Phil Kuang Date: Wed, 19 Apr 2023 14:21:56 -0400 Subject: [PATCH] feat(GithubMultiOrgEntityProvider): support events Signed-off-by: Phil Kuang --- .changeset/unlucky-deers-teach.md | 5 + docs/integrations/github/org.md | 72 + .../api-report.md | 10 +- .../src/lib/github.test.ts | 32 + .../src/lib/github.ts | 37 + .../GithubMultiOrgEntityProvider.test.ts | 1249 ++++++++++++++++- .../providers/GithubMultiOrgEntityProvider.ts | 550 +++++++- 7 files changed, 1928 insertions(+), 27 deletions(-) create mode 100644 .changeset/unlucky-deers-teach.md diff --git a/.changeset/unlucky-deers-teach.md b/.changeset/unlucky-deers-teach.md new file mode 100644 index 0000000000..4073c92ba4 --- /dev/null +++ b/.changeset/unlucky-deers-teach.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Implement events support for `GithubMultiOrgEntityProvider` diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index ccffe60bd1..39532e806f 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -65,6 +65,42 @@ export default async function createPlugin( } ``` +Alternatively, if you wish to ingest data from multiple GitHub organizations you can use +the `GithubMultiOrgEntityProvider` instead. Note that by default, this provider will namespace +groups according to the org they originate from to avoid potential name duplicates: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { GithubMultiOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + + /* highlight-add-start */ + // The GitHub URL below needs to match a configured integrations.github entry + // specified in your app-config. + builder.addEntityProvider( + GithubMultiOrgEntityProvider.fromConfig(env.config, { + id: 'production', + githubUrl: 'https://github.com', + // Set the following to list the GitHub orgs you wish to ingest from. You can + // also omit this option to ingest all orgs accessible by your GitHub integration + orgs: ['org-a', 'org-b'], + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, + }), + }), + ); + /* highlight-add-end */ + + // .. +} +``` + ## Installation with Events Support Please follow the installation instructions at @@ -110,6 +146,42 @@ export default async function createPlugin( } ``` +Or, alternatively, if using the `GithubMultiOrgEntityProvider`: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { GithubMultiOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = await CatalogBuilder.create(env); + + /* highlight-add-start */ + // The GitHub URL below needs to match a configured integrations.github entry + // specified in your app-config. + builder.addEntityProvider( + GithubMultiOrgEntityProvider.fromConfig(env.config, { + id: 'production', + githubUrl: 'https://github.com', + // Set the following to list the GitHub orgs you wish to ingest from. You can + // also omit this option to ingest all orgs accessible by your GitHub integration + orgs: ['org-a', 'org-b'], + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, + }), + // Pass in the eventBroker to allow this provider to subscribe to GitHub events + eventBroker: env.eventBroker, + }), + ); + /* highlight-add-end */ + + // .. +} +``` + You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `organization`,`team` and `membership` events. diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 0698fae8a0..6366a2d571 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -10,6 +10,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { EventBroker } from '@backstage/plugin-events-node'; import { EventParams } from '@backstage/plugin-events-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; import { GithubCredentialsProvider } from '@backstage/integration'; @@ -133,7 +134,9 @@ export type GithubMultiOrgConfig = Array<{ }>; // @public -export class GithubMultiOrgEntityProvider implements EntityProvider { +export class GithubMultiOrgEntityProvider + implements EntityProvider, EventSubscriber +{ constructor(options: { id: string; gitHubConfig: GithubIntegrationConfig; @@ -153,11 +156,16 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { ): GithubMultiOrgEntityProvider; // (undocumented) getProviderName(): string; + // (undocumented) + onEvent(params: EventParams): Promise; read(options?: { logger?: Logger }): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @public export interface GithubMultiOrgEntityProviderOptions { + eventBroker?: EventBroker; githubCredentialsProvider?: GithubCredentialsProvider; githubUrl: string; id: string; diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index df4565a174..7a0a29e098 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -21,6 +21,7 @@ import { graphql as graphqlMsw } from 'msw'; import { setupServer } from 'msw/node'; import { TeamTransformer, UserTransformer } from './defaultTransformers'; import { + getOrganizationsFromUser, getOrganizationTeams, getOrganizationUsers, getTeamMembers, @@ -448,6 +449,37 @@ describe('github', () => { }); }); + describe('getOrganizationsFromUser', () => { + it('reads orgs from user', async () => { + const input: QueryResponse = { + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + }, + { + login: 'b', + }, + { + login: 'c', + }, + ], + }, + }, + }; + + server.use( + graphqlMsw.query('orgs', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect(getOrganizationsFromUser(graphql, 'foo')).resolves.toEqual({ + orgs: ['a', 'b', 'c'], + }); + }); + }); + describe('getTeamMembers', () => { it('reads team members', async () => { const input: QueryResponse = { diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index d0b629092a..412b751c38 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -33,6 +33,7 @@ import { DeferredEntity } from '@backstage/plugin-catalog-node'; export type QueryResponse = { organization?: OrganizationResponse; repositoryOwner?: RepositoryOwnerResponse; + user?: UserResponse; }; type RepositoryOwnerResponse = { @@ -46,11 +47,19 @@ export type OrganizationResponse = { repositories?: Connection; }; +export type UserResponse = { + organizations?: Connection; +}; + export type PageInfo = { hasNextPage: boolean; endCursor?: string; }; +export type GithubOrg = { + login: string; +}; + /** * Github User * @@ -334,6 +343,34 @@ export async function getOrganizationTeamsFromUsers( return { groups }; } +export async function getOrganizationsFromUser( + client: typeof graphql, + user: string, +): Promise<{ + orgs: string[]; +}> { + const query = ` + query orgs($user: String!) { + user(login: $user) { + organizations(first: 100) { + nodes { login } + pageInfo { hasNextPage, endCursor } + } + } + }`; + + const orgs = await queryWithPaging( + client, + query, + '', + r => r.user?.organizations, + async o => o.login, + { user }, + ); + + return { orgs }; +} + export async function getOrganizationTeam( client: typeof graphql, org: string, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts index 76ab55552d..69076bf737 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts @@ -160,7 +160,7 @@ describe('GithubMultiOrgEntityProvider', () => { const logger = getVoidLogger(); const gitHubConfig: GithubIntegrationConfig = { - host: 'https://github.com', + host: 'github.com', }; const mockGetCredentials = jest.fn().mockReturnValue({ @@ -201,9 +201,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'github.com/user-login': 'a', }, description: 'c', @@ -227,9 +227,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'github.com/user-login': 'x', }, description: 'z', @@ -253,9 +253,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'github.com/user-login': 'q', }, description: 's', @@ -279,9 +279,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgA/teams/team', + 'url:https://github.com/orgs/orgA/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgA/teams/team', + 'url:https://github.com/orgs/orgA/teams/team', 'github.com/team-slug': 'orgA/team', }, namespace: 'orga', @@ -308,9 +308,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgB/teams/team', + 'url:https://github.com/orgs/orgB/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgB/teams/team', + 'url:https://github.com/orgs/orgB/teams/team', 'github.com/team-slug': 'orgB/team', }, namespace: 'orgb', @@ -467,7 +467,7 @@ describe('GithubMultiOrgEntityProvider', () => { const logger = getVoidLogger(); const gitHubConfig: GithubIntegrationConfig = { - host: 'https://github.com', + host: 'github.com', }; const mockGetCredentials = jest.fn().mockReturnValue({ @@ -507,9 +507,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/a', + 'url:https://github.com/a', 'github.com/user-login': 'a', }, description: 'c', @@ -533,9 +533,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/x', + 'url:https://github.com/x', 'github.com/user-login': 'x', }, description: 'z', @@ -559,9 +559,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/q', + 'url:https://github.com/q', 'github.com/user-login': 'q', }, description: 's', @@ -585,9 +585,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgC/teams/team', + 'url:https://github.com/orgs/orgC/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgC/teams/team', + 'url:https://github.com/orgs/orgC/teams/team', 'github.com/team-slug': 'orgC/team', }, namespace: 'orgc', @@ -614,9 +614,9 @@ describe('GithubMultiOrgEntityProvider', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://https://github.com/orgs/orgD/teams/team', + 'url:https://github.com/orgs/orgD/teams/team', 'backstage.io/managed-by-origin-location': - 'url:https://https://github.com/orgs/orgD/teams/team', + 'url:https://github.com/orgs/orgD/teams/team', 'github.com/team-slug': 'orgD/team', }, namespace: 'orgd', @@ -713,4 +713,1209 @@ describe('GithubMultiOrgEntityProvider', () => { }); }); }); + + describe('events', () => { + let entityProvider: GithubMultiOrgEntityProvider; + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + beforeEach(() => { + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + }); + + entityProvider.connect(entityProviderConnection); + }); + + afterEach(() => jest.resetAllMocks()); + + it('should ignore events from non-applicable orgs', async () => { + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_added', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }); + + expect(entityProviderConnection.applyMutation).not.toHaveBeenCalled(); + + await entityProvider.onEvent({ + topic: 'github.installation', + eventPayload: { + action: 'created', + installation: { + account: { + login: 'test-org', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).not.toHaveBeenCalled(); + }); + + describe('installation', () => { + it('adds users and groups from new org', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.installation', + eventPayload: { + action: 'created', + installation: { + account: { + login: 'orgB', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgB/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgB/teams/team', + 'github.com/team-slug': 'orgB/team', + }, + namespace: 'orgb', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamB', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + }); + + describe('organization', () => { + it('should add a new user', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + { + login: 'orgB', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_added', + organization: { + login: 'orgB', + }, + membership: { + user: { + name: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + login: 'a', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orga/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + + it('should remove a user', async () => { + const mockClient = jest.fn(); + + mockClient.mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgC', + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + organization: { + login: 'orgB', + }, + membership: { + user: { + name: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + login: 'a', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: [], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + }); + }); + + it('should only update group memberships of a user that still belongs to an applicable org', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + organization: { + login: 'orgB', + }, + membership: { + user: { + name: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + login: 'a', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orga/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + }); + + describe('team', () => { + it('should create a new group from a new team', async () => { + await entityProvider.onEvent({ + topic: 'github.team', + eventPayload: { + action: 'created', + organization: { + login: 'orgB', + }, + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/orgB/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + namespace: 'orgb', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/orgB/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'github.com/team-slug': 'orgB/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + + it('should remove a group from a deleted team', async () => { + await entityProvider.onEvent({ + topic: 'github.team', + eventPayload: { + action: 'deleted', + organization: { + login: 'orgB', + }, + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/orgB/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + namespace: 'orgb', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/orgB/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgB/teams/new-team', + 'github.com/team-slug': 'orgB/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + }); + }); + + it('should update group and user entities when a team is edited', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'w', + name: 'x', + bio: 'y', + email: 'z', + avatarUrl: 'q', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.team', + eventPayload: { + action: 'edited', + changes: { + name: { + from: 'oldName', + }, + description: { + from: 'oldDescription', + }, + }, + team: { + slug: 'team', + parent: { + slug: 'parent', + }, + }, + organization: { + login: 'orgA', + }, + }, + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + description: 'c', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/edit-url': 'https://example.com', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamA', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/oldname', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/oldname', + 'github.com/team-slug': 'orgA/oldname', + }, + namespace: 'orga', + name: 'oldname', + description: 'oldDescription', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'oldName', + }, + type: 'team', + members: [], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + }); + }); + }); + + describe('membership', () => { + it('should update group and user entities when a member is added', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + }, + }) + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + { + login: 'orgB', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.membership', + eventPayload: { + action: 'added', + organization: { + login: 'orgA', + }, + team: { + slug: 'teama', + }, + member: { + name: 'a', + login: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + }); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/edit-url': 'https://example.com', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamA', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + + it('should update group and user entities when a member is removed', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + }, + }) + .mockResolvedValueOnce({ + user: { + organizations: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'orgA', + }, + { + login: 'orgB', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'TeamA', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'TeamB', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'https://example.com', + parentTeam: { + slug: 'parent', + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a', name: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + await entityProvider.onEvent({ + topic: 'github.membership', + eventPayload: { + action: 'removed', + organization: { + login: 'orgA', + }, + team: { + slug: 'teama', + }, + member: { + name: 'a', + login: 'a', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + }); + await new Promise(process.nextTick); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + name: 'a', + }, + spec: { + memberOf: ['orgb/team'], + profile: { + displayName: 'a', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/edit-url': 'https://example.com', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'TeamA', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: [], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + removed: [], + }); + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index ce45f30d96..d3943bb66a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -21,7 +21,9 @@ import { DEFAULT_NAMESPACE, Entity, GroupEntity, + parseEntityRef, stringifyEntityRef, + UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -32,10 +34,28 @@ import { ScmIntegrations, } from '@backstage/integration'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { + EventBroker, + EventParams, + EventSubscriber, +} from '@backstage/plugin-events-node'; import { graphql } from '@octokit/graphql'; +import { + InstallationCreatedEvent, + InstallationEvent, + OrganizationEvent, + OrganizationMemberAddedEvent, + OrganizationMemberRemovedEvent, + MembershipEvent, + TeamCreatedEvent, + TeamDeletedEvent, + TeamEditedEvent, + TeamEvent, +} from '@octokit/webhooks-types'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; @@ -44,6 +64,7 @@ import { assignGroupsToUsers, buildOrgHierarchy, defaultOrganizationTeamTransformer, + defaultUserTransformer, getOrganizationTeams, getOrganizationUsers, GithubTeam, @@ -55,6 +76,11 @@ import { ANNOTATION_GITHUB_TEAM_SLUG, ANNOTATION_GITHUB_USER_LOGIN, } from '../lib/annotation'; +import { + getOrganizationsFromUser, + getOrganizationTeam, + getOrganizationTeamsFromUsers, +} from '../lib/github'; import { splitTeamSlug } from '../lib/util'; /** @@ -118,14 +144,26 @@ export interface GithubMultiOrgEntityProviderOptions { * By default groups will be namespaced according to their GitHub org. */ teamTransformer?: TeamTransformer; + + /** + * An EventBroker to subscribe this provider to GitHub events to trigger delta mutations + */ + eventBroker?: EventBroker; } +type CreateDeltaOperation = (entities: Entity[]) => { + added: DeferredEntity[]; + removed: DeferredEntity[]; +}; + /** * Ingests org data (users and groups) from GitHub. * * @public */ -export class GithubMultiOrgEntityProvider implements EntityProvider { +export class GithubMultiOrgEntityProvider + implements EntityProvider, EventSubscriber +{ private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; @@ -161,6 +199,10 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { provider.schedule(options.schedule); + if (options.eventBroker) { + options.eventBroker.subscribe(provider); + } + return provider; } @@ -271,6 +313,478 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { markCommitComplete(); } + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return [ + 'github.installation', + 'github.organization', + 'github.team', + 'github.membership', + ]; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + const { logger } = this.options; + logger.debug(`Received event from ${params.topic}`); + + const orgs = this.options.orgs?.length + ? this.options.orgs + : await this.getAllOrgs(this.options.gitHubConfig); + + const eventPayload = params.eventPayload as + | InstallationEvent + | OrganizationEvent + | MembershipEvent + | TeamEvent; + + if ( + !orgs.includes( + (eventPayload as InstallationEvent).installation?.account?.login, + ) && + !orgs.includes( + (eventPayload as OrganizationEvent | MembershipEvent | TeamEvent) + .organization?.login, + ) + ) { + return; + } + + // https://docs.github.com/webhooks-and-events/webhooks/webhook-events-and-payloads#installation + if ( + params.topic.includes('installation') && + eventPayload.action === 'created' + ) { + // We can only respond to installation.created events to add new users/groups since a + // installation.deleted event won't provide us info on what user/groups we should remove and + // we can't query the uninstalled org since we will no longer have access. This will need to be + // eventually resolved via occasional full mutation runs by calling read() + await this.onInstallationChange( + eventPayload as InstallationCreatedEvent, + orgs, + ); + } + + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization + if ( + params.topic.includes('organization') && + (eventPayload.action === 'member_added' || + eventPayload.action === 'member_removed') + ) { + await this.onMemberChangeInOrganization(eventPayload, orgs); + } + + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team + if (params.topic.includes('team')) { + if ( + eventPayload.action === 'created' || + eventPayload.action === 'deleted' + ) { + await this.onTeamChangeInOrganization( + eventPayload as TeamCreatedEvent | TeamDeletedEvent, + ); + } else if (eventPayload.action === 'edited') { + await this.onTeamEditedInOrganization( + eventPayload as TeamEditedEvent, + orgs, + ); + } + } + + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (params.topic.includes('membership')) { + this.onMembershipChangedInTeam(eventPayload as MembershipEvent, orgs); + } + + return; + } + + private async onInstallationChange( + event: InstallationCreatedEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const org = event.installation.account.login; + const { headers, type: tokenType } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const { groups } = await getOrganizationTeams( + client, + org, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + // Fetch group memberships of users in case they already exist and + // have memberships in groups from other applicable orgs + for (const userOrg of applicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups: userGroups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + users.map( + u => + u.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] || + u.metadata.name, + ), + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers(users, userGroups); + } + + const { added, removed } = this.createAddEntitiesOperation([ + ...users, + ...groups, + ]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMemberChangeInOrganization( + event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.membership.user; + const org = event.organization.login; + const { headers } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { orgs } = await getOrganizationsFromUser(client, login); + const userApplicableOrgs = orgs.filter(o => applicableOrgs.includes(o)); + + let updateMemberships: boolean; + let createDeltaOperation: CreateDeltaOperation; + if (event.action === 'member_removed') { + if (userApplicableOrgs.length) { + // If the user is still associated with another applicable org then we don't want to remove + // them, just update the entity to remove any potential group memberships from the old org + createDeltaOperation = this.createAddEntitiesOperation.bind(this); + updateMemberships = true; + } else { + // User is no longer part of any applicable orgs so we can remove it, + // no need to take memberships into account + createDeltaOperation = this.createRemoveEntitiesOperation.bind(this); + updateMemberships = false; + } + } else { + // We're not sure if this user was already added as part of another applicable org + // so grab the latest memberships (potentially from teams of other orgs) to ensure + // we're not accidentally omitting them + createDeltaOperation = this.createAddEntitiesOperation.bind(this); + updateMemberships = true; + } + + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email ?? undefined, + }, + { + org, + client, + query: '', + }, + )) as UserEntity; + + if (updateMemberships) { + for (const userOrg of userApplicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + [login], + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers([user], groups); + } + } + + const { added, removed } = createDeltaOperation([user]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onTeamChangeInOrganization( + event: TeamCreatedEvent | TeamDeletedEvent, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const org = event.organization.login; + const { headers } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { name, html_url: url, description, slug } = event.team; + const group = (await this.defaultMultiOrgTeamTransformer( + { + name, + slug, + editTeamUrl: `${url}/edit`, + combinedSlug: `${org}/${slug}`, + description: description ?? undefined, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed or is new + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const createDeltaOperation = + event.action === 'created' + ? this.createAddEntitiesOperation.bind(this) + : this.createRemoveEntitiesOperation.bind(this); + const { added, removed } = createDeltaOperation([group]); + + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onTeamEditedInOrganization( + event: TeamEditedEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const org = event.organization.login; + const { headers, type: tokenType } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const teamSlug = event.team.slug; + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const usersFromChangedGroup = + group.spec.members?.map(m => + stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })), + ) || []; + const usersToRebuild = users.filter(u => + usersFromChangedGroup.includes(stringifyEntityRef(u)), + ); + + // Update memberships of associated members of this group in case the group entity ref changed + for (const userOrg of applicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + usersToRebuild.map( + u => + u.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] || + u.metadata.name, + ), + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers(usersToRebuild, groups); + } + + const oldName = event.changes.name?.from || ''; + const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-'); + const oldGroup = (await this.defaultMultiOrgTeamTransformer( + { + name: event.changes.name?.from, + slug: oldSlug, + combinedSlug: `${org}/${oldSlug}`, + description: event.changes.description?.from, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + // Remove the old group entity in case the entity ref is now different + const { removed } = this.createRemoveEntitiesOperation([oldGroup]); + const { added } = this.createAddEntitiesOperation([ + ...usersToRebuild, + group, + ]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMembershipChangedInTeam( + event: MembershipEvent, + applicableOrgs: string[], + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + // The docs are saying I will receive the slug for the removed event, + // but the types don't reflect that, + // so I will just check to be sure the slug is there + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (!('slug' in event.team)) { + return; + } + + const org = event.organization.login; + const { headers } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const teamSlug = event.team.slug; + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.member; + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email ?? undefined, + }, + { + org, + client, + query: '', + }, + )) as UserEntity; + + const { orgs } = await getOrganizationsFromUser(client, login); + const userApplicableOrgs = orgs.filter(o => applicableOrgs.includes(o)); + for (const userOrg of userApplicableOrgs) { + const { headers: orgHeaders } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${userOrg}`, + }); + const orgClient = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers: orgHeaders, + }); + + const { groups } = await getOrganizationTeamsFromUsers( + orgClient, + userOrg, + [login], + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + assignGroupsToUsers([user], groups); + } + + const { added, removed } = this.createAddEntitiesOperation([user, group]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private schedule(schedule: GithubMultiOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return; @@ -301,9 +815,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { team: GithubTeam, ctx: TransformerContext, ): Promise { - const result = this.options.teamTransformer - ? await this.options.teamTransformer(team, ctx) - : await defaultOrganizationTeamTransformer(team, ctx); + if (this.options.teamTransformer) { + return await this.options.teamTransformer(team, ctx); + } + + const result = await defaultOrganizationTeamTransformer(team, ctx); if (result) { result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US'); @@ -333,6 +849,32 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { ) .filter(Boolean) as string[]; } + + private createAddEntitiesOperation(entities: Entity[]) { + return { + removed: [], + added: entities.map(entity => ({ + locationKey: `github-multi-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + entity, + ), + })), + }; + } + + private createRemoveEntitiesOperation(entities: Entity[]) { + return { + added: [], + removed: entities.map(entity => ({ + locationKey: `github-multi-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + entity, + ), + })), + }; + } } // Helps wrap the timing and logging behaviors