From b9dc05b91e8bcefb723ea89c708e93ca3dfe9681 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Fri, 25 Nov 2022 17:44:02 +0000 Subject: [PATCH 01/11] feat(events,catalog/github): handle github organization,team and membership events Signed-off-by: Rogerio Angeliski --- docs/integrations/github/org.md | 50 ++- .../api-report.md | 8 +- .../providers/GithubOrgEntityProvider.test.ts | 359 ++++++++++++++++++ .../src/providers/GithubOrgEntityProvider.ts | 172 ++++++++- 4 files changed, 586 insertions(+), 3 deletions(-) diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 9e3c450adc..ada6c31421 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -17,7 +17,7 @@ entities that mirror your org setup. > provide authentication. See the > [GitHub auth provider](../../auth/github/provider.md) for that. -## Installation +## Installation without Events Support This guide will use the Entity Provider method. If you for some reason prefer the Processor method (not recommended), it is described separately below. @@ -60,6 +60,54 @@ schedule it: + ); ``` +## Installation with Events Support + +Please follow the installation instructions at + +- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md +- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```diff +// packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + import { EntityProvider } from '@backstage/plugin-catalog-node'; + import { EventSubscriber } from '@backstage/plugin-events-node'; + import { PluginEnvironment } from '../types'; + export default async function createCatalogEventBasedProviders( +- _: PluginEnvironment, ++ env: PluginEnvironment, + ): Promise> { + const providers: Array< + (EntityProvider & EventSubscriber) | Array + > = []; +- // add your event-based entity providers here ++ providers.push( ++ GithubOrgEntityProvider.fromConfig(env.config, { ++ id: 'production', ++ orgUrl: 'https://github.com/backstage', ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 60 }, ++ timeout: { minutes: 15 }, ++ }), ++ }), ++ ); + return providers.flat(); + } +``` + +You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). +The webhook will need to be configured to forward `organizatio` and `team` events. + +**Please note that the event `team` with action `edited` and `removed_from_repository` will not be handled for this provider.** + ## Configuration As mentioned above, you also must have some configuration in your app-config diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 7a85d685a3..590b7d2d6b 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -177,7 +177,9 @@ export class GitHubOrgEntityProvider extends GithubOrgEntityProvider { } // @public -export class GithubOrgEntityProvider implements EntityProvider { +export class GithubOrgEntityProvider + implements EntityProvider, EventSubscriber +{ constructor(options: { id: string; orgUrl: string; @@ -196,7 +198,11 @@ export class GithubOrgEntityProvider implements EntityProvider { ): GithubOrgEntityProvider; // (undocumented) getProviderName(): string; + // (undocumented) + onEvent(params: EventParams): Promise; read(options?: { logger?: Logger }): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @public @deprecated (undocumented) diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index 0b8765b93e..2f6376cb09 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -22,6 +22,7 @@ import { } from '@backstage/integration'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; +import { EventParams } from '@backstage/plugin-events-node'; import { GithubOrgEntityProvider, withLocations, @@ -237,4 +238,362 @@ describe('GithubOrgEntityProvider', () => { }); }); }); + + describe('receiving events from github', () => { + afterEach(() => jest.resetAllMocks()); + + it('should apply delta added on receive a new member in the organization', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + profile: { + displayName: 'githubuser', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }; + + const event: EventParams = { + topic: 'github.organization', + eventPayload: { + action: 'member_added', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + removed: [], + }); + }); + + it('should apply delta removed on receive a removed member in the organization', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/test-org', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + profile: { + displayName: 'githubuser', + email: 'user1@test.com', + picture: 'https://avatars.githubusercontent.com/u/83820368', + }, + }, + }; + + const event: EventParams = { + topic: 'github.organization', + eventPayload: { + action: 'member_removed', + membership: { + user: { + name: 'githubuser', + login: 'githubuser', + avatar_url: 'https://avatars.githubusercontent.com/u/83820368', + email: 'user1@test.com', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + added: [], + }); + }); + + it('should apply delta added on receive a created team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/test-org', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }; + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'created', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + removed: [], + }); + }); + + it('should apply delta removed on receive a deleted team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/test-org', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: [], + parent: 'father-team', + profile: { + displayName: 'New Team', + }, + }, + }; + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'deleted', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + added: [], + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 210df46363..5de931286d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -28,23 +28,39 @@ import { ScmIntegrations, SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; +import { EventParams } from '@backstage/plugin-events-node'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { graphql } from '@octokit/graphql'; +import { + OrganizationEvent, + OrganizationMemberAddedEvent, + OrganizationMemberRemovedEvent, + TeamEvent, +} from '@octokit/webhooks-types'; import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { assignGroupsToUsers, buildOrgHierarchy, + defaultOrganizationTeamTransformer, + defaultUserTransformer, getOrganizationTeams, getOrganizationUsers, + GithubTeam, parseGithubOrgUrl, } from '../lib'; import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers'; +type DeltaOperationFactory = ( + org: string, + entities: Entity[], +) => { added: DeferredEntity[]; removed: DeferredEntity[] }; /** * Options for {@link GithubOrgEntityProvider}. * @@ -107,7 +123,9 @@ export interface GithubOrgEntityProviderOptions { * * @public */ -export class GithubOrgEntityProvider implements EntityProvider { +export class GithubOrgEntityProvider + implements EntityProvider, EventSubscriber +{ private readonly credentialsProvider: GithubCredentialsProvider; private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; @@ -224,6 +242,158 @@ export class GithubOrgEntityProvider implements EntityProvider { markCommitComplete(); } + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + const { logger } = this.options; + logger.debug(`Received event from ${params.topic}`); + + const addEntities: DeltaOperationFactory = (org, entities) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + const removeEntities: DeltaOperationFactory = (org, entities) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + // handle change users in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization + if (params.topic.includes('organization')) { + const orgEvent = params.eventPayload as OrganizationEvent; + + if (orgEvent.action === 'member_added') { + await this.onMemberChangeInOrganization(orgEvent, addEntities); + } else if (orgEvent.action === 'member_removed') { + await this.onMemberChangeInOrganization(orgEvent, removeEntities); + } + } + + // handle change teams in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team + // we don't handle team.edited because isn't possible now to correlate this process when the team name changes + // so the full refresh will be responsible for that operation + if (params.topic.includes('team')) { + const teamEvent = params.eventPayload as TeamEvent; + + if (teamEvent.action === 'created') { + await this.onTeamDeltaChangeInOrganization(teamEvent, addEntities); + } else if (teamEvent.action === 'deleted') { + await this.onTeamDeltaChangeInOrganization(teamEvent, removeEntities); + } + } + + return; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return ['github.organization', 'github.team']; + } + + private async onTeamDeltaChangeInOrganization( + event: TeamEvent, + createDeltaOperation: DeltaOperationFactory, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const organizationTeamTransformer = + this.options.teamTransformer || defaultOrganizationTeamTransformer; + const { name, html_url: url, description, slug } = event.team; + const org = event.organization.login; + const { headers } = await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const group = (await organizationTeamTransformer( + { + name, + slug, + editTeamUrl: `${url}/edit`, + combinedSlug: `${org}/${slug}`, + description: description || undefined, + parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam, + // entity will be removed + members: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const { added, removed } = createDeltaOperation(org, [group]); + + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMemberChangeInOrganization( + event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, + createDeltaOperation: DeltaOperationFactory, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const userTransformer = + this.options.userTransformer || defaultUserTransformer; + const { name, avatar_url: avatarUrl, email, login } = event.membership.user; + const org = event.organization.login; + const { headers } = await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const user = (await userTransformer( + { + name, + avatarUrl, + login, + email: email || undefined, + // we don't have this information in the event, so the refresh will handle that for us + organizationVerifiedDomainEmails: [], + }, + { + org, + client, + query: '', + }, + )) as Entity; + + const { added, removed } = createDeltaOperation(org, [user]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return; From 704959ff9cb6305b5dfdda33d99c35a55b662671 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 30 Nov 2022 01:19:01 +0000 Subject: [PATCH 02/11] feat(events,catalog/github): handle github team.edited and membership events Signed-off-by: Rogerio Angeliski --- .../api-report.md | 8 + .../src/lib/defaultTransformers.ts | 1 + .../src/lib/github.test.ts | 10 + .../src/lib/github.ts | 2 + .../providers/GithubOrgEntityProvider.test.ts | 483 +++++++++++++++++- .../src/providers/GithubOrgEntityProvider.ts | 218 +++++++- 6 files changed, 712 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 590b7d2d6b..4cb70cf065 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -5,6 +5,7 @@ ```ts import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CatalogApi } from '@backstage/catalog-client'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -188,6 +189,8 @@ export class GithubOrgEntityProvider githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + catalogApi?: CatalogApi; + tokenManager?: TokenManager; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -210,12 +213,16 @@ export type GitHubOrgEntityProviderOptions = GithubOrgEntityProviderOptions; // @public export interface GithubOrgEntityProviderOptions { + // (undocumented) + catalogApi?: CatalogApi; githubCredentialsProvider?: GithubCredentialsProvider; id: string; logger: Logger; orgUrl: string; schedule?: 'manual' | TaskRunner; teamTransformer?: TeamTransformer; + // (undocumented) + tokenManager?: TokenManager; userTransformer?: UserTransformer; } @@ -246,6 +253,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { // @public export type GithubTeam = { + id: string; slug: string; combinedSlug: string; name?: string; diff --git a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts index efd3eedd5d..3563b4cbc4 100644 --- a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts @@ -88,6 +88,7 @@ export const defaultOrganizationTeamTransformer: TeamTransformer = async team => { const annotations: { [annotationName: string]: string } = { 'github.com/team-slug': team.combinedSlug, + 'github.com/node-id': team.id, }; if (team.editTeamUrl) { diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index d07d2f6af8..fa79c03903 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -204,6 +204,7 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { + id: 'node-id-1', slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -211,6 +212,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { + id: 'node-id-parent', slug: 'parent', combinedSlug: '', members: [], @@ -235,6 +237,7 @@ describe('github', () => { description: 'The one and only team', annotations: { 'github.com/team-slug': 'blah/team', + 'github.com/node-id': 'node-id-1', 'backstage.io/edit-url': 'http://example.com/orgs/blah/teams/team/edit', }, @@ -304,6 +307,7 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { + id: 'node-id-1', slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -311,6 +315,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { + id: 'node-id-parent', slug: 'parent', combinedSlug: '', members: [], @@ -369,6 +374,7 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { + id: 'node-id-1', slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -376,6 +382,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { + id: 'node-id-parent', slug: 'parent', combinedSlug: '', members: [], @@ -386,6 +393,7 @@ describe('github', () => { }, }, { + id: 'node-id-2', slug: 'team', combinedSlug: 'blah/team', name: 'aa', @@ -393,6 +401,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { + id: 'node-id-parent', slug: 'parent', combinedSlug: '', members: [], @@ -453,6 +462,7 @@ describe('github', () => { const input: QueryResponse = { organization: { team: { + id: 'node-id-1', slug: '', combinedSlug: '', members: { diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 4d583d0730..f1d0acc068 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -68,6 +68,7 @@ export type GithubUser = { * @public */ export type GithubTeam = { + id: string; slug: string; combinedSlug: string; name?: string; @@ -182,6 +183,7 @@ export async function getOrganizationTeams( teams(first: 100, after: $cursor) { pageInfo { hasNextPage, endCursor } nodes { + id slug combinedSlug name diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index 2f6376cb09..2775da7228 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -14,8 +14,9 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, TokenManager } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { CatalogApi } from '@backstage/catalog-client'; import { GithubCredentialsProvider, GithubIntegrationConfig, @@ -27,8 +28,19 @@ import { GithubOrgEntityProvider, withLocations, } from './GithubOrgEntityProvider'; +import { cloneDeep } from 'lodash'; jest.mock('@octokit/graphql'); +const tokenManager: jest.Mocked = { + getToken: jest.fn(), + authenticate: jest.fn(), +}; + +const mockCatalogApi = { + getEntityByRef: jest.fn(), + getEntities: jest.fn(), +}; +const catalogApi = mockCatalogApi as Partial as CatalogApi; describe('GithubOrgEntityProvider', () => { describe('read', () => { @@ -261,6 +273,7 @@ describe('GithubOrgEntityProvider', () => { const githubCredentialsProvider: GithubCredentialsProvider = { getCredentials: mockGetCredentials, }; + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', @@ -268,6 +281,8 @@ describe('GithubOrgEntityProvider', () => { orgUrl: 'https://github.com/backstage', gitHubConfig, logger, + tokenManager, + catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -348,12 +363,16 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); + const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, - orgUrl: 'https://github.com/test-org', + orgUrl: 'https://github.com/backstage', gitHubConfig, logger, + tokenManager, + catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -434,12 +453,15 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, - orgUrl: 'https://github.com/test-org', + orgUrl: 'https://github.com/backstage', gitHubConfig, logger, + tokenManager, + catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -525,12 +547,15 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, - orgUrl: 'https://github.com/test-org', + orgUrl: 'https://github.com/backstage', gitHubConfig, logger, + tokenManager, + catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -595,5 +620,455 @@ describe('GithubOrgEntityProvider', () => { added: [], }); }); + + it('should apply delta on receive a edited team', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + tokenManager, + catalogApi, + }); + entityProvider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.team', + eventPayload: { + action: 'edited', + changes: { + name: { + from: 'mygroup', + }, + }, + team: { + node_id: 'xpto', + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + organization: { + login: 'test-org', + }, + }, + }; + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/mygroup"', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/mygroup"', + }, + name: 'mygroup', + }, + spec: { + type: 'team', + children: [], + members: ['user-1'], + }, + }; + + const expectedEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + description: 'description from the new team', + annotations: { + 'backstage.io/edit-url': + 'https://github.com/orgs/test-org/teams/new-team/edit', + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'github.com/team-slug': 'test-org/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1'], + profile: { + displayName: 'New Team', + }, + }, + }; + + mockCatalogApi.getEntities.mockResolvedValue({ + items: [cloneDeep(entity)], + }); + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: entity, + }, + ], + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedEntity, + }, + ], + }); + }); + + it('should apply delta on receive a membership added', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + tokenManager, + catalogApi, + }); + + entityProvider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.membership', + eventPayload: { + action: 'added', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + member: { + login: 'githubuser', + }, + organization: { + login: 'test-org', + }, + }, + }; + + const groupEntity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1'], + }, + }; + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + }, + }; + + const expectedGroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1', 'githubuser'], + }, + }; + + const expectedUserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + + mockCatalogApi.getEntityByRef + .mockResolvedValueOnce(cloneDeep(groupEntity)) + .mockResolvedValueOnce(cloneDeep(userEntity)); + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: groupEntity, + }, + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedGroupEntity, + }, + { + locationKey: 'github-org-provider:my-id', + entity: expectedUserEntity, + }, + ], + }); + }); + + it('should apply delta on receive a membership removed', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); + + const entityProvider = new GithubOrgEntityProvider({ + id: 'my-id', + githubCredentialsProvider, + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + tokenManager, + catalogApi, + }); + + entityProvider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.membership', + eventPayload: { + action: 'removed', + team: { + name: 'New Team', + slug: 'new-team', + description: 'description from the new team', + html_url: 'https://github.com/orgs/test-org/teams/new-team', + parent: { + slug: 'father-team', + }, + }, + member: { + login: 'githubuser', + }, + organization: { + login: 'test-org', + }, + }, + }; + + const groupEntity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1', 'githubuser'], + }, + }; + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + + const expectedGroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1'], + }, + }; + + const expectedUserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + }, + }; + + mockCatalogApi.getEntityByRef + .mockResolvedValueOnce(cloneDeep(groupEntity)) + .mockResolvedValueOnce(cloneDeep(userEntity)); + + await entityProvider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: groupEntity, + }, + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: expectedGroupEntity, + }, + { + locationKey: 'github-org-provider:my-id', + entity: expectedUserEntity, + }, + ], + }); + }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 5de931286d..e605b75956 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -18,7 +18,10 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, + DEFAULT_NAMESPACE, Entity, + GroupEntity, + UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -41,8 +44,12 @@ import { OrganizationMemberAddedEvent, OrganizationMemberRemovedEvent, TeamEvent, + TeamEditedEvent, + MembershipEvent, } from '@octokit/webhooks-types'; -import { merge } from 'lodash'; +import { CatalogApi } from '@backstage/catalog-client'; +import { TokenManager } from '@backstage/backend-common'; +import { merge, omit } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { @@ -115,6 +122,9 @@ export interface GithubOrgEntityProviderOptions { * Optionally include a team transformer for transforming from GitHub teams to Group Entities */ teamTransformer?: TeamTransformer; + + catalogApi?: CatalogApi; + tokenManager?: TokenManager; } // TODO: Consider supporting an (optional) webhook that reacts on org changes @@ -130,6 +140,8 @@ export class GithubOrgEntityProvider private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; + private eventConfigErrorThrown = false; + static fromConfig(config: Config, options: GithubOrgEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; @@ -154,6 +166,8 @@ export class GithubOrgEntityProvider DefaultGithubCredentialsProvider.fromIntegrations(integrations), userTransformer: options.userTransformer, teamTransformer: options.teamTransformer, + catalogApi: options.catalogApi, + tokenManager: options.tokenManager, }); provider.schedule(options.schedule); @@ -170,6 +184,8 @@ export class GithubOrgEntityProvider githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + catalogApi?: CatalogApi; + tokenManager?: TokenManager; }, ) { this.credentialsProvider = @@ -247,6 +263,11 @@ export class GithubOrgEntityProvider const { logger } = this.options; logger.debug(`Received event from ${params.topic}`); + if (!this.canHandleEvents()) { + logger.debug(`Skiping event ${params.topic}`); + return; + } + const addEntities: DeltaOperationFactory = (org, entities) => ({ removed: [], added: entities.map(entity => ({ @@ -258,7 +279,6 @@ export class GithubOrgEntityProvider ), })), }); - const removeEntities: DeltaOperationFactory = (org, entities) => ({ added: [], removed: entities.map(entity => ({ @@ -271,6 +291,22 @@ export class GithubOrgEntityProvider })), }); + const replaceEntities: DeltaOperationFactory = (org, entities) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; + // handle change users in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization if (params.topic.includes('organization')) { const orgEvent = params.eventPayload as OrganizationEvent; @@ -283,8 +319,6 @@ export class GithubOrgEntityProvider } // handle change teams in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team - // we don't handle team.edited because isn't possible now to correlate this process when the team name changes - // so the full refresh will be responsible for that operation if (params.topic.includes('team')) { const teamEvent = params.eventPayload as TeamEvent; @@ -292,15 +326,39 @@ export class GithubOrgEntityProvider await this.onTeamDeltaChangeInOrganization(teamEvent, addEntities); } else if (teamEvent.action === 'deleted') { await this.onTeamDeltaChangeInOrganization(teamEvent, removeEntities); + } else if (teamEvent.action === 'edited') { + await this.onTeamEditedInOrganization(teamEvent, replaceEntities); } } + // handle change membership in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (params.topic.includes('membership')) { + const membershipEvent = params.eventPayload as MembershipEvent; + await this.onMemberChangeToTeam(membershipEvent, replaceEntities); + } + return; } /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ supportsEventTopics(): string[] { - return ['github.organization', 'github.team']; + return ['github.organization', 'github.team', 'github.membership']; + } + + private canHandleEvents(): boolean { + if (this.options.catalogApi && this.options.tokenManager) { + return true; + } + + // throw only once + if (!this.eventConfigErrorThrown) { + this.eventConfigErrorThrown = true; + throw new Error( + `${this.getProviderName()} not well configured to handle ${this.supportsEventTopics()}. Missing CatalogApi and/or TokenManager.`, + ); + } + + return false; } private async onTeamDeltaChangeInOrganization( @@ -313,7 +371,7 @@ export class GithubOrgEntityProvider const organizationTeamTransformer = this.options.teamTransformer || defaultOrganizationTeamTransformer; - const { name, html_url: url, description, slug } = event.team; + const { node_id: id, name, html_url: url, description, slug } = event.team; const org = event.organization.login; const { headers } = await this.credentialsProvider.getCredentials({ url: this.options.orgUrl, @@ -325,6 +383,7 @@ export class GithubOrgEntityProvider const group = (await organizationTeamTransformer( { + id, name, slug, editTeamUrl: `${url}/edit`, @@ -350,6 +409,73 @@ export class GithubOrgEntityProvider }); } + private async onTeamEditedInOrganization( + event: TeamEditedEvent, + createDeltaOperation: DeltaOperationFactory, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { name } = event.changes; + const org = event.organization.login; + + const { token } = await this.options.tokenManager!.getToken(); + const groups = await this.options.catalogApi!.getEntities( + { + filter: [ + { + kind: 'Group', + [`metadata.annotations.github.com/node-id`]: event.team.node_id, + }, + ], + fields: ['apiVersion', 'kind', 'metadata', 'spec'], + }, + { token }, + ); + + if (groups.items.length === 0) { + this.options.logger.debug( + `Skipping event because couldn't found group with 'github.com/node-id':${event.team.node_id}`, + ); + return; + } + + const group = groups.items[0] as GroupEntity; + const { removed } = createDeltaOperation(org, [group]); + const { name: newTeamName, html_url: url, description, slug } = event.team; + + if (name) { + merge(group, { + metadata: { + name: slug, + annotations: { + [`backstage.io/edit-url`]: `${url}/edit`, + [`github.com/team-slug`]: `${org}/${slug}`, + [ANNOTATION_LOCATION]: `url:${url}`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${url}`, + }, + }, + spec: { + profile: { + displayName: newTeamName, + }, + }, + }); + } + + if (description) { + group.metadata.description = description; + } + + const { added } = createDeltaOperation(org, [group]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private async onMemberChangeInOrganization( event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, createDeltaOperation: DeltaOperationFactory, @@ -394,6 +520,86 @@ export class GithubOrgEntityProvider }); } + private async onMemberChangeToTeam( + event: MembershipEvent, + createDeltaOperation: DeltaOperationFactory, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + // The docs are saying I will receive the slug for the removed event, but the types don't reflect that, + // so I will just check to be sure the slug is there + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (!('slug' in event.team)) { + return; + } + const teamSlug = event.team.slug; + + const { token } = await this.options.tokenManager!.getToken(); + const org = event.organization.login; + let group = (await this.options.catalogApi!.getEntityByRef( + { + kind: 'Group', + namespace: DEFAULT_NAMESPACE, + name: teamSlug, + }, + { token }, + )) as GroupEntity; + + if (!group) { + this.options.logger.debug( + `Skipping event because couldn't found group ':${event.team.slug} for namespace ${DEFAULT_NAMESPACE}`, + ); + return; + } + + let user = (await this.options.catalogApi!.getEntityByRef( + { + kind: 'User', + namespace: DEFAULT_NAMESPACE, + name: event.member.login, + }, + { token }, + )) as UserEntity; + + group = omit(group, 'relations'); + + if (!user) { + this.options.logger.debug( + `Skipping event because couldn't found user ':${event.member.login} for namespace ${DEFAULT_NAMESPACE}`, + ); + return; + } + user = omit(user, 'relations'); + + const { removed } = createDeltaOperation(org, [group, user]); + + if (event.action === 'added') { + group.spec?.members?.push(event.member.login); + user.spec?.memberOf?.push(event.team.slug); + } else { + group = merge(omit(group, 'spec.members'), { + spec: { + members: group?.spec?.members?.filter(m => m !== event.member.login), + }, + }); + + user = merge(omit(user, 'spec.memberOf'), { + spec: { + memberOf: user?.spec?.memberOf?.filter(m => m !== teamSlug), + }, + }); + } + + const { added } = createDeltaOperation(org, [group, user]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return; From e169bd91d1eac90bfce2c7f2001f6184f6d7e240 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 30 Nov 2022 14:06:43 +0000 Subject: [PATCH 03/11] refct(events,catalog/github): change code for simplicity Signed-off-by: Rogerio Angeliski --- .../src/lib/github.ts | 68 +++++++- .../src/providers/GithubOrgEntityProvider.ts | 154 ++++++++---------- 2 files changed, 136 insertions(+), 86 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index f1d0acc068..e3c492e1a8 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model'; import { GithubCredentialType } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { @@ -24,6 +24,10 @@ import { TransformerContext, UserTransformer, } from './defaultTransformers'; +import { withLocations } from '../providers/GithubOrgEntityProvider'; + +import { DeferredEntity } from '@backstage/plugin-catalog-backend'; +import { merge, omit } from 'lodash'; // Graphql types @@ -351,6 +355,7 @@ export async function getTeamMembers( * * @param client - The octokit client * @param query - The query to execute + * @param org - The slug of the org to read * @param connection - A function that, given the response, picks out the actual * Connection object that's being iterated * @param transformer - A function that, given one of the nodes in the Connection, @@ -408,3 +413,64 @@ export async function queryWithPaging< return result; } + +export type DeferredEntitiesBuilder = ( + org: string, + entities: Entity[], +) => { added: DeferredEntity[]; removed: DeferredEntity[] }; + +export type EntityUpdateOperation = ( + user: UserEntity, + group: GroupEntity, +) => { user: UserEntity; group: GroupEntity }; + +export const createAddEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => ({ + removed: [], + added: entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })), + }); +export const createRemoveEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => ({ + added: [], + removed: entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })), + }); +export const createReplaceEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; + +export function removeUserFromGroup(user: UserEntity, group: GroupEntity) { + return { + group: merge(omit(group, 'spec.members'), { + spec: { + members: group?.spec?.members?.filter(m => m !== user.metadata.name), + }, + }), + + user: merge(omit(user, 'spec.memberOf'), { + spec: { + memberOf: user?.spec?.memberOf?.filter(m => m !== group.metadata.name), + }, + }), + }; +} + +export function addUserToGroup(user: UserEntity, group: GroupEntity) { + group.spec?.members?.push(user.metadata.name); + user.spec?.memberOf?.push(group.metadata.name); + return { group, user }; +} diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index e605b75956..d3fa6ae17f 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -34,7 +34,6 @@ import { import { EventParams } from '@backstage/plugin-events-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; import { - DeferredEntity, EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; @@ -62,12 +61,17 @@ import { GithubTeam, parseGithubOrgUrl, } from '../lib'; -import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers'; +import { TeamTransformer, UserTransformer } from '../lib'; +import { + addUserToGroup, + removeUserFromGroup, + createAddEntitiesOperation, + createRemoveEntitiesOperation, + createReplaceEntitiesOperation, + DeferredEntitiesBuilder, + EntityUpdateOperation, +} from '../lib/github'; -type DeltaOperationFactory = ( - org: string, - entities: Entity[], -) => { added: DeferredEntity[]; removed: DeferredEntity[] }; /** * Options for {@link GithubOrgEntityProvider}. * @@ -127,7 +131,6 @@ export interface GithubOrgEntityProviderOptions { tokenManager?: TokenManager; } -// TODO: Consider supporting an (optional) webhook that reacts on org changes /** * Ingests org data (users and groups) from GitHub. * @@ -264,77 +267,71 @@ export class GithubOrgEntityProvider logger.debug(`Received event from ${params.topic}`); if (!this.canHandleEvents()) { - logger.debug(`Skiping event ${params.topic}`); + logger.debug(`Skipping event ${params.topic}`); return; } - const addEntities: DeltaOperationFactory = (org, entities) => ({ - removed: [], - added: entities.map(entity => ({ - locationKey: `github-org-provider:${this.options.id}`, - entity: withLocations( - `https://${this.options.gitHubConfig.host}`, - org, - entity, - ), - })), - }); - const removeEntities: DeltaOperationFactory = (org, entities) => ({ - added: [], - removed: entities.map(entity => ({ - locationKey: `github-org-provider:${this.options.id}`, - entity: withLocations( - `https://${this.options.gitHubConfig.host}`, - org, - entity, - ), - })), - }); + const addEntitiesOperation = createAddEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); + const removeEntitiesOperation = createRemoveEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); + const replaceEntitiesOperation = createReplaceEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); - const replaceEntities: DeltaOperationFactory = (org, entities) => { - const entitiesToReplace = entities.map(entity => ({ - locationKey: `github-org-provider:${this.options.id}`, - entity: withLocations( - `https://${this.options.gitHubConfig.host}`, - org, - entity, - ), - })); - - return { - removed: entitiesToReplace, - added: entitiesToReplace, - }; - }; - - // handle change users in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization + // handle change users in the org + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization if (params.topic.includes('organization')) { const orgEvent = params.eventPayload as OrganizationEvent; - if (orgEvent.action === 'member_added') { - await this.onMemberChangeInOrganization(orgEvent, addEntities); - } else if (orgEvent.action === 'member_removed') { - await this.onMemberChangeInOrganization(orgEvent, removeEntities); + if ( + orgEvent.action === 'member_added' || + orgEvent.action === 'member_removed' + ) { + const createDeltaOperation = + orgEvent.action === 'member_added' + ? addEntitiesOperation + : removeEntitiesOperation; + await this.onMemberChangeInOrganization(orgEvent, createDeltaOperation); } } - // handle change teams in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team + // handle change teams in the org + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team if (params.topic.includes('team')) { const teamEvent = params.eventPayload as TeamEvent; - - if (teamEvent.action === 'created') { - await this.onTeamDeltaChangeInOrganization(teamEvent, addEntities); - } else if (teamEvent.action === 'deleted') { - await this.onTeamDeltaChangeInOrganization(teamEvent, removeEntities); + if (teamEvent.action === 'created' || teamEvent.action === 'deleted') { + const createDeltaOperation = + teamEvent.action === 'created' + ? addEntitiesOperation + : removeEntitiesOperation; + await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation); } else if (teamEvent.action === 'edited') { - await this.onTeamEditedInOrganization(teamEvent, replaceEntities); + await this.onTeamEditedInOrganization( + teamEvent, + replaceEntitiesOperation, + ); } } - // handle change membership in the org https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + // handle change membership in the org + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership if (params.topic.includes('membership')) { const membershipEvent = params.eventPayload as MembershipEvent; - await this.onMemberChangeToTeam(membershipEvent, replaceEntities); + const updateOperation = + membershipEvent.action === 'added' + ? addUserToGroup + : removeUserFromGroup; + await this.onMemberChangeToTeam( + membershipEvent, + replaceEntitiesOperation, + updateOperation, + ); } return; @@ -361,9 +358,9 @@ export class GithubOrgEntityProvider return false; } - private async onTeamDeltaChangeInOrganization( + private async onTeamChangeInOrganization( event: TeamEvent, - createDeltaOperation: DeltaOperationFactory, + createDeltaOperation: DeferredEntitiesBuilder, ) { if (!this.connection) { throw new Error('Not initialized'); @@ -411,7 +408,7 @@ export class GithubOrgEntityProvider private async onTeamEditedInOrganization( event: TeamEditedEvent, - createDeltaOperation: DeltaOperationFactory, + createDeltaOperation: DeferredEntitiesBuilder, ) { if (!this.connection) { throw new Error('Not initialized'); @@ -478,7 +475,7 @@ export class GithubOrgEntityProvider private async onMemberChangeInOrganization( event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, - createDeltaOperation: DeltaOperationFactory, + createDeltaOperation: DeferredEntitiesBuilder, ) { if (!this.connection) { throw new Error('Not initialized'); @@ -522,7 +519,8 @@ export class GithubOrgEntityProvider private async onMemberChangeToTeam( event: MembershipEvent, - createDeltaOperation: DeltaOperationFactory, + createDeltaOperation: DeferredEntitiesBuilder, + updateEntities: EntityUpdateOperation, ) { if (!this.connection) { throw new Error('Not initialized'); @@ -535,6 +533,7 @@ export class GithubOrgEntityProvider return; } const teamSlug = event.team.slug; + const userSlug = event.member.login; const { token } = await this.options.tokenManager!.getToken(); const org = event.organization.login; @@ -549,7 +548,7 @@ export class GithubOrgEntityProvider if (!group) { this.options.logger.debug( - `Skipping event because couldn't found group ':${event.team.slug} for namespace ${DEFAULT_NAMESPACE}`, + `Skipping event because couldn't found group ':${teamSlug} for namespace ${DEFAULT_NAMESPACE}`, ); return; } @@ -558,7 +557,7 @@ export class GithubOrgEntityProvider { kind: 'User', namespace: DEFAULT_NAMESPACE, - name: event.member.login, + name: userSlug, }, { token }, )) as UserEntity; @@ -567,7 +566,7 @@ export class GithubOrgEntityProvider if (!user) { this.options.logger.debug( - `Skipping event because couldn't found user ':${event.member.login} for namespace ${DEFAULT_NAMESPACE}`, + `Skipping event because couldn't found user ':${userSlug} for namespace ${DEFAULT_NAMESPACE}`, ); return; } @@ -575,24 +574,9 @@ export class GithubOrgEntityProvider const { removed } = createDeltaOperation(org, [group, user]); - if (event.action === 'added') { - group.spec?.members?.push(event.member.login); - user.spec?.memberOf?.push(event.team.slug); - } else { - group = merge(omit(group, 'spec.members'), { - spec: { - members: group?.spec?.members?.filter(m => m !== event.member.login), - }, - }); + const result = updateEntities(user, group); - user = merge(omit(user, 'spec.memberOf'), { - spec: { - memberOf: user?.spec?.memberOf?.filter(m => m !== teamSlug), - }, - }); - } - - const { added } = createDeltaOperation(org, [group, user]); + const { added } = createDeltaOperation(org, [result.group, result.user]); await this.connection.applyMutation({ type: 'delta', removed, From 427d8f4411d82dfec014c577a1442cab3e417887 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 30 Nov 2022 15:32:41 +0000 Subject: [PATCH 04/11] docs: update docs about the new event subscriber for github org Signed-off-by: Rogerio Angeliski --- .changeset/good-foxes-fail.md | 13 +++++++++++++ docs/integrations/github/org.md | 10 +++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 .changeset/good-foxes-fail.md diff --git a/.changeset/good-foxes-fail.md b/.changeset/good-foxes-fail.md new file mode 100644 index 0000000000..6c0740b8d7 --- /dev/null +++ b/.changeset/good-foxes-fail.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Handle GitHub `organization`, `team` and `membership` events at the `GithubOrgEntityProvider` by subscribing to the respective topics + +Implements `EventSubscriber` to receive events for the topic `github.push`. + +On `organization`, `team` and `membership` events, the affected User or Group will be refreshed. +This includes adding new entities, refreshing existing ones, and removing obsolete ones. + +Please find more information at +https://backstage.io/docs/integrations/github/org#installation-with-events-support diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index ada6c31421..c2e67843e9 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -76,6 +76,7 @@ Set up your provider ```diff // packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { CatalogClient } from '@backstage/catalog-client'; +import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; @@ -93,6 +94,8 @@ Set up your provider + id: 'production', + orgUrl: 'https://github.com/backstage', + logger: env.logger, ++ catalogApi: new CatalogClient({ discoveryApi: env.discovery }), ++ tokenManager: env.tokenManager, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, @@ -103,10 +106,11 @@ Set up your provider } ``` -You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). -The webhook will need to be configured to forward `organizatio` and `team` events. +**Attention:** +`catalogApi` and `tokenManager` are required at this variant compared to the one without events support. -**Please note that the event `team` with action `edited` and `removed_from_repository` will not be handled for this provider.** +You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). +The webhook will need to be configured to forward `organization`,`team` and `membership` events. ## Configuration From 9451e8f20dc116e39a58acd7a118584bc685c802 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 30 Nov 2022 16:18:02 +0000 Subject: [PATCH 05/11] refct(events,catalog/github): add tests to helper functions Signed-off-by: Rogerio Angeliski --- .../src/lib/github.test.ts | 240 ++++++++++++++++++ .../src/lib/github.ts | 16 ++ 2 files changed, 256 insertions(+) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index fa79c03903..0c0038f6d7 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -29,6 +29,11 @@ import { QueryResponse, GithubUser, GithubTeam, + createAddEntitiesOperation, + createRemoveEntitiesOperation, + createReplaceEntitiesOperation, + removeUserFromGroup, + addUserToGroup, } from './github'; import fetch from 'node-fetch'; @@ -567,4 +572,239 @@ describe('github', () => { ).resolves.toEqual(output); }); }); + + describe('createAddEntitiesOperation', () => { + it('create a function to add deferred entities to a delta operation', () => { + const operation = createAddEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + removed: [], + }); + }); + }); + + describe('createRemoveEntitiesOperation', () => { + it('create a function to remove deferred entities to a delta operation', () => { + const operation = createRemoveEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [], + }); + }); + }); + + describe('createReplaceEntitiesOperation', () => { + it('create a function to replace deferred entities to a delta operation', () => { + const operation = createReplaceEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + }); + }); + }); + + describe('removeUserFromGroup', () => { + it('should remove a user from the group', () => { + const groupEntity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1', 'githubuser'], + }, + }; + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + + const { user, group } = removeUserFromGroup(userEntity, groupEntity); + + expect(user.spec.memberOf).toEqual([]); + expect(group.spec.members).toEqual(['user-1']); + }); + }); + + describe('addUserToGroup', () => { + it('should add a user to the group', () => { + const groupEntity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + members: ['user-1'], + }, + }; + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + }, + }; + + const { user, group } = addUserToGroup(userEntity, groupEntity); + expect(user.spec.memberOf).toEqual(['new-team']); + expect(group.spec.members).toEqual(['user-1', 'githubuser']); + }); + it('should add a user to the group when the fields are not defined', () => { + const groupEntity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'new-team', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/test-org/teams/new-team', + }, + }, + spec: { + type: 'team', + children: [], + }, + }; + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: {}, + }; + + const { user, group } = addUserToGroup(userEntity, groupEntity); + + expect(user.spec.memberOf).toEqual(['new-team']); + expect(group.spec.members).toEqual(['githubuser']); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index e3c492e1a8..b643075966 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -432,6 +432,7 @@ export const createAddEntitiesOperation = entity: withLocations(`https://${host}`, org, entity), })), }); + export const createRemoveEntitiesOperation = (id: string, host: string) => (org: string, entities: Entity[]) => ({ added: [], @@ -440,6 +441,7 @@ export const createRemoveEntitiesOperation = entity: withLocations(`https://${host}`, org, entity), })), }); + export const createReplaceEntitiesOperation = (id: string, host: string) => (org: string, entities: Entity[]) => { const entitiesToReplace = entities.map(entity => ({ @@ -470,6 +472,20 @@ export function removeUserFromGroup(user: UserEntity, group: GroupEntity) { } export function addUserToGroup(user: UserEntity, group: GroupEntity) { + if (!group.spec?.members) { + group.spec = { + ...group.spec, + members: [], + }; + } + + if (!user.spec?.memberOf) { + user.spec = { + ...user.spec, + memberOf: [], + }; + } + group.spec?.members?.push(user.metadata.name); user.spec?.memberOf?.push(group.metadata.name); return { group, user }; From 936d1cc98a1398a924a8b3bfe001cd1a6ccf36ee Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 30 Nov 2022 16:20:54 +0000 Subject: [PATCH 06/11] docs: remove typo from changeset Signed-off-by: Rogerio Angeliski --- .changeset/good-foxes-fail.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/good-foxes-fail.md b/.changeset/good-foxes-fail.md index 6c0740b8d7..322d892b85 100644 --- a/.changeset/good-foxes-fail.md +++ b/.changeset/good-foxes-fail.md @@ -4,7 +4,7 @@ Handle GitHub `organization`, `team` and `membership` events at the `GithubOrgEntityProvider` by subscribing to the respective topics -Implements `EventSubscriber` to receive events for the topic `github.push`. +Implements `EventSubscriber` to receive events for the topics `github.organization`, `github.team` and `github.membership`. On `organization`, `team` and `membership` events, the affected User or Group will be refreshed. This includes adding new entities, refreshing existing ones, and removing obsolete ones. From 59a87c9149de236f454b3d53e41b52bbc60a09e3 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 30 Nov 2022 16:26:18 +0000 Subject: [PATCH 07/11] docs: add useful information to custom transformers Signed-off-by: Rogerio Angeliski --- docs/integrations/github/org.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index c2e67843e9..234004073a 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -165,6 +165,10 @@ There is also a `defaultUserTransformer` and `defaultOrganizationTeamTransformer You could use these and simply decorate the response from the default transformation if you only need to change a few properties. +**Attention:** +When you use the Events Support with a `TeamTransformer`, you need to ensure your entity will have the +annotation `github.com/node-id` with the value from `team.id`. Without this, the `team.edit` event could fail to update + ### Resolving GitHub users via organization email When you authenticate users you should resolve them to an entity within the From 1f18561c9b4fbd372d1efb921b597647bdeb0231 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Thu, 1 Dec 2022 10:46:28 -0300 Subject: [PATCH 08/11] docs: update changeset to be more direct Co-authored-by: Johan Haals Signed-off-by: Rogerio Angeliski --- .changeset/good-foxes-fail.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.changeset/good-foxes-fail.md b/.changeset/good-foxes-fail.md index 322d892b85..5d31161e64 100644 --- a/.changeset/good-foxes-fail.md +++ b/.changeset/good-foxes-fail.md @@ -2,11 +2,8 @@ '@backstage/plugin-catalog-backend-module-github': patch --- -Handle GitHub `organization`, `team` and `membership` events at the `GithubOrgEntityProvider` by subscribing to the respective topics - -Implements `EventSubscriber` to receive events for the topics `github.organization`, `github.team` and `github.membership`. - -On `organization`, `team` and `membership` events, the affected User or Group will be refreshed. +Added support for event based updates in the `GithubOrgEntityProvider`! +Based on webhook events from GitHub the affected `User` or `Group` entity will be refreshed. This includes adding new entities, refreshing existing ones, and removing obsolete ones. Please find more information at From cf716d4a25cefc234fde179df64e4fac97fc059b Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Thu, 1 Dec 2022 19:49:53 +0000 Subject: [PATCH 09/11] refct(events,catalog/github): use team-id instead node-id Signed-off-by: Rogerio Angeliski --- docs/integrations/github/org.md | 2 +- .../api-report.md | 2 +- .../src/lib/defaultTransformers.ts | 5 ++++- .../src/lib/github.test.ts | 20 +++++++++---------- .../src/lib/github.ts | 4 ++-- .../providers/GithubOrgEntityProvider.test.ts | 1 + .../src/providers/GithubOrgEntityProvider.ts | 12 ++++++++--- 7 files changed, 28 insertions(+), 18 deletions(-) diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 234004073a..bfda79d1f2 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -167,7 +167,7 @@ transformation if you only need to change a few properties. **Attention:** When you use the Events Support with a `TeamTransformer`, you need to ensure your entity will have the -annotation `github.com/node-id` with the value from `team.id`. Without this, the `team.edit` event could fail to update +annotation `github.com/team-id` with the value from `team.databaseId`. Without this, the `team.edit` event could fail to update ### Resolving GitHub users via organization email diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 4cb70cf065..bdf42aa6b3 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -253,7 +253,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { // @public export type GithubTeam = { - id: string; + databaseId: number; slug: string; combinedSlug: string; name?: string; diff --git a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts index 3563b4cbc4..361942543b 100644 --- a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts @@ -88,9 +88,12 @@ export const defaultOrganizationTeamTransformer: TeamTransformer = async team => { const annotations: { [annotationName: string]: string } = { 'github.com/team-slug': team.combinedSlug, - 'github.com/node-id': team.id, }; + if (team.databaseId) { + annotations['github.com/team-id'] = `${team.databaseId}`; + } + if (team.editTeamUrl) { annotations['backstage.io/edit-url'] = team.editTeamUrl; } diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 0c0038f6d7..593450aeda 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -209,7 +209,7 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { - id: 'node-id-1', + databaseId: 1, slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -217,7 +217,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - id: 'node-id-parent', + databaseId: 2, slug: 'parent', combinedSlug: '', members: [], @@ -242,7 +242,7 @@ describe('github', () => { description: 'The one and only team', annotations: { 'github.com/team-slug': 'blah/team', - 'github.com/node-id': 'node-id-1', + 'github.com/team-id': '1', 'backstage.io/edit-url': 'http://example.com/orgs/blah/teams/team/edit', }, @@ -312,7 +312,7 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { - id: 'node-id-1', + databaseId: 1, slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -320,7 +320,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - id: 'node-id-parent', + databaseId: 3, slug: 'parent', combinedSlug: '', members: [], @@ -379,7 +379,7 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { - id: 'node-id-1', + databaseId: 1, slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -387,7 +387,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - id: 'node-id-parent', + databaseId: 3, slug: 'parent', combinedSlug: '', members: [], @@ -398,7 +398,7 @@ describe('github', () => { }, }, { - id: 'node-id-2', + databaseId: 2, slug: 'team', combinedSlug: 'blah/team', name: 'aa', @@ -406,7 +406,7 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - id: 'node-id-parent', + databaseId: 3, slug: 'parent', combinedSlug: '', members: [], @@ -467,7 +467,7 @@ describe('github', () => { const input: QueryResponse = { organization: { team: { - id: 'node-id-1', + databaseId: 1, slug: '', combinedSlug: '', members: { diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index b643075966..22c405d7d3 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -72,7 +72,7 @@ export type GithubUser = { * @public */ export type GithubTeam = { - id: string; + databaseId: number; slug: string; combinedSlug: string; name?: string; @@ -187,7 +187,7 @@ export async function getOrganizationTeams( teams(first: 100, after: $cursor) { pageInfo { hasNextPage, endCursor } nodes { - id + databaseId slug combinedSlug name diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index 2775da7228..a13215d91b 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -592,6 +592,7 @@ describe('GithubOrgEntityProvider', () => { eventPayload: { action: 'deleted', team: { + databaseId: 1, name: 'New Team', slug: 'new-team', description: 'description from the new team', diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index d3fa6ae17f..7d7941f5ec 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -368,7 +368,13 @@ export class GithubOrgEntityProvider const organizationTeamTransformer = this.options.teamTransformer || defaultOrganizationTeamTransformer; - const { node_id: id, name, html_url: url, description, slug } = event.team; + const { + id: databaseId, + name, + html_url: url, + description, + slug, + } = event.team; const org = event.organization.login; const { headers } = await this.credentialsProvider.getCredentials({ url: this.options.orgUrl, @@ -380,7 +386,7 @@ export class GithubOrgEntityProvider const group = (await organizationTeamTransformer( { - id, + databaseId, name, slug, editTeamUrl: `${url}/edit`, @@ -423,7 +429,7 @@ export class GithubOrgEntityProvider filter: [ { kind: 'Group', - [`metadata.annotations.github.com/node-id`]: event.team.node_id, + [`metadata.annotations.github.com/team-id`]: `${event.team.id}`, }, ], fields: ['apiVersion', 'kind', 'metadata', 'spec'], From 7418d09737194211b329b1a8300ccb14b10b83ed Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Mon, 12 Dec 2022 19:22:44 +0000 Subject: [PATCH 10/11] refct(events,catalog/github): remove catalog api dependency Signed-off-by: Rogerio Angeliski --- docs/integrations/github/org.md | 10 - .../api-report.md | 7 - .../src/lib/github.test.ts | 172 ------ .../src/lib/github.ts | 55 -- .../providers/GithubOrgEntityProvider.test.ts | 576 +++++++++--------- .../src/providers/GithubOrgEntityProvider.ts | 199 +----- 6 files changed, 306 insertions(+), 713 deletions(-) diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index bfda79d1f2..0aa9997bae 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -76,7 +76,6 @@ Set up your provider ```diff // packages/backend/src/plugins/catalogEventBasedProviders.ts -+import { CatalogClient } from '@backstage/catalog-client'; +import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; @@ -94,8 +93,6 @@ Set up your provider + id: 'production', + orgUrl: 'https://github.com/backstage', + logger: env.logger, -+ catalogApi: new CatalogClient({ discoveryApi: env.discovery }), -+ tokenManager: env.tokenManager, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, @@ -106,9 +103,6 @@ Set up your provider } ``` -**Attention:** -`catalogApi` and `tokenManager` are required at this variant compared to the one without events support. - You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `organization`,`team` and `membership` events. @@ -165,10 +159,6 @@ There is also a `defaultUserTransformer` and `defaultOrganizationTeamTransformer You could use these and simply decorate the response from the default transformation if you only need to change a few properties. -**Attention:** -When you use the Events Support with a `TeamTransformer`, you need to ensure your entity will have the -annotation `github.com/team-id` with the value from `team.databaseId`. Without this, the `team.edit` event could fail to update - ### Resolving GitHub users via organization email When you authenticate users you should resolve them to an entity within the diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index bdf42aa6b3..4b07ffd593 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -5,7 +5,6 @@ ```ts import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; import { BackendFeature } from '@backstage/backend-plugin-api'; -import { CatalogApi } from '@backstage/catalog-client'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -189,8 +188,6 @@ export class GithubOrgEntityProvider githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; - catalogApi?: CatalogApi; - tokenManager?: TokenManager; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -213,16 +210,12 @@ export type GitHubOrgEntityProviderOptions = GithubOrgEntityProviderOptions; // @public export interface GithubOrgEntityProviderOptions { - // (undocumented) - catalogApi?: CatalogApi; githubCredentialsProvider?: GithubCredentialsProvider; id: string; logger: Logger; orgUrl: string; schedule?: 'manual' | TaskRunner; teamTransformer?: TeamTransformer; - // (undocumented) - tokenManager?: TokenManager; userTransformer?: UserTransformer; } diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 593450aeda..a9bcf2c49b 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -31,9 +31,6 @@ import { GithubTeam, createAddEntitiesOperation, createRemoveEntitiesOperation, - createReplaceEntitiesOperation, - removeUserFromGroup, - addUserToGroup, } from './github'; import fetch from 'node-fetch'; @@ -638,173 +635,4 @@ describe('github', () => { }); }); }); - - describe('createReplaceEntitiesOperation', () => { - it('create a function to replace deferred entities to a delta operation', () => { - const operation = createReplaceEntitiesOperation('my-id', 'host'); - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - expect(operation('org', [userEntity])).toEqual({ - removed: [ - { - locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - }); - }); - }); - - describe('removeUserFromGroup', () => { - it('should remove a user from the group', () => { - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1', 'githubuser'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - - const { user, group } = removeUserFromGroup(userEntity, groupEntity); - - expect(user.spec.memberOf).toEqual([]); - expect(group.spec.members).toEqual(['user-1']); - }); - }); - - describe('addUserToGroup', () => { - it('should add a user to the group', () => { - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: [], - }, - }; - - const { user, group } = addUserToGroup(userEntity, groupEntity); - expect(user.spec.memberOf).toEqual(['new-team']); - expect(group.spec.members).toEqual(['user-1', 'githubuser']); - }); - it('should add a user to the group when the fields are not defined', () => { - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: {}, - }; - - const { user, group } = addUserToGroup(userEntity, groupEntity); - - expect(user.spec.memberOf).toEqual(['new-team']); - expect(group.spec.members).toEqual(['githubuser']); - }); - }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 22c405d7d3..d33ef4fb34 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -27,7 +27,6 @@ import { import { withLocations } from '../providers/GithubOrgEntityProvider'; import { DeferredEntity } from '@backstage/plugin-catalog-backend'; -import { merge, omit } from 'lodash'; // Graphql types @@ -419,11 +418,6 @@ export type DeferredEntitiesBuilder = ( entities: Entity[], ) => { added: DeferredEntity[]; removed: DeferredEntity[] }; -export type EntityUpdateOperation = ( - user: UserEntity, - group: GroupEntity, -) => { user: UserEntity; group: GroupEntity }; - export const createAddEntitiesOperation = (id: string, host: string) => (org: string, entities: Entity[]) => ({ removed: [], @@ -441,52 +435,3 @@ export const createRemoveEntitiesOperation = entity: withLocations(`https://${host}`, org, entity), })), }); - -export const createReplaceEntitiesOperation = - (id: string, host: string) => (org: string, entities: Entity[]) => { - const entitiesToReplace = entities.map(entity => ({ - locationKey: `github-org-provider:${id}`, - entity: withLocations(`https://${host}`, org, entity), - })); - - return { - removed: entitiesToReplace, - added: entitiesToReplace, - }; - }; - -export function removeUserFromGroup(user: UserEntity, group: GroupEntity) { - return { - group: merge(omit(group, 'spec.members'), { - spec: { - members: group?.spec?.members?.filter(m => m !== user.metadata.name), - }, - }), - - user: merge(omit(user, 'spec.memberOf'), { - spec: { - memberOf: user?.spec?.memberOf?.filter(m => m !== group.metadata.name), - }, - }), - }; -} - -export function addUserToGroup(user: UserEntity, group: GroupEntity) { - if (!group.spec?.members) { - group.spec = { - ...group.spec, - members: [], - }; - } - - if (!user.spec?.memberOf) { - user.spec = { - ...user.spec, - memberOf: [], - }; - } - - group.spec?.members?.push(user.metadata.name); - user.spec?.memberOf?.push(group.metadata.name); - return { group, user }; -} diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index a13215d91b..270f184d7a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -14,9 +14,8 @@ * limitations under the License. */ -import { getVoidLogger, TokenManager } from '@backstage/backend-common'; +import { getVoidLogger } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; -import { CatalogApi } from '@backstage/catalog-client'; import { GithubCredentialsProvider, GithubIntegrationConfig, @@ -28,19 +27,8 @@ import { GithubOrgEntityProvider, withLocations, } from './GithubOrgEntityProvider'; -import { cloneDeep } from 'lodash'; jest.mock('@octokit/graphql'); -const tokenManager: jest.Mocked = { - getToken: jest.fn(), - authenticate: jest.fn(), -}; - -const mockCatalogApi = { - getEntityByRef: jest.fn(), - getEntities: jest.fn(), -}; -const catalogApi = mockCatalogApi as Partial as CatalogApi; describe('GithubOrgEntityProvider', () => { describe('read', () => { @@ -273,7 +261,6 @@ describe('GithubOrgEntityProvider', () => { const githubCredentialsProvider: GithubCredentialsProvider = { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', @@ -281,8 +268,6 @@ describe('GithubOrgEntityProvider', () => { orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -363,16 +348,12 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -453,15 +434,12 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -547,15 +525,12 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -642,17 +617,61 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + entityProvider.connect(entityProviderConnection); const event: EventParams = { @@ -680,71 +699,68 @@ describe('GithubOrgEntityProvider', () => { }, }; - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/mygroup"', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/mygroup"', - }, - name: 'mygroup', - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const expectedEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - description: 'description from the new team', - annotations: { - 'backstage.io/edit-url': - 'https://github.com/orgs/test-org/teams/new-team/edit', - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'github.com/team-slug': 'test-org/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - profile: { - displayName: 'New Team', - }, - }, - }; - - mockCatalogApi.getEntities.mockResolvedValue({ - items: [cloneDeep(entity)], - }); await entityProvider.onEvent(event); + await new Promise(process.nextTick); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [ + entities: [ { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['a'], + }, + }, locationKey: 'github-org-provider:my-id', - entity: entity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: expectedEntity, }, ], + type: 'full', }); }); @@ -768,18 +784,60 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); entityProvider.connect(entityProviderConnection); const event: EventParams = { @@ -804,109 +862,68 @@ describe('GithubOrgEntityProvider', () => { }, }; - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: [], - }, - }; - - const expectedGroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1', 'githubuser'], - }, - }; - - const expectedUserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - - mockCatalogApi.getEntityByRef - .mockResolvedValueOnce(cloneDeep(groupEntity)) - .mockResolvedValueOnce(cloneDeep(userEntity)); - await entityProvider.onEvent(event); + await new Promise(process.nextTick); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [ + entities: [ { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, locationKey: 'github-org-provider:my-id', - entity: groupEntity, }, { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['a'], + }, + }, locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: expectedGroupEntity, - }, - { - locationKey: 'github-org-provider:my-id', - entity: expectedUserEntity, }, ], + type: 'full', }); }); @@ -930,18 +947,60 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); entityProvider.connect(entityProviderConnection); const event: EventParams = { @@ -966,109 +1025,68 @@ describe('GithubOrgEntityProvider', () => { }, }; - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1', 'githubuser'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - - const expectedGroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const expectedUserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: [], - }, - }; - - mockCatalogApi.getEntityByRef - .mockResolvedValueOnce(cloneDeep(groupEntity)) - .mockResolvedValueOnce(cloneDeep(userEntity)); - await entityProvider.onEvent(event); + await new Promise(process.nextTick); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [ + entities: [ { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, locationKey: 'github-org-provider:my-id', - entity: groupEntity, }, { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['a'], + }, + }, locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: expectedGroupEntity, - }, - { - locationKey: 'github-org-provider:my-id', - entity: expectedUserEntity, }, ], + type: 'full', }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 7d7941f5ec..9b11f81697 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -18,10 +18,7 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, - DEFAULT_NAMESPACE, Entity, - GroupEntity, - UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -43,12 +40,8 @@ import { OrganizationMemberAddedEvent, OrganizationMemberRemovedEvent, TeamEvent, - TeamEditedEvent, - MembershipEvent, } from '@octokit/webhooks-types'; -import { CatalogApi } from '@backstage/catalog-client'; -import { TokenManager } from '@backstage/backend-common'; -import { merge, omit } from 'lodash'; +import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { @@ -63,13 +56,9 @@ import { } from '../lib'; import { TeamTransformer, UserTransformer } from '../lib'; import { - addUserToGroup, - removeUserFromGroup, createAddEntitiesOperation, createRemoveEntitiesOperation, - createReplaceEntitiesOperation, DeferredEntitiesBuilder, - EntityUpdateOperation, } from '../lib/github'; /** @@ -126,9 +115,6 @@ export interface GithubOrgEntityProviderOptions { * Optionally include a team transformer for transforming from GitHub teams to Group Entities */ teamTransformer?: TeamTransformer; - - catalogApi?: CatalogApi; - tokenManager?: TokenManager; } /** @@ -143,8 +129,6 @@ export class GithubOrgEntityProvider private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; - private eventConfigErrorThrown = false; - static fromConfig(config: Config, options: GithubOrgEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; @@ -169,8 +153,6 @@ export class GithubOrgEntityProvider DefaultGithubCredentialsProvider.fromIntegrations(integrations), userTransformer: options.userTransformer, teamTransformer: options.teamTransformer, - catalogApi: options.catalogApi, - tokenManager: options.tokenManager, }); provider.schedule(options.schedule); @@ -187,8 +169,6 @@ export class GithubOrgEntityProvider githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; - catalogApi?: CatalogApi; - tokenManager?: TokenManager; }, ) { this.credentialsProvider = @@ -266,11 +246,6 @@ export class GithubOrgEntityProvider const { logger } = this.options; logger.debug(`Received event from ${params.topic}`); - if (!this.canHandleEvents()) { - logger.debug(`Skipping event ${params.topic}`); - return; - } - const addEntitiesOperation = createAddEntitiesOperation( this.options.id, this.options.gitHubConfig.host, @@ -279,10 +254,6 @@ export class GithubOrgEntityProvider this.options.id, this.options.gitHubConfig.host, ); - const replaceEntitiesOperation = createReplaceEntitiesOperation( - this.options.id, - this.options.gitHubConfig.host, - ); // handle change users in the org // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization @@ -312,26 +283,14 @@ export class GithubOrgEntityProvider : removeEntitiesOperation; await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation); } else if (teamEvent.action === 'edited') { - await this.onTeamEditedInOrganization( - teamEvent, - replaceEntitiesOperation, - ); + this.onEventToRebuildOrg(); } } // handle change membership in the org // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership if (params.topic.includes('membership')) { - const membershipEvent = params.eventPayload as MembershipEvent; - const updateOperation = - membershipEvent.action === 'added' - ? addUserToGroup - : removeUserFromGroup; - await this.onMemberChangeToTeam( - membershipEvent, - replaceEntitiesOperation, - updateOperation, - ); + this.onEventToRebuildOrg(); } return; @@ -342,22 +301,6 @@ export class GithubOrgEntityProvider return ['github.organization', 'github.team', 'github.membership']; } - private canHandleEvents(): boolean { - if (this.options.catalogApi && this.options.tokenManager) { - return true; - } - - // throw only once - if (!this.eventConfigErrorThrown) { - this.eventConfigErrorThrown = true; - throw new Error( - `${this.getProviderName()} not well configured to handle ${this.supportsEventTopics()}. Missing CatalogApi and/or TokenManager.`, - ); - } - - return false; - } - private async onTeamChangeInOrganization( event: TeamEvent, createDeltaOperation: DeferredEntitiesBuilder, @@ -412,71 +355,14 @@ export class GithubOrgEntityProvider }); } - private async onTeamEditedInOrganization( - event: TeamEditedEvent, - createDeltaOperation: DeferredEntitiesBuilder, - ) { - if (!this.connection) { - throw new Error('Not initialized'); - } - - const { name } = event.changes; - const org = event.organization.login; - - const { token } = await this.options.tokenManager!.getToken(); - const groups = await this.options.catalogApi!.getEntities( - { - filter: [ - { - kind: 'Group', - [`metadata.annotations.github.com/team-id`]: `${event.team.id}`, - }, - ], - fields: ['apiVersion', 'kind', 'metadata', 'spec'], - }, - { token }, - ); - - if (groups.items.length === 0) { - this.options.logger.debug( - `Skipping event because couldn't found group with 'github.com/node-id':${event.team.node_id}`, - ); - return; - } - - const group = groups.items[0] as GroupEntity; - const { removed } = createDeltaOperation(org, [group]); - const { name: newTeamName, html_url: url, description, slug } = event.team; - - if (name) { - merge(group, { - metadata: { - name: slug, - annotations: { - [`backstage.io/edit-url`]: `${url}/edit`, - [`github.com/team-slug`]: `${org}/${slug}`, - [ANNOTATION_LOCATION]: `url:${url}`, - [ANNOTATION_ORIGIN_LOCATION]: `url:${url}`, - }, - }, - spec: { - profile: { - displayName: newTeamName, - }, - }, + private onEventToRebuildOrg() { + try { + this.read(this.options).then(() => { + this.options.logger.debug('Event process finished'); }); + } catch (e) { + this.options.logger.debug(`Event process error ${e}`); } - - if (description) { - group.metadata.description = description; - } - - const { added } = createDeltaOperation(org, [group]); - await this.connection.applyMutation({ - type: 'delta', - removed, - added, - }); } private async onMemberChangeInOrganization( @@ -523,73 +409,6 @@ export class GithubOrgEntityProvider }); } - private async onMemberChangeToTeam( - event: MembershipEvent, - createDeltaOperation: DeferredEntitiesBuilder, - updateEntities: EntityUpdateOperation, - ) { - if (!this.connection) { - throw new Error('Not initialized'); - } - - // The docs are saying I will receive the slug for the removed event, but the types don't reflect that, - // so I will just check to be sure the slug is there - // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership - if (!('slug' in event.team)) { - return; - } - const teamSlug = event.team.slug; - const userSlug = event.member.login; - - const { token } = await this.options.tokenManager!.getToken(); - const org = event.organization.login; - let group = (await this.options.catalogApi!.getEntityByRef( - { - kind: 'Group', - namespace: DEFAULT_NAMESPACE, - name: teamSlug, - }, - { token }, - )) as GroupEntity; - - if (!group) { - this.options.logger.debug( - `Skipping event because couldn't found group ':${teamSlug} for namespace ${DEFAULT_NAMESPACE}`, - ); - return; - } - - let user = (await this.options.catalogApi!.getEntityByRef( - { - kind: 'User', - namespace: DEFAULT_NAMESPACE, - name: userSlug, - }, - { token }, - )) as UserEntity; - - group = omit(group, 'relations'); - - if (!user) { - this.options.logger.debug( - `Skipping event because couldn't found user ':${userSlug} for namespace ${DEFAULT_NAMESPACE}`, - ); - return; - } - user = omit(user, 'relations'); - - const { removed } = createDeltaOperation(org, [group, user]); - - const result = updateEntities(user, group); - - const { added } = createDeltaOperation(org, [result.group, result.user]); - await this.connection.applyMutation({ - type: 'delta', - removed, - added, - }); - } - private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return; From 8c758fed63fc8399c35ce32bc8f53fd788205711 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Mon, 2 Jan 2023 20:00:59 +0000 Subject: [PATCH 11/11] refct(events,catalog/github): change membership and team.edit handlers Signed-off-by: Rogerio Angeliski --- .../api-report.md | 1 - .../src/lib/defaultTransformers.ts | 4 - .../src/lib/github.test.ts | 48 ++- .../src/lib/github.ts | 182 ++++++++- .../providers/GithubOrgEntityProvider.test.ts | 371 ++++++++++++++---- .../src/providers/GithubOrgEntityProvider.ts | 177 ++++++++- 6 files changed, 671 insertions(+), 112 deletions(-) diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 4b07ffd593..590b7d2d6b 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -246,7 +246,6 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { // @public export type GithubTeam = { - databaseId: number; slug: string; combinedSlug: string; name?: string; diff --git a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts index 361942543b..efd3eedd5d 100644 --- a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts @@ -90,10 +90,6 @@ export const defaultOrganizationTeamTransformer: TeamTransformer = 'github.com/team-slug': team.combinedSlug, }; - if (team.databaseId) { - annotations['github.com/team-id'] = `${team.databaseId}`; - } - if (team.editTeamUrl) { annotations['backstage.io/edit-url'] = team.editTeamUrl; } diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index a9bcf2c49b..387dc8a2ee 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -31,6 +31,7 @@ import { GithubTeam, createAddEntitiesOperation, createRemoveEntitiesOperation, + createReplaceEntitiesOperation, } from './github'; import fetch from 'node-fetch'; @@ -206,7 +207,6 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { - databaseId: 1, slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -214,7 +214,6 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - databaseId: 2, slug: 'parent', combinedSlug: '', members: [], @@ -239,7 +238,6 @@ describe('github', () => { description: 'The one and only team', annotations: { 'github.com/team-slug': 'blah/team', - 'github.com/team-id': '1', 'backstage.io/edit-url': 'http://example.com/orgs/blah/teams/team/edit', }, @@ -309,7 +307,6 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { - databaseId: 1, slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -317,7 +314,6 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - databaseId: 3, slug: 'parent', combinedSlug: '', members: [], @@ -376,7 +372,6 @@ describe('github', () => { pageInfo: { hasNextPage: false }, nodes: [ { - databaseId: 1, slug: 'team', combinedSlug: 'blah/team', name: 'Team', @@ -384,7 +379,6 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - databaseId: 3, slug: 'parent', combinedSlug: '', members: [], @@ -395,7 +389,6 @@ describe('github', () => { }, }, { - databaseId: 2, slug: 'team', combinedSlug: 'blah/team', name: 'aa', @@ -403,7 +396,6 @@ describe('github', () => { avatarUrl: 'http://example.com/team.jpeg', editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', parentTeam: { - databaseId: 3, slug: 'parent', combinedSlug: '', members: [], @@ -464,7 +456,6 @@ describe('github', () => { const input: QueryResponse = { organization: { team: { - databaseId: 1, slug: '', combinedSlug: '', members: { @@ -635,4 +626,41 @@ describe('github', () => { }); }); }); + describe('createReplaceEntitiesOperation', () => { + it('create a function to replace deferred entities to a delta operation', () => { + const operation = createReplaceEntitiesOperation('my-id', 'host'); + + const userEntity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: ['new-team'], + }, + }; + expect(operation('org', [userEntity])).toEqual({ + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + added: [ + { + locationKey: 'github-org-provider:my-id', + entity: userEntity, + }, + ], + }); + }); + }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index d33ef4fb34..f719e3d506 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -71,7 +71,6 @@ export type GithubUser = { * @public */ export type GithubTeam = { - databaseId: number; slug: string; combinedSlug: string; name?: string; @@ -186,7 +185,6 @@ export async function getOrganizationTeams( teams(first: 100, after: $cursor) { pageInfo { hasNextPage, endCursor } nodes { - databaseId slug combinedSlug name @@ -196,7 +194,14 @@ export async function getOrganizationTeams( parentTeam { slug } members(first: 100, membership: IMMEDIATE) { pageInfo { hasNextPage } - nodes { login } + nodes { + avatarUrl, + bio, + email, + login, + name, + organizationVerifiedDomainEmails(login: $org) + } } } } @@ -243,6 +248,164 @@ export async function getOrganizationTeams( return { groups }; } +export async function getOrganizationTeamsFromUsers( + client: typeof graphql, + org: string, + userLogins: string[], + teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, +): Promise<{ + groups: GroupEntity[]; +}> { + const query = ` + query teams($org: String!, $cursor: String, $userLogins: [String!] = "") { + organization(login: $org) { + teams(first: 100, after: $cursor, userLogins: $userLogins) { + pageInfo { + hasNextPage + endCursor + } + nodes { + slug + combinedSlug + name + description + avatarUrl + editTeamUrl + parentTeam { + slug + } + members(first: 100, membership: IMMEDIATE) { + pageInfo { + hasNextPage + } + nodes { + avatarUrl, + bio, + email, + login, + name, + organizationVerifiedDomainEmails(login: $org) + } + } + } + } + } +}`; + + const materialisedTeams = async ( + item: GithubTeamResponse, + ctx: TransformerContext, + ): Promise => { + const memberNames: GithubUser[] = []; + + if (!item.members.pageInfo.hasNextPage) { + // We got all the members in one go, run the fast path + for (const user of item.members.nodes) { + memberNames.push(user); + } + } else { + // There were more than a hundred immediate members - run the slow + // path of fetching them explicitly + const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug); + for (const userLogin of members) { + memberNames.push(userLogin); + } + } + + const team: GithubTeam = { + ...item, + members: memberNames, + }; + + return await teamTransformer(team, ctx); + }; + + const groups = await queryWithPaging( + client, + query, + org, + r => r.organization?.teams, + materialisedTeams, + { org, userLogins }, + ); + + return { groups }; +} + +export async function getOrganizationTeam( + client: typeof graphql, + org: string, + teamSlug: string, + teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, +): Promise<{ + group: GroupEntity; +}> { + const query = ` + query teams($org: String!, $teamSlug: String!) { + organization(login: $org) { + team(slug:$teamSlug) { + slug + combinedSlug + name + description + avatarUrl + editTeamUrl + parentTeam { slug } + members(first: 100, membership: IMMEDIATE) { + pageInfo { hasNextPage } + nodes { login } + } + } + } + }`; + + const materialisedTeam = async ( + item: GithubTeamResponse, + ctx: TransformerContext, + ): Promise => { + const memberNames: GithubUser[] = []; + + if (!item.members.pageInfo.hasNextPage) { + // We got all the members in one go, run the fast path + for (const user of item.members.nodes) { + memberNames.push(user); + } + } else { + // There were more than a hundred immediate members - run the slow + // path of fetching them explicitly + const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug); + for (const userLogin of members) { + memberNames.push(userLogin); + } + } + + const team: GithubTeam = { + ...item, + members: memberNames, + }; + + return await teamTransformer(team, ctx); + }; + + const response: QueryResponse = await client(query, { + org, + teamSlug, + }); + + if (!response.organization?.team) + throw new Error(`Found no match for group ${teamSlug}`); + + const group = await materialisedTeam(response.organization?.team, { + query, + client, + org, + }); + + if (!group) throw new Error(`Can't transform for group ${teamSlug}`); + + return { group }; +} + export async function getOrganizationRepositories( client: typeof graphql, org: string, @@ -435,3 +598,16 @@ export const createRemoveEntitiesOperation = entity: withLocations(`https://${host}`, org, entity), })), }); + +export const createReplaceEntitiesOperation = + (id: string, host: string) => (org: string, entities: Entity[]) => { + const entitiesToReplace = entities.map(entity => ({ + locationKey: `github-org-provider:${id}`, + entity: withLocations(`https://${host}`, org, entity), + })); + + return { + removed: entitiesToReplace, + added: entitiesToReplace, + }; + }; diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index 270f184d7a..c24cd573e0 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -628,6 +628,26 @@ describe('GithubOrgEntityProvider', () => { const mockClient = jest.fn(); mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'githubuser' }], + }, + }, + }, + }) .mockResolvedValueOnce({ organization: { membersWithRole: { @@ -640,6 +660,13 @@ describe('GithubOrgEntityProvider', () => { email: 'd', avatarUrl: 'e', }, + { + login: 'githubuser', + name: 'githubuser', + bio: 'githubuser', + email: 'd', + avatarUrl: 'e', + }, ], }, }, @@ -662,7 +689,7 @@ describe('GithubOrgEntityProvider', () => { }, members: { pageInfo: { hasNextPage: false }, - nodes: [{ login: 'a' }], + nodes: [{ login: 'a' }, { login: 'githubuser' }], }, }, ], @@ -680,7 +707,7 @@ describe('GithubOrgEntityProvider', () => { action: 'edited', changes: { name: { - from: 'mygroup', + from: 'mygroup with spaces', }, }, team: { @@ -704,11 +731,10 @@ describe('GithubOrgEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - entities: [ + added: [ { + locationKey: 'github-org-provider:my-id', entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', metadata: { annotations: { 'backstage.io/managed-by-location': @@ -717,24 +743,50 @@ describe('GithubOrgEntityProvider', () => { 'url:https://github.com/a', 'github.com/user-login': 'a', }, - description: 'c', name: 'a', + description: 'c', }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', spec: { - memberOf: ['team'], profile: { displayName: 'b', email: 'd', picture: 'e', }, + memberOf: ['team'], }, }, - locationKey: 'github-org-provider:my-id', }, { + locationKey: 'github-org-provider:my-id', entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: ['team'], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { metadata: { annotations: { 'backstage.io/managed-by-location': @@ -746,21 +798,50 @@ describe('GithubOrgEntityProvider', () => { name: 'team', description: 'The one and only team', }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', spec: { - children: [], - parent: 'parent', + type: 'team', profile: { displayName: 'Team', picture: 'http://example.com/team.jpeg', }, - type: 'team', - members: ['a'], + children: [], + parent: 'parent', + members: ['a', 'githubuser'], }, }, - locationKey: 'github-org-provider:my-id', }, ], - type: 'full', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/mygroup-with-spaces', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/mygroup-with-spaces', + }, + name: 'mygroup-with-spaces', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a', 'githubuser'], + }, + }, + }, + ], + type: 'delta', }); }); @@ -795,6 +876,26 @@ describe('GithubOrgEntityProvider', () => { const mockClient = jest.fn(); mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'githubuser' }], + }, + }, + }, + }) .mockResolvedValueOnce({ organization: { membersWithRole: { @@ -807,6 +908,13 @@ describe('GithubOrgEntityProvider', () => { email: 'd', avatarUrl: 'e', }, + { + login: 'githubuser', + name: 'githubuser', + bio: 'githubuser', + email: 'd', + avatarUrl: 'e', + }, ], }, }, @@ -829,7 +937,7 @@ describe('GithubOrgEntityProvider', () => { }, members: { pageInfo: { hasNextPage: false }, - nodes: [{ login: 'a' }], + nodes: [{ login: 'a' }, { login: 'githubuser' }], }, }, ], @@ -867,37 +975,36 @@ describe('GithubOrgEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - entities: [ + added: [ { + locationKey: 'github-org-provider:my-id', entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://github.com/a', + 'url:https://github.com/githubuser', 'backstage.io/managed-by-origin-location': - 'url:https://github.com/a', - 'github.com/user-login': 'a', + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', }, - description: 'c', - name: 'a', + name: 'githubuser', + description: 'githubuser', }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', spec: { - memberOf: ['team'], profile: { - displayName: 'b', + displayName: 'githubuser', email: 'd', picture: 'e', }, + memberOf: ['team'], }, }, - locationKey: 'github-org-provider:my-id', }, { + locationKey: 'github-org-provider:my-id', entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', metadata: { annotations: { 'backstage.io/managed-by-location': @@ -909,21 +1016,78 @@ describe('GithubOrgEntityProvider', () => { name: 'team', description: 'The one and only team', }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', spec: { - children: [], - parent: 'parent', + type: 'team', profile: { displayName: 'Team', picture: 'http://example.com/team.jpeg', }, - type: 'team', - members: ['a'], + children: [], + parent: 'parent', + members: ['a', 'githubuser'], }, }, - locationKey: 'github-org-provider:my-id', }, ], - type: 'full', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: ['team'], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a', 'githubuser'], + }, + }, + }, + ], + type: 'delta', }); }); @@ -958,15 +1122,35 @@ describe('GithubOrgEntityProvider', () => { const mockClient = jest.fn(); mockClient + .mockResolvedValueOnce({ + organization: { + team: { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + }, + }) .mockResolvedValueOnce({ organization: { membersWithRole: { pageInfo: { hasNextPage: false }, nodes: [ { - login: 'a', - name: 'b', - bio: 'c', + login: 'githubuser', + name: 'githubuser', + bio: 'githubuser', email: 'd', avatarUrl: 'e', }, @@ -978,24 +1162,7 @@ describe('GithubOrgEntityProvider', () => { organization: { teams: { pageInfo: { hasNextPage: false }, - nodes: [ - { - slug: 'team', - combinedSlug: 'blah/team', - name: 'Team', - description: 'The one and only team', - avatarUrl: 'http://example.com/team.jpeg', - parentTeam: { - slug: 'parent', - combinedSlug: '', - members: { pageInfo: { hasNextPage: false }, nodes: [] }, - }, - members: { - pageInfo: { hasNextPage: false }, - nodes: [{ login: 'a' }], - }, - }, - ], + nodes: [], }, }, }); @@ -1030,37 +1197,36 @@ describe('GithubOrgEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - entities: [ + added: [ { + locationKey: 'github-org-provider:my-id', entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', metadata: { annotations: { 'backstage.io/managed-by-location': - 'url:https://github.com/a', + 'url:https://github.com/githubuser', 'backstage.io/managed-by-origin-location': - 'url:https://github.com/a', - 'github.com/user-login': 'a', + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', }, - description: 'c', - name: 'a', + name: 'githubuser', + description: 'githubuser', }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', spec: { - memberOf: ['team'], profile: { - displayName: 'b', + displayName: 'githubuser', email: 'd', picture: 'e', }, + memberOf: [], }, }, - locationKey: 'github-org-provider:my-id', }, { + locationKey: 'github-org-provider:my-id', entity: { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', metadata: { annotations: { 'backstage.io/managed-by-location': @@ -1072,21 +1238,78 @@ describe('GithubOrgEntityProvider', () => { name: 'team', description: 'The one and only team', }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', spec: { - children: [], - parent: 'parent', + type: 'team', profile: { displayName: 'Team', picture: 'http://example.com/team.jpeg', }, - type: 'team', + children: [], + parent: 'parent', members: ['a'], }, }, - locationKey: 'github-org-provider:my-id', }, ], - type: 'full', + removed: [ + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + name: 'githubuser', + description: 'githubuser', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + spec: { + profile: { + displayName: 'githubuser', + email: 'd', + picture: 'e', + }, + memberOf: [], + }, + }, + }, + { + locationKey: 'github-org-provider:my-id', + entity: { + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + spec: { + type: 'team', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + children: [], + parent: 'parent', + members: ['a'], + }, + }, + }, + ], + type: 'delta', }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 9b11f81697..46ad089625 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -40,6 +40,8 @@ import { OrganizationMemberAddedEvent, OrganizationMemberRemovedEvent, TeamEvent, + TeamEditedEvent, + MembershipEvent, } from '@octokit/webhooks-types'; import { merge } from 'lodash'; import * as uuid from 'uuid'; @@ -58,7 +60,10 @@ import { TeamTransformer, UserTransformer } from '../lib'; import { createAddEntitiesOperation, createRemoveEntitiesOperation, + createReplaceEntitiesOperation, DeferredEntitiesBuilder, + getOrganizationTeam, + getOrganizationTeamsFromUsers, } from '../lib/github'; /** @@ -255,6 +260,11 @@ export class GithubOrgEntityProvider this.options.gitHubConfig.host, ); + const replaceEntitiesOperation = createReplaceEntitiesOperation( + this.options.id, + this.options.gitHubConfig.host, + ); + // handle change users in the org // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization if (params.topic.includes('organization')) { @@ -283,14 +293,21 @@ export class GithubOrgEntityProvider : removeEntitiesOperation; await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation); } else if (teamEvent.action === 'edited') { - this.onEventToRebuildOrg(); + await this.onTeamEditedInOrganization( + teamEvent, + replaceEntitiesOperation, + ); } } // handle change membership in the org // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership if (params.topic.includes('membership')) { - this.onEventToRebuildOrg(); + const membershipEvent = params.eventPayload as MembershipEvent; + this.onMembershipChangedInOrganization( + membershipEvent, + replaceEntitiesOperation, + ); } return; @@ -301,6 +318,143 @@ export class GithubOrgEntityProvider return ['github.organization', 'github.team', 'github.membership']; } + private async onTeamEditedInOrganization( + event: TeamEditedEvent, + createDeltaOperation: DeferredEntitiesBuilder, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const teamSlug = event.team.slug; + const { headers, type: tokenType } = + await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGithubOrgUrl(this.options.orgUrl); + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.options.teamTransformer, + ); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const usersFromChangedGroup = group.spec.members || []; + const usersToRebuild = users.filter(u => + usersFromChangedGroup.includes(u.metadata.name), + ); + + const { groups } = await getOrganizationTeamsFromUsers( + client, + org, + usersToRebuild.map(u => u.metadata.name), + this.options.teamTransformer, + ); + + assignGroupsToUsers(usersToRebuild, groups); + buildOrgHierarchy(groups); + + const oldName = event.changes.name?.from || ''; + const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-'); + + const { removed } = createDeltaOperation(org, [ + { + ...group, + metadata: { + name: oldSlug, + }, + }, + ]); + const { added } = createDeltaOperation(org, [...usersToRebuild, ...groups]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + + private async onMembershipChangedInOrganization( + event: MembershipEvent, + createDeltaOperation: DeferredEntitiesBuilder, + ) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + // The docs are saying I will receive the slug for the removed event, + // but the types don't reflect that, + // so I will just check to be sure the slug is there + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership + if (!('slug' in event.team)) { + return; + } + + const teamSlug = event.team.slug; + const userLogin = event.member.login; + const { headers, type: tokenType } = + await this.credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGithubOrgUrl(this.options.orgUrl); + const { group } = await getOrganizationTeam( + client, + org, + teamSlug, + this.options.teamTransformer, + ); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const usersToRebuild = users.filter(u => u.metadata.name === userLogin); + + const { groups } = await getOrganizationTeamsFromUsers( + client, + org, + [userLogin], + this.options.teamTransformer, + ); + + // we include group because the removed event need to update the old group too + if (!groups.some(g => g.metadata.name === group.metadata.name)) { + groups.push(group); + } + + assignGroupsToUsers(usersToRebuild, groups); + buildOrgHierarchy(groups); + + const { added, removed } = createDeltaOperation(org, [ + ...usersToRebuild, + ...groups, + ]); + await this.connection.applyMutation({ + type: 'delta', + removed, + added, + }); + } + private async onTeamChangeInOrganization( event: TeamEvent, createDeltaOperation: DeferredEntitiesBuilder, @@ -311,13 +465,7 @@ export class GithubOrgEntityProvider const organizationTeamTransformer = this.options.teamTransformer || defaultOrganizationTeamTransformer; - const { - id: databaseId, - name, - html_url: url, - description, - slug, - } = event.team; + const { name, html_url: url, description, slug } = event.team; const org = event.organization.login; const { headers } = await this.credentialsProvider.getCredentials({ url: this.options.orgUrl, @@ -329,7 +477,6 @@ export class GithubOrgEntityProvider const group = (await organizationTeamTransformer( { - databaseId, name, slug, editTeamUrl: `${url}/edit`, @@ -355,16 +502,6 @@ export class GithubOrgEntityProvider }); } - private onEventToRebuildOrg() { - try { - this.read(this.options).then(() => { - this.options.logger.debug('Event process finished'); - }); - } catch (e) { - this.options.logger.debug(`Event process error ${e}`); - } - } - private async onMemberChangeInOrganization( event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent, createDeltaOperation: DeferredEntitiesBuilder,