Merge pull request #14870 from angeliski/add-github-org-events-handler

Implements EventSubscriber in GithubOrgEntityProvider
This commit is contained in:
Johan Haals
2023-01-03 14:16:01 +01:00
committed by GitHub
7 changed files with 1778 additions and 7 deletions
+10
View File
@@ -0,0 +1,10 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Added support for event based updates in the `GithubOrgEntityProvider`!
Based on webhook events from GitHub the affected `User` or `Group` entity will be refreshed.
This includes adding new entities, refreshing existing ones, and removing obsolete ones.
Please find more information at
https://backstage.io/docs/integrations/github/org#installation-with-events-support
+47 -1
View File
@@ -17,7 +17,7 @@ entities that mirror your org setup.
> provide authentication. See the
> [GitHub auth provider](../../auth/github/provider.md) for that.
## Installation
## Installation without Events Support
This guide will use the Entity Provider method. If you for some reason prefer
the Processor method (not recommended), it is described separately below.
@@ -60,6 +60,52 @@ schedule it:
+ );
```
## Installation with Events Support
Please follow the installation instructions at
- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md
- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md
Additionally, you need to decide how you want to receive events from external sources like
- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
Set up your provider
```diff
// packages/backend/src/plugins/catalogEventBasedProviders.ts
+import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { PluginEnvironment } from '../types';
export default async function createCatalogEventBasedProviders(
- _: PluginEnvironment,
+ env: PluginEnvironment,
): Promise<Array<EntityProvider & EventSubscriber>> {
const providers: Array<
(EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
> = [];
- // add your event-based entity providers here
+ providers.push(
+ GithubOrgEntityProvider.fromConfig(env.config, {
+ id: 'production',
+ orgUrl: 'https://github.com/backstage',
+ logger: env.logger,
+ schedule: env.scheduler.createScheduledTaskRunner({
+ frequency: { minutes: 60 },
+ timeout: { minutes: 15 },
+ }),
+ }),
+ );
return providers.flat();
}
```
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks).
The webhook will need to be configured to forward `organization`,`team` and `membership` events.
## Configuration
As mentioned above, you also must have some configuration in your app-config
@@ -178,7 +178,9 @@ export class GitHubOrgEntityProvider extends GithubOrgEntityProvider {
}
// @public
export class GithubOrgEntityProvider implements EntityProvider {
export class GithubOrgEntityProvider
implements EntityProvider, EventSubscriber
{
constructor(options: {
id: string;
orgUrl: string;
@@ -197,7 +199,11 @@ export class GithubOrgEntityProvider implements EntityProvider {
): GithubOrgEntityProvider;
// (undocumented)
getProviderName(): string;
// (undocumented)
onEvent(params: EventParams): Promise<void>;
read(options?: { logger?: Logger }): Promise<void>;
// (undocumented)
supportsEventTopics(): string[];
}
// @public @deprecated (undocumented)
@@ -29,6 +29,9 @@ import {
QueryResponse,
GithubUser,
GithubTeam,
createAddEntitiesOperation,
createRemoveEntitiesOperation,
createReplaceEntitiesOperation,
} from './github';
import fetch from 'node-fetch';
@@ -557,4 +560,107 @@ describe('github', () => {
).resolves.toEqual(output);
});
});
describe('createAddEntitiesOperation', () => {
it('create a function to add deferred entities to a delta operation', () => {
const operation = createAddEntitiesOperation('my-id', 'host');
const userEntity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'githubuser',
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/githubuser',
'backstage.io/managed-by-origin-location':
'url:https://github.com/githubuser',
'github.com/user-login': 'githubuser',
},
},
spec: {
memberOf: ['new-team'],
},
};
expect(operation('org', [userEntity])).toEqual({
added: [
{
locationKey: 'github-org-provider:my-id',
entity: userEntity,
},
],
removed: [],
});
});
});
describe('createRemoveEntitiesOperation', () => {
it('create a function to remove deferred entities to a delta operation', () => {
const operation = createRemoveEntitiesOperation('my-id', 'host');
const userEntity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'githubuser',
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/githubuser',
'backstage.io/managed-by-origin-location':
'url:https://github.com/githubuser',
'github.com/user-login': 'githubuser',
},
},
spec: {
memberOf: ['new-team'],
},
};
expect(operation('org', [userEntity])).toEqual({
removed: [
{
locationKey: 'github-org-provider:my-id',
entity: userEntity,
},
],
added: [],
});
});
});
describe('createReplaceEntitiesOperation', () => {
it('create a function to replace deferred entities to a delta operation', () => {
const operation = createReplaceEntitiesOperation('my-id', 'host');
const userEntity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: 'githubuser',
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/githubuser',
'backstage.io/managed-by-origin-location':
'url:https://github.com/githubuser',
'github.com/user-login': 'githubuser',
},
},
spec: {
memberOf: ['new-team'],
},
};
expect(operation('org', [userEntity])).toEqual({
removed: [
{
locationKey: 'github-org-provider:my-id',
entity: userEntity,
},
],
added: [
{
locationKey: 'github-org-provider:my-id',
entity: userEntity,
},
],
});
});
});
});
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model';
import { GithubCredentialType } from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import {
@@ -24,6 +24,9 @@ import {
TransformerContext,
UserTransformer,
} from './defaultTransformers';
import { withLocations } from '../providers/GithubOrgEntityProvider';
import { DeferredEntity } from '@backstage/plugin-catalog-backend';
// Graphql types
@@ -191,7 +194,14 @@ export async function getOrganizationTeams(
parentTeam { slug }
members(first: 100, membership: IMMEDIATE) {
pageInfo { hasNextPage }
nodes { login }
nodes {
avatarUrl,
bio,
email,
login,
name,
organizationVerifiedDomainEmails(login: $org)
}
}
}
}
@@ -238,6 +248,164 @@ export async function getOrganizationTeams(
return { groups };
}
export async function getOrganizationTeamsFromUsers(
client: typeof graphql,
org: string,
userLogins: string[],
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
): Promise<{
groups: GroupEntity[];
}> {
const query = `
query teams($org: String!, $cursor: String, $userLogins: [String!] = "") {
organization(login: $org) {
teams(first: 100, after: $cursor, userLogins: $userLogins) {
pageInfo {
hasNextPage
endCursor
}
nodes {
slug
combinedSlug
name
description
avatarUrl
editTeamUrl
parentTeam {
slug
}
members(first: 100, membership: IMMEDIATE) {
pageInfo {
hasNextPage
}
nodes {
avatarUrl,
bio,
email,
login,
name,
organizationVerifiedDomainEmails(login: $org)
}
}
}
}
}
}`;
const materialisedTeams = async (
item: GithubTeamResponse,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> => {
const memberNames: GithubUser[] = [];
if (!item.members.pageInfo.hasNextPage) {
// We got all the members in one go, run the fast path
for (const user of item.members.nodes) {
memberNames.push(user);
}
} else {
// There were more than a hundred immediate members - run the slow
// path of fetching them explicitly
const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
for (const userLogin of members) {
memberNames.push(userLogin);
}
}
const team: GithubTeam = {
...item,
members: memberNames,
};
return await teamTransformer(team, ctx);
};
const groups = await queryWithPaging(
client,
query,
org,
r => r.organization?.teams,
materialisedTeams,
{ org, userLogins },
);
return { groups };
}
export async function getOrganizationTeam(
client: typeof graphql,
org: string,
teamSlug: string,
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
): Promise<{
group: GroupEntity;
}> {
const query = `
query teams($org: String!, $teamSlug: String!) {
organization(login: $org) {
team(slug:$teamSlug) {
slug
combinedSlug
name
description
avatarUrl
editTeamUrl
parentTeam { slug }
members(first: 100, membership: IMMEDIATE) {
pageInfo { hasNextPage }
nodes { login }
}
}
}
}`;
const materialisedTeam = async (
item: GithubTeamResponse,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> => {
const memberNames: GithubUser[] = [];
if (!item.members.pageInfo.hasNextPage) {
// We got all the members in one go, run the fast path
for (const user of item.members.nodes) {
memberNames.push(user);
}
} else {
// There were more than a hundred immediate members - run the slow
// path of fetching them explicitly
const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
for (const userLogin of members) {
memberNames.push(userLogin);
}
}
const team: GithubTeam = {
...item,
members: memberNames,
};
return await teamTransformer(team, ctx);
};
const response: QueryResponse = await client(query, {
org,
teamSlug,
});
if (!response.organization?.team)
throw new Error(`Found no match for group ${teamSlug}`);
const group = await materialisedTeam(response.organization?.team, {
query,
client,
org,
});
if (!group) throw new Error(`Can't transform for group ${teamSlug}`);
return { group };
}
export async function getOrganizationRepositories(
client: typeof graphql,
org: string,
@@ -349,6 +517,7 @@ export async function getTeamMembers(
*
* @param client - The octokit client
* @param query - The query to execute
* @param org - The slug of the org to read
* @param connection - A function that, given the response, picks out the actual
* Connection object that's being iterated
* @param transformer - A function that, given one of the nodes in the Connection,
@@ -406,3 +575,39 @@ export async function queryWithPaging<
return result;
}
export type DeferredEntitiesBuilder = (
org: string,
entities: Entity[],
) => { added: DeferredEntity[]; removed: DeferredEntity[] };
export const createAddEntitiesOperation =
(id: string, host: string) => (org: string, entities: Entity[]) => ({
removed: [],
added: entities.map(entity => ({
locationKey: `github-org-provider:${id}`,
entity: withLocations(`https://${host}`, org, entity),
})),
});
export const createRemoveEntitiesOperation =
(id: string, host: string) => (org: string, entities: Entity[]) => ({
added: [],
removed: entities.map(entity => ({
locationKey: `github-org-provider:${id}`,
entity: withLocations(`https://${host}`, org, entity),
})),
});
export const createReplaceEntitiesOperation =
(id: string, host: string) => (org: string, entities: Entity[]) => {
const entitiesToReplace = entities.map(entity => ({
locationKey: `github-org-provider:${id}`,
entity: withLocations(`https://${host}`, org, entity),
}));
return {
removed: entitiesToReplace,
added: entitiesToReplace,
};
};
@@ -28,22 +28,43 @@ import {
ScmIntegrations,
SingleInstanceGithubCredentialsProvider,
} from '@backstage/integration';
import { EventParams } from '@backstage/plugin-events-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-backend';
import { graphql } from '@octokit/graphql';
import {
OrganizationEvent,
OrganizationMemberAddedEvent,
OrganizationMemberRemovedEvent,
TeamEvent,
TeamEditedEvent,
MembershipEvent,
} from '@octokit/webhooks-types';
import { merge } from 'lodash';
import * as uuid from 'uuid';
import { Logger } from 'winston';
import {
assignGroupsToUsers,
buildOrgHierarchy,
defaultOrganizationTeamTransformer,
defaultUserTransformer,
getOrganizationTeams,
getOrganizationUsers,
GithubTeam,
parseGithubOrgUrl,
} from '../lib';
import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers';
import { TeamTransformer, UserTransformer } from '../lib';
import {
createAddEntitiesOperation,
createRemoveEntitiesOperation,
createReplaceEntitiesOperation,
DeferredEntitiesBuilder,
getOrganizationTeam,
getOrganizationTeamsFromUsers,
} from '../lib/github';
/**
* Options for {@link GithubOrgEntityProvider}.
@@ -101,13 +122,14 @@ export interface GithubOrgEntityProviderOptions {
teamTransformer?: TeamTransformer;
}
// TODO: Consider supporting an (optional) webhook that reacts on org changes
/**
* Ingests org data (users and groups) from GitHub.
*
* @public
*/
export class GithubOrgEntityProvider implements EntityProvider {
export class GithubOrgEntityProvider
implements EntityProvider, EventSubscriber
{
private readonly credentialsProvider: GithubCredentialsProvider;
private connection?: EntityProviderConnection;
private scheduleFn?: () => Promise<void>;
@@ -224,6 +246,306 @@ export class GithubOrgEntityProvider implements EntityProvider {
markCommitComplete();
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
async onEvent(params: EventParams): Promise<void> {
const { logger } = this.options;
logger.debug(`Received event from ${params.topic}`);
const addEntitiesOperation = createAddEntitiesOperation(
this.options.id,
this.options.gitHubConfig.host,
);
const removeEntitiesOperation = createRemoveEntitiesOperation(
this.options.id,
this.options.gitHubConfig.host,
);
const replaceEntitiesOperation = createReplaceEntitiesOperation(
this.options.id,
this.options.gitHubConfig.host,
);
// handle change users in the org
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization
if (params.topic.includes('organization')) {
const orgEvent = params.eventPayload as OrganizationEvent;
if (
orgEvent.action === 'member_added' ||
orgEvent.action === 'member_removed'
) {
const createDeltaOperation =
orgEvent.action === 'member_added'
? addEntitiesOperation
: removeEntitiesOperation;
await this.onMemberChangeInOrganization(orgEvent, createDeltaOperation);
}
}
// handle change teams in the org
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team
if (params.topic.includes('team')) {
const teamEvent = params.eventPayload as TeamEvent;
if (teamEvent.action === 'created' || teamEvent.action === 'deleted') {
const createDeltaOperation =
teamEvent.action === 'created'
? addEntitiesOperation
: removeEntitiesOperation;
await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation);
} else if (teamEvent.action === 'edited') {
await this.onTeamEditedInOrganization(
teamEvent,
replaceEntitiesOperation,
);
}
}
// handle change membership in the org
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership
if (params.topic.includes('membership')) {
const membershipEvent = params.eventPayload as MembershipEvent;
this.onMembershipChangedInOrganization(
membershipEvent,
replaceEntitiesOperation,
);
}
return;
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
supportsEventTopics(): string[] {
return ['github.organization', 'github.team', 'github.membership'];
}
private async onTeamEditedInOrganization(
event: TeamEditedEvent,
createDeltaOperation: DeferredEntitiesBuilder,
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const teamSlug = event.team.slug;
const { headers, type: tokenType } =
await this.credentialsProvider.getCredentials({
url: this.options.orgUrl,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const { org } = parseGithubOrgUrl(this.options.orgUrl);
const { group } = await getOrganizationTeam(
client,
org,
teamSlug,
this.options.teamTransformer,
);
const { users } = await getOrganizationUsers(
client,
org,
tokenType,
this.options.userTransformer,
);
const usersFromChangedGroup = group.spec.members || [];
const usersToRebuild = users.filter(u =>
usersFromChangedGroup.includes(u.metadata.name),
);
const { groups } = await getOrganizationTeamsFromUsers(
client,
org,
usersToRebuild.map(u => u.metadata.name),
this.options.teamTransformer,
);
assignGroupsToUsers(usersToRebuild, groups);
buildOrgHierarchy(groups);
const oldName = event.changes.name?.from || '';
const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-');
const { removed } = createDeltaOperation(org, [
{
...group,
metadata: {
name: oldSlug,
},
},
]);
const { added } = createDeltaOperation(org, [...usersToRebuild, ...groups]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onMembershipChangedInOrganization(
event: MembershipEvent,
createDeltaOperation: DeferredEntitiesBuilder,
) {
if (!this.connection) {
throw new Error('Not initialized');
}
// The docs are saying I will receive the slug for the removed event,
// but the types don't reflect that,
// so I will just check to be sure the slug is there
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership
if (!('slug' in event.team)) {
return;
}
const teamSlug = event.team.slug;
const userLogin = event.member.login;
const { headers, type: tokenType } =
await this.credentialsProvider.getCredentials({
url: this.options.orgUrl,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const { org } = parseGithubOrgUrl(this.options.orgUrl);
const { group } = await getOrganizationTeam(
client,
org,
teamSlug,
this.options.teamTransformer,
);
const { users } = await getOrganizationUsers(
client,
org,
tokenType,
this.options.userTransformer,
);
const usersToRebuild = users.filter(u => u.metadata.name === userLogin);
const { groups } = await getOrganizationTeamsFromUsers(
client,
org,
[userLogin],
this.options.teamTransformer,
);
// we include group because the removed event need to update the old group too
if (!groups.some(g => g.metadata.name === group.metadata.name)) {
groups.push(group);
}
assignGroupsToUsers(usersToRebuild, groups);
buildOrgHierarchy(groups);
const { added, removed } = createDeltaOperation(org, [
...usersToRebuild,
...groups,
]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onTeamChangeInOrganization(
event: TeamEvent,
createDeltaOperation: DeferredEntitiesBuilder,
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const organizationTeamTransformer =
this.options.teamTransformer || defaultOrganizationTeamTransformer;
const { name, html_url: url, description, slug } = event.team;
const org = event.organization.login;
const { headers } = await this.credentialsProvider.getCredentials({
url: this.options.orgUrl,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const group = (await organizationTeamTransformer(
{
name,
slug,
editTeamUrl: `${url}/edit`,
combinedSlug: `${org}/${slug}`,
description: description || undefined,
parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam,
// entity will be removed
members: [],
},
{
org,
client,
query: '',
},
)) as Entity;
const { added, removed } = createDeltaOperation(org, [group]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onMemberChangeInOrganization(
event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent,
createDeltaOperation: DeferredEntitiesBuilder,
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const userTransformer =
this.options.userTransformer || defaultUserTransformer;
const { name, avatar_url: avatarUrl, email, login } = event.membership.user;
const org = event.organization.login;
const { headers } = await this.credentialsProvider.getCredentials({
url: this.options.orgUrl,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const user = (await userTransformer(
{
name,
avatarUrl,
login,
email: email || undefined,
// we don't have this information in the event, so the refresh will handle that for us
organizationVerifiedDomainEmails: [],
},
{
org,
client,
query: '',
},
)) as Entity;
const { added, removed } = createDeltaOperation(org, [user]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) {
if (!schedule || schedule === 'manual') {
return;