feat(GithubMultiOrgEntityProvider): support events

Signed-off-by: Phil Kuang <pkuang@factset.com>
This commit is contained in:
Phil Kuang
2023-04-19 14:21:56 -04:00
parent 9cc08dabc9
commit 970678adbe
7 changed files with 1928 additions and 27 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Implement events support for `GithubMultiOrgEntityProvider`
+72
View File
@@ -65,6 +65,42 @@ export default async function createPlugin(
}
```
Alternatively, if you wish to ingest data from multiple GitHub organizations you can use
the `GithubMultiOrgEntityProvider` instead. Note that by default, this provider will namespace
groups according to the org they originate from to avoid potential name duplicates:
```ts title="packages/backend/src/plugins/catalog.ts"
/* highlight-add-next-line */
import { GithubMultiOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
/* highlight-add-start */
// The GitHub URL below needs to match a configured integrations.github entry
// specified in your app-config.
builder.addEntityProvider(
GithubMultiOrgEntityProvider.fromConfig(env.config, {
id: 'production',
githubUrl: 'https://github.com',
// Set the following to list the GitHub orgs you wish to ingest from. You can
// also omit this option to ingest all orgs accessible by your GitHub integration
orgs: ['org-a', 'org-b'],
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 60 },
timeout: { minutes: 15 },
}),
}),
);
/* highlight-add-end */
// ..
}
```
## Installation with Events Support
Please follow the installation instructions at
@@ -110,6 +146,42 @@ export default async function createPlugin(
}
```
Or, alternatively, if using the `GithubMultiOrgEntityProvider`:
```ts title="packages/backend/src/plugins/catalog.ts"
/* highlight-add-next-line */
import { GithubMultiOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
/* highlight-add-start */
// The GitHub URL below needs to match a configured integrations.github entry
// specified in your app-config.
builder.addEntityProvider(
GithubMultiOrgEntityProvider.fromConfig(env.config, {
id: 'production',
githubUrl: 'https://github.com',
// Set the following to list the GitHub orgs you wish to ingest from. You can
// also omit this option to ingest all orgs accessible by your GitHub integration
orgs: ['org-a', 'org-b'],
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 60 },
timeout: { minutes: 15 },
}),
// Pass in the eventBroker to allow this provider to subscribe to GitHub events
eventBroker: env.eventBroker,
}),
);
/* highlight-add-end */
// ..
}
```
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks).
The webhook will need to be configured to forward `organization`,`team` and `membership` events.
@@ -10,6 +10,7 @@ import { Config } from '@backstage/config';
import { Entity } from '@backstage/catalog-model';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
import { EventBroker } from '@backstage/plugin-events-node';
import { EventParams } from '@backstage/plugin-events-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { GithubCredentialsProvider } from '@backstage/integration';
@@ -133,7 +134,9 @@ export type GithubMultiOrgConfig = Array<{
}>;
// @public
export class GithubMultiOrgEntityProvider implements EntityProvider {
export class GithubMultiOrgEntityProvider
implements EntityProvider, EventSubscriber
{
constructor(options: {
id: string;
gitHubConfig: GithubIntegrationConfig;
@@ -153,11 +156,16 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
): GithubMultiOrgEntityProvider;
// (undocumented)
getProviderName(): string;
// (undocumented)
onEvent(params: EventParams): Promise<void>;
read(options?: { logger?: Logger }): Promise<void>;
// (undocumented)
supportsEventTopics(): string[];
}
// @public
export interface GithubMultiOrgEntityProviderOptions {
eventBroker?: EventBroker;
githubCredentialsProvider?: GithubCredentialsProvider;
githubUrl: string;
id: string;
@@ -21,6 +21,7 @@ import { graphql as graphqlMsw } from 'msw';
import { setupServer } from 'msw/node';
import { TeamTransformer, UserTransformer } from './defaultTransformers';
import {
getOrganizationsFromUser,
getOrganizationTeams,
getOrganizationUsers,
getTeamMembers,
@@ -448,6 +449,37 @@ describe('github', () => {
});
});
describe('getOrganizationsFromUser', () => {
it('reads orgs from user', async () => {
const input: QueryResponse = {
user: {
organizations: {
pageInfo: { hasNextPage: false },
nodes: [
{
login: 'a',
},
{
login: 'b',
},
{
login: 'c',
},
],
},
},
};
server.use(
graphqlMsw.query('orgs', (_req, res, ctx) => res(ctx.data(input))),
);
await expect(getOrganizationsFromUser(graphql, 'foo')).resolves.toEqual({
orgs: ['a', 'b', 'c'],
});
});
});
describe('getTeamMembers', () => {
it('reads team members', async () => {
const input: QueryResponse = {
@@ -33,6 +33,7 @@ import { DeferredEntity } from '@backstage/plugin-catalog-node';
export type QueryResponse = {
organization?: OrganizationResponse;
repositoryOwner?: RepositoryOwnerResponse;
user?: UserResponse;
};
type RepositoryOwnerResponse = {
@@ -46,11 +47,19 @@ export type OrganizationResponse = {
repositories?: Connection<RepositoryResponse>;
};
export type UserResponse = {
organizations?: Connection<GithubOrg>;
};
export type PageInfo = {
hasNextPage: boolean;
endCursor?: string;
};
export type GithubOrg = {
login: string;
};
/**
* Github User
*
@@ -334,6 +343,34 @@ export async function getOrganizationTeamsFromUsers(
return { groups };
}
export async function getOrganizationsFromUser(
client: typeof graphql,
user: string,
): Promise<{
orgs: string[];
}> {
const query = `
query orgs($user: String!) {
user(login: $user) {
organizations(first: 100) {
nodes { login }
pageInfo { hasNextPage, endCursor }
}
}
}`;
const orgs = await queryWithPaging(
client,
query,
'',
r => r.user?.organizations,
async o => o.login,
{ user },
);
return { orgs };
}
export async function getOrganizationTeam(
client: typeof graphql,
org: string,
@@ -21,7 +21,9 @@ import {
DEFAULT_NAMESPACE,
Entity,
GroupEntity,
parseEntityRef,
stringifyEntityRef,
UserEntity,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
@@ -32,10 +34,28 @@ import {
ScmIntegrations,
} from '@backstage/integration';
import {
DeferredEntity,
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-node';
import {
EventBroker,
EventParams,
EventSubscriber,
} from '@backstage/plugin-events-node';
import { graphql } from '@octokit/graphql';
import {
InstallationCreatedEvent,
InstallationEvent,
OrganizationEvent,
OrganizationMemberAddedEvent,
OrganizationMemberRemovedEvent,
MembershipEvent,
TeamCreatedEvent,
TeamDeletedEvent,
TeamEditedEvent,
TeamEvent,
} from '@octokit/webhooks-types';
import { merge } from 'lodash';
import * as uuid from 'uuid';
import { Logger } from 'winston';
@@ -44,6 +64,7 @@ import {
assignGroupsToUsers,
buildOrgHierarchy,
defaultOrganizationTeamTransformer,
defaultUserTransformer,
getOrganizationTeams,
getOrganizationUsers,
GithubTeam,
@@ -55,6 +76,11 @@ import {
ANNOTATION_GITHUB_TEAM_SLUG,
ANNOTATION_GITHUB_USER_LOGIN,
} from '../lib/annotation';
import {
getOrganizationsFromUser,
getOrganizationTeam,
getOrganizationTeamsFromUsers,
} from '../lib/github';
import { splitTeamSlug } from '../lib/util';
/**
@@ -118,14 +144,26 @@ export interface GithubMultiOrgEntityProviderOptions {
* By default groups will be namespaced according to their GitHub org.
*/
teamTransformer?: TeamTransformer;
/**
* An EventBroker to subscribe this provider to GitHub events to trigger delta mutations
*/
eventBroker?: EventBroker;
}
type CreateDeltaOperation = (entities: Entity[]) => {
added: DeferredEntity[];
removed: DeferredEntity[];
};
/**
* Ingests org data (users and groups) from GitHub.
*
* @public
*/
export class GithubMultiOrgEntityProvider implements EntityProvider {
export class GithubMultiOrgEntityProvider
implements EntityProvider, EventSubscriber
{
private connection?: EntityProviderConnection;
private scheduleFn?: () => Promise<void>;
@@ -161,6 +199,10 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
provider.schedule(options.schedule);
if (options.eventBroker) {
options.eventBroker.subscribe(provider);
}
return provider;
}
@@ -271,6 +313,478 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
markCommitComplete();
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
supportsEventTopics(): string[] {
return [
'github.installation',
'github.organization',
'github.team',
'github.membership',
];
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
async onEvent(params: EventParams): Promise<void> {
const { logger } = this.options;
logger.debug(`Received event from ${params.topic}`);
const orgs = this.options.orgs?.length
? this.options.orgs
: await this.getAllOrgs(this.options.gitHubConfig);
const eventPayload = params.eventPayload as
| InstallationEvent
| OrganizationEvent
| MembershipEvent
| TeamEvent;
if (
!orgs.includes(
(eventPayload as InstallationEvent).installation?.account?.login,
) &&
!orgs.includes(
(eventPayload as OrganizationEvent | MembershipEvent | TeamEvent)
.organization?.login,
)
) {
return;
}
// https://docs.github.com/webhooks-and-events/webhooks/webhook-events-and-payloads#installation
if (
params.topic.includes('installation') &&
eventPayload.action === 'created'
) {
// We can only respond to installation.created events to add new users/groups since a
// installation.deleted event won't provide us info on what user/groups we should remove and
// we can't query the uninstalled org since we will no longer have access. This will need to be
// eventually resolved via occasional full mutation runs by calling read()
await this.onInstallationChange(
eventPayload as InstallationCreatedEvent,
orgs,
);
}
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization
if (
params.topic.includes('organization') &&
(eventPayload.action === 'member_added' ||
eventPayload.action === 'member_removed')
) {
await this.onMemberChangeInOrganization(eventPayload, orgs);
}
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#team
if (params.topic.includes('team')) {
if (
eventPayload.action === 'created' ||
eventPayload.action === 'deleted'
) {
await this.onTeamChangeInOrganization(
eventPayload as TeamCreatedEvent | TeamDeletedEvent,
);
} else if (eventPayload.action === 'edited') {
await this.onTeamEditedInOrganization(
eventPayload as TeamEditedEvent,
orgs,
);
}
}
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership
if (params.topic.includes('membership')) {
this.onMembershipChangedInTeam(eventPayload as MembershipEvent, orgs);
}
return;
}
private async onInstallationChange(
event: InstallationCreatedEvent,
applicableOrgs: string[],
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const org = event.installation.account.login;
const { headers, type: tokenType } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${org}`,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const { users } = await getOrganizationUsers(
client,
org,
tokenType,
this.options.userTransformer,
);
const { groups } = await getOrganizationTeams(
client,
org,
this.defaultMultiOrgTeamTransformer.bind(this),
);
// Fetch group memberships of users in case they already exist and
// have memberships in groups from other applicable orgs
for (const userOrg of applicableOrgs) {
const { headers: orgHeaders } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${userOrg}`,
});
const orgClient = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers: orgHeaders,
});
const { groups: userGroups } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
users.map(
u =>
u.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] ||
u.metadata.name,
),
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers(users, userGroups);
}
const { added, removed } = this.createAddEntitiesOperation([
...users,
...groups,
]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onMemberChangeInOrganization(
event: OrganizationMemberAddedEvent | OrganizationMemberRemovedEvent,
applicableOrgs: string[],
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const userTransformer =
this.options.userTransformer || defaultUserTransformer;
const { name, avatar_url: avatarUrl, email, login } = event.membership.user;
const org = event.organization.login;
const { headers } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${org}`,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const { orgs } = await getOrganizationsFromUser(client, login);
const userApplicableOrgs = orgs.filter(o => applicableOrgs.includes(o));
let updateMemberships: boolean;
let createDeltaOperation: CreateDeltaOperation;
if (event.action === 'member_removed') {
if (userApplicableOrgs.length) {
// If the user is still associated with another applicable org then we don't want to remove
// them, just update the entity to remove any potential group memberships from the old org
createDeltaOperation = this.createAddEntitiesOperation.bind(this);
updateMemberships = true;
} else {
// User is no longer part of any applicable orgs so we can remove it,
// no need to take memberships into account
createDeltaOperation = this.createRemoveEntitiesOperation.bind(this);
updateMemberships = false;
}
} else {
// We're not sure if this user was already added as part of another applicable org
// so grab the latest memberships (potentially from teams of other orgs) to ensure
// we're not accidentally omitting them
createDeltaOperation = this.createAddEntitiesOperation.bind(this);
updateMemberships = true;
}
const user = (await userTransformer(
{
name,
avatarUrl,
login,
email: email ?? undefined,
},
{
org,
client,
query: '',
},
)) as UserEntity;
if (updateMemberships) {
for (const userOrg of userApplicableOrgs) {
const { headers: orgHeaders } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${userOrg}`,
});
const orgClient = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers: orgHeaders,
});
const { groups } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
[login],
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers([user], groups);
}
}
const { added, removed } = createDeltaOperation([user]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onTeamChangeInOrganization(
event: TeamCreatedEvent | TeamDeletedEvent,
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const org = event.organization.login;
const { headers } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${org}`,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const { name, html_url: url, description, slug } = event.team;
const group = (await this.defaultMultiOrgTeamTransformer(
{
name,
slug,
editTeamUrl: `${url}/edit`,
combinedSlug: `${org}/${slug}`,
description: description ?? undefined,
parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam,
// entity will be removed or is new
members: [],
},
{
org,
client,
query: '',
},
)) as Entity;
const createDeltaOperation =
event.action === 'created'
? this.createAddEntitiesOperation.bind(this)
: this.createRemoveEntitiesOperation.bind(this);
const { added, removed } = createDeltaOperation([group]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onTeamEditedInOrganization(
event: TeamEditedEvent,
applicableOrgs: string[],
) {
if (!this.connection) {
throw new Error('Not initialized');
}
const org = event.organization.login;
const { headers, type: tokenType } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${org}`,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const teamSlug = event.team.slug;
const { group } = await getOrganizationTeam(
client,
org,
teamSlug,
this.defaultMultiOrgTeamTransformer.bind(this),
);
const { users } = await getOrganizationUsers(
client,
org,
tokenType,
this.options.userTransformer,
);
const usersFromChangedGroup =
group.spec.members?.map(m =>
stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })),
) || [];
const usersToRebuild = users.filter(u =>
usersFromChangedGroup.includes(stringifyEntityRef(u)),
);
// Update memberships of associated members of this group in case the group entity ref changed
for (const userOrg of applicableOrgs) {
const { headers: orgHeaders } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${userOrg}`,
});
const orgClient = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers: orgHeaders,
});
const { groups } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
usersToRebuild.map(
u =>
u.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] ||
u.metadata.name,
),
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers(usersToRebuild, groups);
}
const oldName = event.changes.name?.from || '';
const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-');
const oldGroup = (await this.defaultMultiOrgTeamTransformer(
{
name: event.changes.name?.from,
slug: oldSlug,
combinedSlug: `${org}/${oldSlug}`,
description: event.changes.description?.from,
parentTeam: { slug: event.team?.parent?.slug || '' } as GithubTeam,
// entity will be removed
members: [],
},
{
org,
client,
query: '',
},
)) as Entity;
// Remove the old group entity in case the entity ref is now different
const { removed } = this.createRemoveEntitiesOperation([oldGroup]);
const { added } = this.createAddEntitiesOperation([
...usersToRebuild,
group,
]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private async onMembershipChangedInTeam(
event: MembershipEvent,
applicableOrgs: string[],
) {
if (!this.connection) {
throw new Error('Not initialized');
}
// The docs are saying I will receive the slug for the removed event,
// but the types don't reflect that,
// so I will just check to be sure the slug is there
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership
if (!('slug' in event.team)) {
return;
}
const org = event.organization.login;
const { headers } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${org}`,
});
const client = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers,
});
const teamSlug = event.team.slug;
const { group } = await getOrganizationTeam(
client,
org,
teamSlug,
this.defaultMultiOrgTeamTransformer.bind(this),
);
const userTransformer =
this.options.userTransformer || defaultUserTransformer;
const { name, avatar_url: avatarUrl, email, login } = event.member;
const user = (await userTransformer(
{
name,
avatarUrl,
login,
email: email ?? undefined,
},
{
org,
client,
query: '',
},
)) as UserEntity;
const { orgs } = await getOrganizationsFromUser(client, login);
const userApplicableOrgs = orgs.filter(o => applicableOrgs.includes(o));
for (const userOrg of userApplicableOrgs) {
const { headers: orgHeaders } =
await this.options.githubCredentialsProvider.getCredentials({
url: `${this.options.githubUrl}/${userOrg}`,
});
const orgClient = graphql.defaults({
baseUrl: this.options.gitHubConfig.apiBaseUrl,
headers: orgHeaders,
});
const { groups } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
[login],
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers([user], groups);
}
const { added, removed } = this.createAddEntitiesOperation([user, group]);
await this.connection.applyMutation({
type: 'delta',
removed,
added,
});
}
private schedule(schedule: GithubMultiOrgEntityProviderOptions['schedule']) {
if (!schedule || schedule === 'manual') {
return;
@@ -301,9 +815,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
team: GithubTeam,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> {
const result = this.options.teamTransformer
? await this.options.teamTransformer(team, ctx)
: await defaultOrganizationTeamTransformer(team, ctx);
if (this.options.teamTransformer) {
return await this.options.teamTransformer(team, ctx);
}
const result = await defaultOrganizationTeamTransformer(team, ctx);
if (result) {
result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US');
@@ -333,6 +849,32 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
)
.filter(Boolean) as string[];
}
private createAddEntitiesOperation(entities: Entity[]) {
return {
removed: [],
added: entities.map(entity => ({
locationKey: `github-multi-org-provider:${this.options.id}`,
entity: withLocations(
`https://${this.options.gitHubConfig.host}`,
entity,
),
})),
};
}
private createRemoveEntitiesOperation(entities: Entity[]) {
return {
added: [],
removed: entities.map(entity => ({
locationKey: `github-multi-org-provider:${this.options.id}`,
entity: withLocations(
`https://${this.options.gitHubConfig.host}`,
entity,
),
})),
};
}
}
// Helps wrap the timing and logging behaviors