diff --git a/.changeset/proud-queens-warn.md b/.changeset/proud-queens-warn.md new file mode 100644 index 0000000000..0eb133f290 --- /dev/null +++ b/.changeset/proud-queens-warn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +- Make it possible to inject custom user and team transformers when configuring the `GithubOrgEntityProvider` diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 8212c254d7..9e3c450adc 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -97,6 +97,66 @@ that must be approved first before the changes are applied.** ![email](../../assets/integrations/github/email.png) +### Custom Transformers + +You can inject your own transformation logic to help map from GH API responses +into backstage entities. You can do this on the user and team requests to +enable you to do further processing or updates to the entities. + +To enable this you pass a function into the `GitHubOrgEntityProvider`. You can +pass a `UserTransformer`, `TeamTransformer` or both. The function is invoked +for each item (user or team) that is returned from the API. You can either +return an Entity (User or Group) or `undefined` if you do not want to import +that item. + +There is also a `defaultUserTransformer` and `defaultOrganizationTeamTransformer`. +You could use these and simply decorate the response from the default +transformation if you only need to change a few properties. + +### Resolving GitHub users via organization email + +When you authenticate users you should resolve them to an entity within the +catalog. Often the authentication you use could be a corporate SSO system that +provides you with email as a key. To enable you to find and resolve GitHub users +it's useful to also import the private domain verified emails into the User +entity in backstage. + +The integration attempts to return `organizationVerifiedDomainEmails` from the +GitHub API and makes this available as part of the object passed to +`UserTransformer`. The GitHub API will only return emails that use a domain +that's a verified domain for your GitHub Org. It also relies on the user having +configured such an email in their own account. The API will only return these +values when using GitHub App authentication and with the correct app permission +allowing access to emails. + +You can decorate the `defaultUserTransformer` to replace the org email in the +returned identity. + +```typescript +async (user, ctx): Promise => { + const entity = await defaultUserTransformer(user, ctx); + + if (entity && user.organizationVerifiedDomainEmails) { + entity.spec.profile!.email = user.organizationVerifiedDomainEmails[0] || ''; + } + + return entity; +}, +``` + +Once you have imported the emails you can resolve users in your sign-in in +resolver using the catalog entity search via email + +```typescript +// packages/backend/src/plugins/auth.ts +ctx.signInWithCatalogUser({ + filter: { + kind: ['User'], + 'spec.profile.email': email as string, + }, +}); +``` + ## Using a Processor instead of a Provider An alternative to using the Provider for ingesting organizational entities is to diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index a9587e172b..ea6bc9e0ea 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -13,6 +13,8 @@ import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GithubIntegrationConfig } from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -21,6 +23,13 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend'; import { TaskRunner } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; +import { UserEntity } from '@backstage/catalog-model'; + +// @public +export const defaultOrganizationTeamTransformer: TeamTransformer; + +// @public +export const defaultUserTransformer: UserTransformer; // @public export class GithubDiscoveryProcessor implements CatalogProcessor { @@ -165,6 +174,8 @@ export class GithubOrgEntityProvider implements EntityProvider { gitHubConfig: GithubIntegrationConfig; logger: Logger; githubCredentialsProvider?: GithubCredentialsProvider; + userTransformer?: UserTransformer; + teamTransformer?: TeamTransformer; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -188,6 +199,8 @@ export interface GithubOrgEntityProviderOptions { logger: Logger; orgUrl: string; schedule?: 'manual' | TaskRunner; + teamTransformer?: TeamTransformer; + userTransformer?: UserTransformer; } // @public @@ -214,4 +227,48 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, ): Promise; } + +// @public +export type GithubTeam = { + slug: string; + combinedSlug: string; + name?: string; + description?: string; + avatarUrl?: string; + editTeamUrl?: string; + parentTeam?: GithubTeam; + members: GithubUser[]; +}; + +// @public +export type GithubUser = { + login: string; + bio?: string; + avatarUrl?: string; + email?: string; + name?: string; + organizationVerifiedDomainEmails?: string[]; +}; + +// @public +export type TeamTransformer = ( + item: GithubTeam, + ctx: TransformerContext, +) => Promise; + +// @public +export interface TransformerContext { + // (undocumented) + client: typeof graphql; + // (undocumented) + org: string; + // (undocumented) + query: string; +} + +// @public +export type UserTransformer = ( + item: GithubUser, + ctx: TransformerContext, +) => Promise; ``` diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index f38035db15..5e2e55fc24 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -15,20 +15,31 @@ */ /** - * A Backstage catalog backend module that helps integrate towards GitHub + * A Backstage catalog backend module that helps integrate towards Github * * @packageDocumentation */ export { GithubLocationAnalyzer } from './analyzers/GithubLocationAnalyzer'; export type { GithubLocationAnalyzerOptions } from './analyzers/GithubLocationAnalyzer'; -export type { GithubMultiOrgConfig } from './lib'; export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor'; export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor'; export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; export { GithubEntityProvider } from './providers/GithubEntityProvider'; -export { GithubOrgEntityProvider } from './providers/GithubOrgEntityProvider'; -export type { GithubOrgEntityProviderOptions } from './providers/GithubOrgEntityProvider'; +export type { + GithubOrgEntityProvider, + GithubOrgEntityProviderOptions, +} from './providers/GithubOrgEntityProvider'; export { githubEntityProviderCatalogModule } from './service/GithubEntityProviderCatalogModule'; +export { + type GithubMultiOrgConfig, + type GithubTeam, + type GithubUser, + type UserTransformer, + defaultUserTransformer, + type TeamTransformer, + defaultOrganizationTeamTransformer, + type TransformerContext, +} from './lib'; export * from './deprecated'; diff --git a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts new file mode 100644 index 0000000000..efd3eedd5d --- /dev/null +++ b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { graphql } from '@octokit/graphql'; +import { GithubTeam, GithubUser } from './github'; + +/** + * Context passed to Transformers + * + * @public + */ +export interface TransformerContext { + client: typeof graphql; + query: string; + org: string; +} + +/** + * Transformer for GitHub users to UserEntity + * + * @public + */ +export type UserTransformer = ( + item: GithubUser, + ctx: TransformerContext, +) => Promise; + +/** + * Transformer for GitHub Team to GroupEntity + * + * @public + */ +export type TeamTransformer = ( + item: GithubTeam, + ctx: TransformerContext, +) => Promise; + +/** + * Default transformer for GitHub users to UserEntity + * + * @public + */ +export const defaultUserTransformer: UserTransformer = async ( + item: GithubUser, +) => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: item.login, + annotations: { + 'github.com/user-login': item.login, + }, + }, + spec: { + profile: {}, + memberOf: [], + }, + }; + + if (item.bio) entity.metadata.description = item.bio; + if (item.name) entity.spec.profile!.displayName = item.name; + if (item.email) entity.spec.profile!.email = item.email; + if (item.avatarUrl) entity.spec.profile!.picture = item.avatarUrl; + return entity; +}; + +/** + * Default transformer for GitHub Team to GroupEntity + * + * @public + */ +export const defaultOrganizationTeamTransformer: TeamTransformer = + async team => { + const annotations: { [annotationName: string]: string } = { + 'github.com/team-slug': team.combinedSlug, + }; + + if (team.editTeamUrl) { + annotations['backstage.io/edit-url'] = team.editTeamUrl; + } + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: team.slug, + annotations, + }, + spec: { + type: 'team', + profile: {}, + children: [], + }, + }; + + if (team.description) { + entity.metadata.description = team.description; + } + if (team.name) { + entity.spec.profile!.displayName = team.name; + } + if (team.avatarUrl) { + entity.spec.profile!.picture = team.avatarUrl; + } + if (team.parentTeam) { + entity.spec.parent = team.parentTeam.slug; + } + + entity.spec.members = team.members.map(user => user.login); + + return entity; + }; diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 0002c4cef0..d07d2f6af8 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -15,15 +15,20 @@ */ import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { graphql } from '@octokit/graphql'; import { graphql as graphqlMsw } from 'msw'; import { setupServer } from 'msw/node'; +import { TeamTransformer, UserTransformer } from './defaultTransformers'; + import { getOrganizationTeams, getOrganizationUsers, getTeamMembers, getOrganizationRepositories, QueryResponse, + GithubUser, + GithubTeam, } from './github'; import fetch from 'node-fetch'; @@ -35,7 +40,7 @@ describe('github', () => { const server = setupServer(); setupRequestMockHandlers(server); - describe('getOrganizationUsers', () => { + describe('getOrganizationUsers using defaultUserMapper', () => { it('reads members', async () => { const input: QueryResponse = { organization: { @@ -76,7 +81,120 @@ describe('github', () => { }); }); - describe('getOrganizationTeams', () => { + describe('getOrganizationUsers using custom UserTransformer', () => { + const customUserTransformer: UserTransformer = async ( + item: GithubUser, + {}, + ) => { + if (item.login === 'aa') { + return undefined; + } + + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: `${item.login}-custom`, + annotations: { + 'github.com/user-login': item.login, + }, + }, + spec: { + profile: {}, + memberOf: [], + }, + } as UserEntity; + }; + + it('reads members', async () => { + const input: QueryResponse = { + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }; + + const output = { + users: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + name: 'a-custom', + }), + }), + ], + }; + + server.use( + graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect( + getOrganizationUsers(graphql, 'a', 'token', customUserTransformer), + ).resolves.toEqual(output); + }); + + it('reads members if undefined is returned from transformer', async () => { + const input: QueryResponse = { + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'aa', + name: 'bb', + bio: 'cc', + email: 'dd', + avatarUrl: 'ee', + }, + ], + }, + }, + }; + + const output = { + users: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + name: 'a-custom', + }), + }), + ], + }; + + server.use( + graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), + ); + + const users = await getOrganizationUsers( + graphql, + 'a', + 'token', + customUserTransformer, + ); + + expect(users.users).toHaveLength(1); + expect(users).toEqual(output); + }); + }); + + describe('getOrganizationTeams using default TeamTransformer', () => { let input: QueryResponse; beforeEach(() => { @@ -95,7 +213,7 @@ describe('github', () => { parentTeam: { slug: 'parent', combinedSlug: '', - members: { pageInfo: { hasNextPage: false }, nodes: [] }, + members: [], }, members: { pageInfo: { hasNextPage: false }, @@ -129,10 +247,10 @@ describe('github', () => { }, parent: 'parent', children: [], + members: ['user'], }, }), ], - groupMemberUsers: new Map([['team', ['user']]]), }; server.use( @@ -141,37 +259,192 @@ describe('github', () => { await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output); }); + }); - it('applies namespaces', async () => { + describe('getOrganizationTeams using custom TeamTransformer', () => { + let input: QueryResponse; + + const customTeamTransformer: TeamTransformer = async ( + item: GithubTeam, + {}, + ) => { + if (item.name === 'aa') { + return undefined; + } + + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: `${item.name}-custom`, + annotations: { + 'github.com/team-slug': 'blah/team', + 'backstage.io/edit-url': + 'http://example.com/orgs/blah/teams/team/edit', + }, + description: item.description, + }, + spec: { + type: 'team', + profile: { + displayName: `${item.name}-custom`, + picture: 'http://example.com/team.jpeg', + }, + parent: 'parent', + children: [], + members: ['user'], + }, + } as GroupEntity; + }; + + beforeEach(() => { + input = { + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: [], + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'user' }], + }, + }, + ], + }, + }, + }; + }); + + it('reads teams', async () => { const output = { groups: [ expect.objectContaining({ metadata: expect.objectContaining({ - name: 'team', - namespace: 'foo', + name: 'Team-custom', description: 'The one and only team', + annotations: { + 'github.com/team-slug': 'blah/team', + 'backstage.io/edit-url': + 'http://example.com/orgs/blah/teams/team/edit', + }, }), spec: { type: 'team', profile: { - displayName: 'Team', + displayName: 'Team-custom', picture: 'http://example.com/team.jpeg', }, parent: 'parent', children: [], + members: ['user'], }, }), ], - groupMemberUsers: new Map([['foo/team', ['user']]]), }; server.use( graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), ); - await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual( - output, + await expect( + getOrganizationTeams(graphql, 'a', customTeamTransformer), + ).resolves.toEqual(output); + }); + + it('reads teams if undefined is returned', async () => { + input = { + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: [], + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'user' }], + }, + }, + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'aa', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + editTeamUrl: 'http://example.com/orgs/blah/teams/team/edit', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: [], + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'user' }], + }, + }, + ], + }, + }, + }; + + const output = { + groups: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + name: 'Team-custom', + description: 'The one and only team', + annotations: { + 'github.com/team-slug': 'blah/team', + 'backstage.io/edit-url': + 'http://example.com/orgs/blah/teams/team/edit', + }, + }), + spec: { + type: 'team', + profile: { + displayName: 'Team-custom', + picture: 'http://example.com/team.jpeg', + }, + parent: 'parent', + children: [], + members: ['user'], + }, + }), + ], + }; + + server.use( + graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), ); + + const teams = await getOrganizationTeams( + graphql, + 'a', + customTeamTransformer, + ); + + expect(teams.groups).toHaveLength(1); + expect(teams).toEqual(output); }); }); @@ -191,7 +464,7 @@ describe('github', () => { }; const output = { - members: ['user'], + members: [{ login: 'user' }], }; server.use( diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index ef6a7f817a..4d583d0730 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -17,19 +17,30 @@ import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import { GithubCredentialType } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; +import { + defaultOrganizationTeamTransformer, + defaultUserTransformer, + TeamTransformer, + TransformerContext, + UserTransformer, +} from './defaultTransformers'; // Graphql types export type QueryResponse = { - organization?: Organization; - repositoryOwner?: Organization | User; + organization?: OrganizationResponse; + repositoryOwner?: RepositoryOwnerResponse; }; -export type Organization = { - membersWithRole?: Connection; - team?: Team; - teams?: Connection; - repositories?: Connection; +type RepositoryOwnerResponse = { + repositories?: Connection; +}; + +export type OrganizationResponse = { + membersWithRole?: Connection; + team?: GithubTeamResponse; + teams?: Connection; + repositories?: Connection; }; export type PageInfo = { @@ -37,27 +48,41 @@ export type PageInfo = { endCursor?: string; }; -export type User = { +/** + * Github User + * + * @public + */ +export type GithubUser = { login: string; bio?: string; avatarUrl?: string; email?: string; name?: string; - repositories?: Connection; + organizationVerifiedDomainEmails?: string[]; }; -export type Team = { +/** + * Github Team + * + * @public + */ +export type GithubTeam = { slug: string; combinedSlug: string; name?: string; description?: string; avatarUrl?: string; editTeamUrl?: string; - parentTeam?: Team; - members: Connection; + parentTeam?: GithubTeam; + members: GithubUser[]; }; -export type Repository = { +export type GithubTeamResponse = Omit & { + members: Connection; +}; + +export type RepositoryResponse = { name: string; url: string; isArchived: boolean; @@ -88,7 +113,7 @@ export type Connection = { }; /** - * Gets all the users out of a GitHub organization. + * Gets all the users out of a Github organization. * * Note that the users will not have their memberships filled in. * @@ -99,7 +124,7 @@ export async function getOrganizationUsers( client: typeof graphql, org: string, tokenType: GithubCredentialType, - userNamespace?: string, + userTransformer: UserTransformer = defaultUserTransformer, ): Promise<{ users: UserEntity[] }> { const query = ` query users($org: String!, $email: Boolean!, $cursor: String) { @@ -111,7 +136,8 @@ export async function getOrganizationUsers( bio, email @include(if: $email), login, - name + name, + organizationVerifiedDomainEmails(login: $org) } } } @@ -119,44 +145,24 @@ export async function getOrganizationUsers( // There is no user -> teams edge, so we leave the memberships empty for // now and let the team iteration handle it instead - const mapper = (user: User) => { - const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: user.login, - annotations: { - 'github.com/user-login': user.login, - }, - }, - spec: { - profile: {}, - memberOf: [], - }, - }; - - if (userNamespace) entity.metadata.namespace = userNamespace; - if (user.bio) entity.metadata.description = user.bio; - if (user.name) entity.spec.profile!.displayName = user.name; - if (user.email) entity.spec.profile!.email = user.email; - if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl; - - return entity; - }; const users = await queryWithPaging( client, query, + org, r => r.organization?.membersWithRole, - mapper, - { org, email: tokenType === 'token' }, + userTransformer, + { + org, + email: tokenType === 'token', + }, ); return { users }; } /** - * Gets all the teams out of a GitHub organization. + * Gets all the teams out of a Github organization. * * Note that the teams will not have any relations apart from parent filled in. * @@ -166,10 +172,9 @@ export async function getOrganizationUsers( export async function getOrganizationTeams( client: typeof graphql, org: string, - orgNamespace?: string, + teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, ): Promise<{ groups: GroupEntity[]; - groupMemberUsers: Map; }> { const query = ` query teams($org: String!, $cursor: String) { @@ -193,86 +198,51 @@ export async function getOrganizationTeams( } }`; - // Gets populated inside the mapper below - const groupMemberUsers = new Map(); + const materialisedTeams = async ( + item: GithubTeamResponse, + ctx: TransformerContext, + ): Promise => { + const memberNames: GithubUser[] = []; - const mapper = async (team: Team) => { - const annotations: { [annotationName: string]: string } = { - 'github.com/team-slug': team.combinedSlug, - }; - - if (team.editTeamUrl) { - annotations['backstage.io/edit-url'] = team.editTeamUrl; - } - - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: team.slug, - annotations, - }, - spec: { - type: 'team', - profile: {}, - children: [], - }, - }; - - if (orgNamespace) { - entity.metadata.namespace = orgNamespace; - } - - if (team.description) { - entity.metadata.description = team.description; - } - if (team.name) { - entity.spec.profile!.displayName = team.name; - } - if (team.avatarUrl) { - entity.spec.profile!.picture = team.avatarUrl; - } - if (team.parentTeam) { - entity.spec.parent = team.parentTeam.slug; - } - - const memberNames: string[] = []; - const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug; - groupMemberUsers.set(groupKey, memberNames); - - if (!team.members.pageInfo.hasNextPage) { + if (!item.members.pageInfo.hasNextPage) { // We got all the members in one go, run the fast path - for (const user of team.members.nodes) { - memberNames.push(user.login); + for (const user of item.members.nodes) { + memberNames.push(user); } } else { // There were more than a hundred immediate members - run the slow // path of fetching them explicitly - const { members } = await getTeamMembers(client, org, team.slug); + const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug); for (const userLogin of members) { memberNames.push(userLogin); } } - return entity; + const team: GithubTeam = { + ...item, + members: memberNames, + }; + + return await teamTransformer(team, ctx); }; const groups = await queryWithPaging( client, query, + org, r => r.organization?.teams, - mapper, + materialisedTeams, { org }, ); - return { groups, groupMemberUsers }; + return { groups }; } export async function getOrganizationRepositories( client: typeof graphql, org: string, catalogPath: string, -): Promise<{ repositories: Repository[] }> { +): Promise<{ repositories: RepositoryResponse[] }> { let relativeCatalogPathRef: string; // We must strip the leading slash or the query for objects does not work if (catalogPath.startsWith('/')) { @@ -321,8 +291,9 @@ export async function getOrganizationRepositories( const repositories = await queryWithPaging( client, query, + org, r => r.repositoryOwner?.repositories, - x => x, + async x => x, { org, catalogPathRef }, ); @@ -330,7 +301,7 @@ export async function getOrganizationRepositories( } /** - * Gets all the users out of a GitHub organization. + * Gets all the users out of a Github organization. * * Note that the users will not have their memberships filled in. * @@ -342,7 +313,7 @@ export async function getTeamMembers( client: typeof graphql, org: string, teamSlug: string, -): Promise<{ members: string[] }> { +): Promise<{ members: GithubUser[] }> { const query = ` query members($org: String!, $teamSlug: String!, $cursor: String) { organization(login: $org) { @@ -358,8 +329,9 @@ export async function getTeamMembers( const members = await queryWithPaging( client, query, + org, r => r.organization?.team?.members, - user => user.login, + async user => user, { org, teamSlug }, ); @@ -379,7 +351,7 @@ export async function getTeamMembers( * @param query - The query to execute * @param connection - A function that, given the response, picks out the actual * Connection object that's being iterated - * @param mapper - A function that, given one of the nodes in the Connection, + * @param transformer - A function that, given one of the nodes in the Connection, * returns the model mapped form of it * @param variables - The variable values that the query needs, minus the cursor */ @@ -391,8 +363,12 @@ export async function queryWithPaging< >( client: typeof graphql, query: string, + org: string, connection: (response: Response) => Connection | undefined, - mapper: (item: GraphqlType) => Promise | OutputType, + transformer: ( + item: GraphqlType, + ctx: TransformerContext, + ) => Promise, variables: Variables, ): Promise { const result: OutputType[] = []; @@ -410,7 +386,15 @@ export async function queryWithPaging< } for (const node of conn.nodes) { - result.push(await mapper(node)); + const transformedNode = await transformer(node, { + client, + query, + org, + }); + + if (transformedNode) { + result.push(transformedNode); + } } if (!conn.pageInfo.hasNextPage) { diff --git a/plugins/catalog-backend-module-github/src/lib/index.ts b/plugins/catalog-backend-module-github/src/lib/index.ts index 6402b69121..14f78fdeee 100644 --- a/plugins/catalog-backend-module-github/src/lib/index.ts +++ b/plugins/catalog-backend-module-github/src/lib/index.ts @@ -20,6 +20,15 @@ export { getOrganizationRepositories, getOrganizationTeams, getOrganizationUsers, + type GithubUser, + type GithubTeam, } from './github'; +export { + type UserTransformer, + defaultUserTransformer, + type TeamTransformer, + defaultOrganizationTeamTransformer, + type TransformerContext, +} from './defaultTransformers'; export { assignGroupsToUsers, buildOrgHierarchy } from './org'; export { parseGithubOrgUrl } from './util'; diff --git a/plugins/catalog-backend-module-github/src/lib/org.test.ts b/plugins/catalog-backend-module-github/src/lib/org.test.ts index 8a8f47642a..712c7c0b88 100644 --- a/plugins/catalog-backend-module-github/src/lib/org.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/org.test.ts @@ -68,13 +68,47 @@ describe('buildOrgHierarchy', () => { describe('assignGroupsToUsers', () => { it('should assign groups to users', () => { const users: UserEntity[] = [u('u1'), u('u2')]; - const groupMemberUsers = new Map([ - ['g1', ['u1', 'u2']], - ['g2', ['u2']], - ['g3', ['u3']], - ]); - assignGroupsToUsers(users, groupMemberUsers); + const groups: GroupEntity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'g1', + }, + spec: { + type: 'team', + children: [], + members: ['u1', 'u2'], + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'g2', + }, + spec: { + type: 'team', + children: [], + members: ['u2'], + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'g3', + }, + spec: { + type: 'team', + children: [], + members: ['u3'], + }, + }, + ]; + + assignGroupsToUsers(users, groups); expect(users[0].spec.memberOf).toEqual(['g1']); expect(users[1].spec.memberOf).toEqual(['g1', 'g2']); diff --git a/plugins/catalog-backend-module-github/src/lib/org.ts b/plugins/catalog-backend-module-github/src/lib/org.ts index 79a96b5e71..4642be0e8c 100644 --- a/plugins/catalog-backend-module-github/src/lib/org.ts +++ b/plugins/catalog-backend-module-github/src/lib/org.ts @@ -52,8 +52,17 @@ export function buildOrgHierarchy(groups: GroupEntity[]) { // Ensure that users have their direct group memberships. export function assignGroupsToUsers( users: UserEntity[], - groupMemberUsers: Map, + groups: GroupEntity[], ) { + const groupMemberUsers = new Map( + groups.map(group => { + const groupKey = group.metadata.namespace + ? `${group.metadata.namespace}/${group.metadata.name}` + : group.metadata.name; + return [groupKey, group.spec.members || []]; + }), + ); + const usersByName = new Map(users.map(u => [u.metadata.name, u])); for (const [groupName, userNames] of groupMemberUsers.entries()) { for (const userName of userNames) { diff --git a/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts index 7bb73990b3..59f9fdeeba 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { GroupEntity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { DefaultGithubCredentialsProvider, @@ -32,7 +33,9 @@ import { import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; import { + assignGroupsToUsers, buildOrgHierarchy, + defaultOrganizationTeamTransformer, getOrganizationTeams, getOrganizationUsers, GithubMultiOrgConfig, @@ -130,12 +133,19 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { client, orgConfig.name, tokenType, - orgConfig.userNamespace, ); - const { groups, groupMemberUsers } = await getOrganizationTeams( + const { groups } = await getOrganizationTeams( client, orgConfig.name, - orgConfig.groupNamespace, + async (team, ctx): Promise => { + const result = await defaultOrganizationTeamTransformer(team, ctx); + + if (result) { + result.metadata.namespace = orgConfig.groupNamespace; + } + + return result; + }, ); const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); @@ -151,15 +161,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { allUsersMap.set(prefix + u.metadata.name, u); } }); - - for (const [groupName, userNames] of groupMemberUsers.entries()) { - for (const userName of userNames) { - const user = allUsersMap.get(prefix + userName); - if (user && !user.spec.memberOf.includes(groupName)) { - user.spec.memberOf.push(groupName); - } - } - } + assignGroupsToUsers(users, groups); buildOrgHierarchy(groups); for (const group of groups) { diff --git a/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts index 0ef81ac907..90c168ccf4 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts @@ -101,17 +101,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { this.logger.info('Reading GitHub users and groups'); const { users } = await getOrganizationUsers(client, org, tokenType); - const { groups, groupMemberUsers } = await getOrganizationTeams( - client, - org, - ); + const { groups } = await getOrganizationTeams(client, org); const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`, ); - assignGroupsToUsers(users, groupMemberUsers); + assignGroupsToUsers(users, groups); buildOrgHierarchy(groups); // Done! diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts similarity index 90% rename from plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts rename to plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts index 2fc8ce7300..cf4adbb18d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubOrgEntityProvider.ts @@ -43,6 +43,7 @@ import { getOrganizationUsers, parseGithubOrgUrl, } from '../lib'; +import { TeamTransformer, UserTransformer } from '../lib/defaultTransformers'; /** * Options for {@link GithubOrgEntityProvider}. @@ -88,6 +89,16 @@ export interface GithubOrgEntityProviderOptions { * Optionally supply a custom credentials provider, replacing the default one. */ githubCredentialsProvider?: GithubCredentialsProvider; + + /** + * Optionally include a user transformer for transforming from GitHub users to User Entities + */ + userTransformer?: UserTransformer; + + /** + * Optionally include a user transformer for transforming from GitHub users to User Entities + */ + teamTransformer?: TeamTransformer; } // TODO: Consider supporting an (optional) webhook that reacts on org changes @@ -123,6 +134,8 @@ export class GithubOrgEntityProvider implements EntityProvider { githubCredentialsProvider: options.githubCredentialsProvider || DefaultGithubCredentialsProvider.fromIntegrations(integrations), + userTransformer: options.userTransformer, + teamTransformer: options.teamTransformer, }); provider.schedule(options.schedule); @@ -137,6 +150,8 @@ export class GithubOrgEntityProvider implements EntityProvider { gitHubConfig: GithubIntegrationConfig; logger: Logger; githubCredentialsProvider?: GithubCredentialsProvider; + userTransformer?: UserTransformer; + teamTransformer?: TeamTransformer; }, ) { this.credentialsProvider = @@ -177,12 +192,19 @@ export class GithubOrgEntityProvider implements EntityProvider { }); const { org } = parseGithubOrgUrl(this.options.orgUrl); - const { users } = await getOrganizationUsers(client, org, tokenType); - const { groups, groupMemberUsers } = await getOrganizationTeams( + const { users } = await getOrganizationUsers( client, org, + tokenType, + this.options.userTransformer, ); - assignGroupsToUsers(users, groupMemberUsers); + const { groups } = await getOrganizationTeams( + client, + org, + this.options.teamTransformer, + ); + + assignGroupsToUsers(users, groups); buildOrgHierarchy(groups); const { markCommitComplete } = markReadComplete({ users, groups }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 6d3badf865..9a75f52afa 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -37,7 +37,7 @@ import { readProviderConfigs, GithubEntityProviderConfig, } from './GithubEntityProviderConfig'; -import { getOrganizationRepositories, Repository } from '../lib/github'; +import { getOrganizationRepositories, RepositoryResponse } from '../lib/github'; import { satisfiesTopicFilter } from '../lib/util'; /** @@ -175,7 +175,7 @@ export class GithubEntityProvider implements EntityProvider { } // go to the server and get all of the repositories - private async findCatalogFiles(): Promise { + private async findCatalogFiles(): Promise { const organization = this.config.organization; const host = this.integration.host; const catalogPath = this.config.catalogPath; @@ -208,7 +208,7 @@ export class GithubEntityProvider implements EntityProvider { return repositories; } - private matchesFilters(repositories: Repository[]) { + private matchesFilters(repositories: RepositoryResponse[]) { const repositoryFilter = this.config.filters?.repository; const topicFilters = this.config.filters?.topic; @@ -226,7 +226,7 @@ export class GithubEntityProvider implements EntityProvider { return matchingRepositories; } - private createLocationUrl(repository: Repository): string { + private createLocationUrl(repository: RepositoryResponse): string { const branch = this.config.filters?.branch || repository.defaultBranchRef?.name || '-'; const catalogFile = this.config.catalogPath.startsWith('/') diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index b3fc5022ec..0b8765b93e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -163,6 +163,7 @@ describe('GithubOrgEntityProvider', () => { picture: 'http://example.com/team.jpeg', }, type: 'team', + members: ['a'], }, }, locationKey: 'github-org-provider:my-id',