diff --git a/.changeset/violet-schools-kneel.md b/.changeset/violet-schools-kneel.md new file mode 100644 index 0000000000..6de2c7e576 --- /dev/null +++ b/.changeset/violet-schools-kneel.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Implement `GithubMultiOrgEntityProvider` for sourcing group and user entities from multiple GitHub organizations diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 23faa31495..0698fae8a0 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -132,6 +132,42 @@ export type GithubMultiOrgConfig = Array<{ userNamespace: string | undefined; }>; +// @public +export class GithubMultiOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + gitHubConfig: GithubIntegrationConfig; + githubCredentialsProvider: GithubCredentialsProvider; + githubUrl: string; + logger: Logger; + orgs?: string[]; + userTransformer?: UserTransformer; + teamTransformer?: TeamTransformer; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: GithubMultiOrgEntityProviderOptions, + ): GithubMultiOrgEntityProvider; + // (undocumented) + getProviderName(): string; + read(options?: { logger?: Logger }): Promise; +} + +// @public +export interface GithubMultiOrgEntityProviderOptions { + githubCredentialsProvider?: GithubCredentialsProvider; + githubUrl: string; + id: string; + logger: Logger; + orgs?: string[]; + schedule?: 'manual' | TaskRunner; + teamTransformer?: TeamTransformer; + userTransformer?: UserTransformer; +} + // @public export class GithubMultiOrgReaderProcessor implements CatalogProcessor { constructor(options: { diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index 17c769419f..ee6b061034 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -26,6 +26,8 @@ export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor' export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor'; export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; export { GithubEntityProvider } from './providers/GithubEntityProvider'; +export { GithubMultiOrgEntityProvider } from './providers/GithubMultiOrgEntityProvider'; +export type { GithubMultiOrgEntityProviderOptions } from './providers/GithubMultiOrgEntityProvider'; export { GithubOrgEntityProvider } from './providers/GithubOrgEntityProvider'; export type { GithubOrgEntityProviderOptions } from './providers/GithubOrgEntityProvider'; export { diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts new file mode 100644 index 0000000000..76ab55552d --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts @@ -0,0 +1,716 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GithubCredentialsProvider, + GithubIntegrationConfig, +} from '@backstage/integration'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { graphql } from '@octokit/graphql'; +import { + GithubMultiOrgEntityProvider, + withLocations, +} from './GithubMultiOrgEntityProvider'; + +jest.mock('@octokit/graphql'); + +const getAllInstallationsMock = jest.fn(); +jest.mock('@backstage/integration', () => ({ + ...jest.requireActual('@backstage/integration'), + GithubAppCredentialsMux: function mockGithubAppCredentialsMux() { + return { + getAllInstallations: getAllInstallationsMock, + }; + }, +})); + +describe('GithubMultiOrgEntityProvider', () => { + describe('read', () => { + afterEach(() => jest.resetAllMocks()); + + it('should read specified orgs', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'x', + name: 'y', + bio: 'z', + email: 'w', + avatarUrl: 'v', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgA/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'x' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'x', + name: 'y', + bio: 'z', + email: 'w', + avatarUrl: 'v', + }, + { + login: 'q', + name: 'r', + bio: 's', + email: 't', + avatarUrl: 'u', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgB/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'q' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'https://github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + }); + + entityProvider.connect(entityProviderConnection); + + await entityProvider.read(); + + expect(mockGetCredentials).toHaveBeenCalledWith({ + url: 'https://github.com/orgA', + }); + expect(mockGetCredentials).toHaveBeenCalledWith({ + url: 'https://github.com/orgB', + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['orga/team', 'orgb/team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/x', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/x', + 'github.com/user-login': 'x', + }, + description: 'z', + name: 'x', + }, + spec: { + memberOf: ['orga/team'], + profile: { + displayName: 'y', + email: 'w', + picture: 'v', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/q', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/q', + 'github.com/user-login': 'q', + }, + description: 's', + name: 'q', + }, + spec: { + memberOf: ['orgb/team'], + profile: { + displayName: 'r', + email: 't', + picture: 'u', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/orgs/orgA/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/orgs/orgA/teams/team', + 'github.com/team-slug': 'orgA/team', + }, + namespace: 'orga', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a', 'default/x'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/orgs/orgB/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/orgs/orgB/teams/team', + 'github.com/team-slug': 'orgB/team', + }, + namespace: 'orgb', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a', 'default/q'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + type: 'full', + }); + }); + + it('should read every accessible org', async () => { + getAllInstallationsMock.mockResolvedValue([ + { + target_type: 'Organization', + account: { + login: 'orgC', + }, + }, + { + target_type: 'Organization', + account: { + login: 'orgD', + }, + }, + ]); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'x', + name: 'y', + bio: 'z', + email: 'w', + avatarUrl: 'v', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgC/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'x' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'x', + name: 'y', + bio: 'z', + email: 'w', + avatarUrl: 'v', + }, + { + login: 'q', + name: 'r', + bio: 's', + email: 't', + avatarUrl: 'u', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgD/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'q' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GithubIntegrationConfig = { + host: 'https://github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + const githubCredentialsProvider: GithubCredentialsProvider = { + getCredentials: mockGetCredentials, + }; + + const entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider, + githubUrl: 'https://github.com', + logger, + }); + + entityProvider.connect(entityProviderConnection); + + await entityProvider.read(); + + expect(mockGetCredentials).toHaveBeenCalledWith({ + url: 'https://github.com/orgC', + }); + expect(mockGetCredentials).toHaveBeenCalledWith({ + url: 'https://github.com/orgD', + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['orgc/team', 'orgd/team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/x', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/x', + 'github.com/user-login': 'x', + }, + description: 'z', + name: 'x', + }, + spec: { + memberOf: ['orgc/team'], + profile: { + displayName: 'y', + email: 'w', + picture: 'v', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/q', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/q', + 'github.com/user-login': 'q', + }, + description: 's', + name: 'q', + }, + spec: { + memberOf: ['orgd/team'], + profile: { + displayName: 'r', + email: 't', + picture: 'u', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/orgs/orgC/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/orgs/orgC/teams/team', + 'github.com/team-slug': 'orgC/team', + }, + namespace: 'orgc', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a', 'default/x'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/orgs/orgD/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/orgs/orgD/teams/team', + 'github.com/team-slug': 'orgD/team', + }, + namespace: 'orgd', + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a', 'default/q'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + type: 'full', + }); + }); + }); + + describe('withLocations', () => { + it('should set location for user', () => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'user-name', + annotations: { + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + }, + }; + + expect(withLocations('https://github.com', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'user-name', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + 'github.com/user-login': 'githubuser', + }, + }, + spec: { + memberOf: [], + }, + }); + }); + + it('should set location for group', () => { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'group-name', + annotations: { + 'github.com/team-slug': 'backstage/mygroup', + }, + }, + spec: { + type: 'team', + children: [], + }, + }; + + expect(withLocations('https://github.com', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'group-name', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + 'github.com/team-slug': 'backstage/mygroup', + }, + }, + spec: { + type: 'team', + children: [], + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts new file mode 100644 index 0000000000..ce45f30d96 --- /dev/null +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -0,0 +1,394 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + DEFAULT_NAMESPACE, + Entity, + GroupEntity, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + DefaultGithubCredentialsProvider, + GithubAppCredentialsMux, + GithubCredentialsProvider, + GithubIntegrationConfig, + ScmIntegrations, +} from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import { graphql } from '@octokit/graphql'; +import { merge } from 'lodash'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; + +import { + assignGroupsToUsers, + buildOrgHierarchy, + defaultOrganizationTeamTransformer, + getOrganizationTeams, + getOrganizationUsers, + GithubTeam, + TeamTransformer, + TransformerContext, + UserTransformer, +} from '../lib'; +import { + ANNOTATION_GITHUB_TEAM_SLUG, + ANNOTATION_GITHUB_USER_LOGIN, +} from '../lib/annotation'; +import { splitTeamSlug } from '../lib/util'; + +/** + * Options for {@link GithubMultiOrgEntityProvider}. + * + * @public + */ +export interface GithubMultiOrgEntityProviderOptions { + /** + * A unique, stable identifier for this provider. + * + * @example "production" + */ + id: string; + + /** + * The target that this provider should consume. + * + * @example "https://mycompany.github.com" + */ + githubUrl: string; + + /** + * The list of the GitHub orgs to consume. By default will consume all accessible + * orgs on the given GitHub instance (support for GitHub App integration only). + */ + orgs?: string[]; + + /** + * The refresh schedule to use. + * + * @defaultValue "manual" + * @remarks + * + * If you pass in 'manual', you are responsible for calling the `read` method + * manually at some interval. + * + * But more commonly you will pass in the result of + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. + */ + schedule?: 'manual' | TaskRunner; + + /** + * The logger to use. + */ + logger: Logger; + + /** + * Optionally supply a custom credentials provider, replacing the default one. + */ + githubCredentialsProvider?: GithubCredentialsProvider; + + /** + * Optionally include a user transformer for transforming from GitHub users to User Entities + */ + userTransformer?: UserTransformer; + + /** + * Optionally include a team transformer for transforming from GitHub teams to Group Entities. + * By default groups will be namespaced according to their GitHub org. + */ + teamTransformer?: TeamTransformer; +} + +/** + * Ingests org data (users and groups) from GitHub. + * + * @public + */ +export class GithubMultiOrgEntityProvider implements EntityProvider { + private connection?: EntityProviderConnection; + private scheduleFn?: () => Promise; + + static fromConfig( + config: Config, + options: GithubMultiOrgEntityProviderOptions, + ) { + const integrations = ScmIntegrations.fromConfig(config); + const gitHubConfig = integrations.github.byUrl(options.githubUrl)?.config; + + if (!gitHubConfig) { + throw new Error( + `There is no GitHub integration that matches ${options.githubUrl}. Please add a configuration entry for it under integrations.github.`, + ); + } + + const logger = options.logger.child({ + target: options.githubUrl, + }); + + const provider = new GithubMultiOrgEntityProvider({ + id: options.id, + gitHubConfig, + githubCredentialsProvider: + options.githubCredentialsProvider || + DefaultGithubCredentialsProvider.fromIntegrations(integrations), + githubUrl: new URL(options.githubUrl).origin, + logger, + orgs: options.orgs, + userTransformer: options.userTransformer, + teamTransformer: options.teamTransformer, + }); + + provider.schedule(options.schedule); + + return provider; + } + + constructor( + private readonly options: { + id: string; + gitHubConfig: GithubIntegrationConfig; + githubCredentialsProvider: GithubCredentialsProvider; + githubUrl: string; + logger: Logger; + orgs?: string[]; + userTransformer?: UserTransformer; + teamTransformer?: TeamTransformer; + }, + ) {} + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName() { + return `GithubMultiOrgEntityProvider:${this.options.id}`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection) { + this.connection = connection; + await this.scheduleFn?.(); + } + + /** + * Runs one single complete ingestion. This is only necessary if you use + * manual scheduling. + */ + async read(options?: { logger?: Logger }) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const logger = options?.logger ?? this.options.logger; + const { markReadComplete } = trackProgress(logger); + + const allUsersMap = new Map(); + const allGroups: Entity[] = []; + + const orgsToProcess = this.options.orgs?.length + ? this.options.orgs + : await this.getAllOrgs(this.options.gitHubConfig); + + for (const org of orgsToProcess) { + try { + const { headers, type: tokenType } = + await this.options.githubCredentialsProvider.getCredentials({ + url: `${this.options.githubUrl}/${org}`, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + logger.info(`Reading GitHub users and teams for org: ${org}`); + + const { users } = await getOrganizationUsers( + client, + org, + tokenType, + this.options.userTransformer, + ); + + const { groups } = await getOrganizationTeams( + client, + org, + this.defaultMultiOrgTeamTransformer.bind(this), + ); + + // Grab current users from `allUsersMap` if they already exist in our + // pending users so we can append to their group membership relations + const pendingUsers = users.map(u => { + const userRef = stringifyEntityRef(u); + if (!allUsersMap.has(userRef)) { + allUsersMap.set(userRef, u); + } + + return allUsersMap.get(userRef); + }); + + assignGroupsToUsers(pendingUsers, groups); + buildOrgHierarchy(groups); + + allGroups.push(...groups); + } catch (e) { + logger.error(`Failed to read GitHub org data for ${org}: ${e}`); + } + } + + const allUsers = Array.from(allUsersMap.values()); + + const { markCommitComplete } = markReadComplete({ allUsers, allGroups }); + + await this.connection.applyMutation({ + type: 'full', + entities: [...allUsers, ...allGroups].map(entity => ({ + locationKey: `github-multi-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + entity, + ), + })), + }); + + markCommitComplete(); + } + + private schedule(schedule: GithubMultiOrgEntityProviderOptions['schedule']) { + if (!schedule || schedule === 'manual') { + return; + } + + this.scheduleFn = async () => { + const id = `${this.getProviderName()}:refresh`; + await schedule.run({ + id, + fn: async () => { + const logger = this.options.logger.child({ + class: GithubMultiOrgEntityProvider.prototype.constructor.name, + taskId: id, + taskInstanceId: uuid.v4(), + }); + + try { + await this.read({ logger }); + } catch (error) { + logger.error(`${this.getProviderName()} refresh failed`, error); + } + }, + }); + }; + } + + private async defaultMultiOrgTeamTransformer( + team: GithubTeam, + ctx: TransformerContext, + ): Promise { + const result = this.options.teamTransformer + ? await this.options.teamTransformer(team, ctx) + : await defaultOrganizationTeamTransformer(team, ctx); + + if (result) { + result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US'); + // Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here + result.spec.members = team.members.map( + user => `${DEFAULT_NAMESPACE}/${user.login}`, + ); + } + + return result; + } + + // Note: Does not support usage of PATs + private async getAllOrgs( + gitHubConfig: GithubIntegrationConfig, + ): Promise { + const githubAppMux = new GithubAppCredentialsMux(gitHubConfig); + const installs = await githubAppMux.getAllInstallations(); + + return installs + .map(install => + install.target_type === 'Organization' && + install.account && + install.account.login + ? install.account.login + : undefined, + ) + .filter(Boolean) as string[]; + } +} + +// Helps wrap the timing and logging behaviors +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + let summary: string; + + logger.info('Reading GitHub users and groups'); + + function markReadComplete(read: { + allUsers: unknown[]; + allGroups: unknown[]; + }) { + summary = `${read.allUsers.length} GitHub users and ${read.allGroups.length} GitHub groups`; + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); + return { markCommitComplete }; + } + + function markCommitComplete() { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info(`Committed ${summary} in ${commitDuration} seconds.`); + } + + return { markReadComplete }; +} + +// Makes sure that emitted entities have a proper location +export function withLocations(baseUrl: string, entity: Entity): Entity { + const login = + entity.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] || + entity.metadata.name; + + let org = entity.metadata.namespace; + let team = entity.metadata.name; + const slug = entity.metadata.annotations?.[ANNOTATION_GITHUB_TEAM_SLUG]; + if (slug) { + const [slugOrg, slugTeam] = splitTeamSlug(slug); + org = slugOrg; + team = slugTeam; + } + + const location = + entity.kind === 'Group' + ? `url:${baseUrl}/orgs/${org}/teams/${team}` + : `url:${baseUrl}/${login}`; + return merge( + { + metadata: { + annotations: { + [ANNOTATION_LOCATION]: location, + [ANNOTATION_ORIGIN_LOCATION]: location, + }, + }, + }, + entity, + ) as Entity; +}