From 3ba87f514e224393c94cd71f9d8ed674fc3b206f Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 15 Oct 2021 11:16:21 +0200 Subject: [PATCH 1/2] Add `GitHubOrgEntityProvider` Signed-off-by: Oliver Sand --- .changeset/sharp-crabs-stare.md | 5 + plugins/catalog-backend/api-report.md | 28 +++ .../catalog-backend/src/ingestion/index.ts | 11 +- .../GithubOrgReaderProcessor.test.ts | 21 +- .../processors/GithubOrgReaderProcessor.ts | 23 +-- .../src/ingestion/processors/github/index.ts | 3 +- .../ingestion/processors/github/util.test.ts | 29 +++ .../src/ingestion/processors/github/util.ts | 25 +++ .../providers/GitHubOrgEntityProvider.test.ts | 83 ++++++++ .../providers/GitHubOrgEntityProvider.ts | 187 ++++++++++++++++++ .../src/ingestion/providers/index.ts | 17 ++ 11 files changed, 394 insertions(+), 38 deletions(-) create mode 100644 .changeset/sharp-crabs-stare.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/util.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/util.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/index.ts diff --git a/.changeset/sharp-crabs-stare.md b/.changeset/sharp-crabs-stare.md new file mode 100644 index 0000000000..a81e83686f --- /dev/null +++ b/.changeset/sharp-crabs-stare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add a `GitHubOrgEntityProvider` that can be used instead of the `GithubOrgReaderProcessor`. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6b7d966b28..fadc2993ff 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityRelationSpec } from '@backstage/catalog-model'; import express from 'express'; +import { GitHubIntegrationConfig } from '@backstage/integration'; import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; @@ -1019,6 +1020,33 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "GitHubOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class GitHubOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger_2; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + id: string; + orgUrl: string; + logger: Logger_2; + }, + ): GitHubOrgEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + read(): Promise; +} + // Warning: (ae-missing-release-tag) "GithubOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index 33c81c56cc..fcb634914b 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ -export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; export { DefaultCatalogRulesEnforcer } from './CatalogRules'; +export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; +export * from './processors'; +export * from './providers'; export type { - AnalyzeLocationRequest, - AnalyzeLocationResponse, - LocationAnalyzer, AnalyzeLocationEntityField, AnalyzeLocationExistingEntity, AnalyzeLocationGenerateEntity, + AnalyzeLocationRequest, + AnalyzeLocationResponse, + LocationAnalyzer, } from './types'; -export * from './processors'; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 8894482484..c05d3c03ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -16,24 +16,15 @@ jest.mock('@octokit/graphql'); import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - ScmIntegrations, - GithubCredentialsProvider, -} from '@backstage/integration'; -import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor'; -import { graphql } from '@octokit/graphql'; import { ConfigReader } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; describe('GithubOrgReaderProcessor', () => { - describe('parseUrl', () => { - it('only supports clean org urls, and decodes them', () => { - expect(() => parseUrl('https://github.com')).toThrow(); - expect(() => parseUrl('https://github.com/org/foo')).toThrow(); - expect(() => parseUrl('https://github.com/org/foo/teams')).toThrow(); - expect(parseUrl('https://github.com/foo%32')).toEqual({ org: 'foo2' }); - }); - }); - describe('implementation', () => { const logger = getVoidLogger(); const integrations = ScmIntegrations.fromConfig( diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 3a82ff6516..9b102601ca 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -23,7 +23,11 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationTeams, getOrganizationUsers } from './github'; +import { + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { buildOrgHierarchy } from './util/org'; @@ -61,7 +65,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { } const { client, tokenType } = await this.createClient(location.target); - const { org } = parseUrl(location.target); + const { org } = parseGitHubOrgUrl(location.target); // Read out all of the raw data const startTimestamp = Date.now(); @@ -126,18 +130,3 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return { client, tokenType }; } } - -/* - * Helpers - */ - -export function parseUrl(urlString: string): { org: string } { - const path = new URL(urlString).pathname.substr(1).split('/'); - - // /backstage - if (path.length === 1 && path[0].length) { - return { org: decodeURIComponent(path[0]) }; - } - - throw new Error(`Expected a URL pointing to /`); -} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index 7c14e21108..3d9e881ddd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -17,7 +17,8 @@ export { readGithubConfig, readGithubMultiOrgConfig } from './config'; export type { GithubMultiOrgConfig, ProviderConfig } from './config'; export { + getOrganizationRepositories, getOrganizationTeams, getOrganizationUsers, - getOrganizationRepositories, } from './github'; +export { parseGitHubOrgUrl } from './util'; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts new file mode 100644 index 0000000000..02f54e0675 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { parseGitHubOrgUrl } from './util'; + +describe('parseGitHubOrgUrl', () => { + it('only supports clean org urls, and decodes them', () => { + expect(() => parseGitHubOrgUrl('https://github.com')).toThrow(); + expect(() => parseGitHubOrgUrl('https://github.com/org/foo')).toThrow(); + expect(() => + parseGitHubOrgUrl('https://github.com/org/foo/teams'), + ).toThrow(); + expect(parseGitHubOrgUrl('https://github.com/foo%32')).toEqual({ + org: 'foo2', + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.ts b/plugins/catalog-backend/src/ingestion/processors/github/util.ts new file mode 100644 index 0000000000..d8df376038 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/util.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function parseGitHubOrgUrl(urlString: string): { org: string } { + const path = new URL(urlString).pathname.substr(1).split('/'); + + // /backstage + if (path.length === 1 && path[0].length) { + return { org: decodeURIComponent(path[0]) }; + } + + throw new Error(`Expected a URL pointing to /`); +} diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts new file mode 100644 index 0000000000..955116f457 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { withLocations } from './GitHubOrgEntityProvider'; + +describe('GitHubOrgEntityProvider', () => { + describe('withLocations', () => { + it('should set location for user', () => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + }, + spec: { + memberOf: [], + }, + }; + + expect(withLocations('https://github.com', 'backstage', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + }, + }, + spec: { + memberOf: [], + }, + }); + }); + + it('should set location for group', () => { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'mygroup', + }, + spec: { + type: 'team', + children: [], + }, + }; + + expect(withLocations('https://github.com', 'backstage', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'mygroup', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + }, + }, + spec: { + type: 'team', + children: [], + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts new file mode 100644 index 0000000000..c2b38142ea --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + GitHubIntegrationConfig, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { merge } from 'lodash'; +import { Logger } from 'winston'; +import { + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from '../processors/github'; +import { buildOrgHierarchy } from '../processors/util/org'; +import { + EntityProvider, + EntityProviderConnection, +} from '../../providers/types'; + +// TODO: Consider supporting an (optional) webhook that reacts on org changes + +export class GitHubOrgEntityProvider implements EntityProvider { + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { id: string; orgUrl: string; logger: Logger }, + ) { + const integrations = ScmIntegrations.fromConfig(config); + const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; + + if (!gitHubConfig) { + throw new Error( + `There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`, + ); + } + + const logger = options.logger.child({ + target: options.orgUrl, + }); + + return new GitHubOrgEntityProvider({ + id: options.id, + orgUrl: options.orgUrl, + logger, + gitHubConfig, + }); + } + + constructor( + private options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger; + }, + ) {} + + getProviderName() { + return `GitHubOrgEntityProvider:${this.options.id}`; + } + + async connect(connection: EntityProviderConnection) { + this.connection = connection; + } + + async read() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { markReadComplete } = trackProgress(this.options.logger); + + const credentialsProvider = GithubCredentialsProvider.create( + this.options.gitHubConfig, + ); + const { headers, type: tokenType } = + await credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGitHubOrgUrl(this.options.orgUrl); + const { users } = await getOrganizationUsers(client, org, tokenType); + const { groups, groupMemberUsers } = await getOrganizationTeams( + client, + org, + ); + // Fill out the hierarchy + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + buildOrgHierarchy(groups); + + const { markCommitComplete } = markReadComplete({ users, groups }); + + await this.connection.applyMutation({ + type: 'full', + entities: [...users, ...groups].map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + markCommitComplete(); + } +} + +// Helps wrap the timing and logging behaviors +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + let summary: string; + + logger.info('Reading GitHub users and groups'); + + function markReadComplete(read: { users: unknown[]; groups: unknown[] }) { + summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`; + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); + return { markCommitComplete }; + } + + function markCommitComplete() { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info(`Committed ${summary} in ${commitDuration} seconds.`); + } + + return { markReadComplete }; +} + +// Makes sure that emitted entities have a proper location +export function withLocations( + baseUrl: string, + org: string, + entity: Entity, +): Entity { + const location = + entity.kind === 'Group' + ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}` + : `url:${baseUrl}/${entity.metadata.name}`; + return merge( + { + metadata: { + annotations: { + [LOCATION_ANNOTATION]: location, + [ORIGIN_LOCATION_ANNOTATION]: location, + }, + }, + }, + entity, + ) as Entity; +} diff --git a/plugins/catalog-backend/src/ingestion/providers/index.ts b/plugins/catalog-backend/src/ingestion/providers/index.ts new file mode 100644 index 0000000000..cd1bc5cb36 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; From 665cfed0969a94606fe19e552e0803cb0e6a4349 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 19 Oct 2021 18:04:49 +0200 Subject: [PATCH 2/2] Add tests Signed-off-by: Oliver Sand --- .../GithubOrgReaderProcessor.test.ts | 3 +- .../processors/GithubOrgReaderProcessor.ts | 13 +- .../src/ingestion/processors/util/org.test.ts | 49 ++++-- .../src/ingestion/processors/util/org.ts | 16 ++ .../providers/GitHubOrgEntityProvider.test.ts | 152 ++++++++++++++++++ .../providers/GitHubOrgEntityProvider.ts | 33 ++-- 6 files changed, 217 insertions(+), 49 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index c05d3c03ab..87a910ee06 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -jest.mock('@octokit/graphql'); import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; @@ -24,6 +23,8 @@ import { import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +jest.mock('@octokit/graphql'); + describe('GithubOrgReaderProcessor', () => { describe('implementation', () => { const logger = getVoidLogger(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 9b102601ca..244dbd8104 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -30,7 +30,7 @@ import { } from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { buildOrgHierarchy } from './util/org'; +import { assignGroupsToUsers, buildOrgHierarchy } from './util/org'; type GraphQL = typeof graphql; @@ -82,16 +82,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`, ); - // Fill out the hierarchy - const usersByName = new Map(users.map(u => [u.metadata.name, u])); - for (const [groupName, userNames] of groupMemberUsers.entries()) { - for (const userName of userNames) { - const user = usersByName.get(userName); - if (user && !user.spec.memberOf.includes(groupName)) { - user.spec.memberOf.push(groupName); - } - } - } + assignGroupsToUsers(users, groupMemberUsers); buildOrgHierarchy(groups); // Done! diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts index 254f511c4f..8a8f47642a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts @@ -15,7 +15,16 @@ */ import { GroupEntity, UserEntity } from '@backstage/catalog-model'; -import { buildMemberOf, buildOrgHierarchy } from './org'; +import { assignGroupsToUsers, buildMemberOf, buildOrgHierarchy } from './org'; + +function u(name: string, memberOf: string[] = []): UserEntity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name }, + spec: { memberOf }, + }; +} function g( name: string, @@ -56,22 +65,33 @@ describe('buildOrgHierarchy', () => { }); }); +describe('assignGroupsToUsers', () => { + it('should assign groups to users', () => { + const users: UserEntity[] = [u('u1'), u('u2')]; + const groupMemberUsers = new Map([ + ['g1', ['u1', 'u2']], + ['g2', ['u2']], + ['g3', ['u3']], + ]); + + assignGroupsToUsers(users, groupMemberUsers); + + expect(users[0].spec.memberOf).toEqual(['g1']); + expect(users[1].spec.memberOf).toEqual(['g1', 'g2']); + }); +}); + describe('buildMemberOf', () => { it('fills indirect member of groups', () => { const a = g('a', undefined, []); const b = g('b', 'a', []); const c = g('c', 'b', []); - const u: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { name: 'n' }, - spec: { profile: {}, memberOf: ['c'] }, - }; + const user = u('n', ['c']); const groups = [a, b, c]; buildOrgHierarchy(groups); - buildMemberOf(groups, [u]); - expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + buildMemberOf(groups, [user]); + expect(user.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); }); it('takes group spec.members into account', () => { @@ -79,16 +99,11 @@ describe('buildMemberOf', () => { const b = g('b', 'a', []); const c = g('c', 'b', []); c.spec.members = ['n']; - const u: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { name: 'n' }, - spec: { profile: {}, memberOf: [] }, - }; + const user = u('n'); const groups = [a, b, c]; buildOrgHierarchy(groups); - buildMemberOf(groups, [u]); - expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + buildMemberOf(groups, [user]); + expect(user.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.ts index bc90362233..3c71f350e6 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/org.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.ts @@ -49,6 +49,22 @@ export function buildOrgHierarchy(groups: GroupEntity[]) { } } +// Ensure that users have their direct group memberships. +export function assignGroupsToUsers( + users: UserEntity[], + groupMemberUsers: Map, +) { + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } +} + // Ensure that users have their transitive group memberships. Requires that // the groups were previously processed with buildOrgHierarchy() export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) { diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts index 955116f457..75edea6909 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts @@ -13,10 +13,162 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { getVoidLogger } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GithubCredentialsProvider, + GitHubIntegrationConfig, +} from '@backstage/integration'; +import { GitHubOrgEntityProvider } from '.'; +import { EntityProviderConnection } from '../../providers'; import { withLocations } from './GitHubOrgEntityProvider'; +import { graphql } from '@octokit/graphql'; + +jest.mock('@octokit/graphql'); describe('GitHubOrgEntityProvider', () => { + describe('read', () => { + afterEach(() => jest.resetAllMocks()); + + it('should read org data and apply mutation', async () => { + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const logger = getVoidLogger(); + const gitHubConfig: GitHubIntegrationConfig = { + host: 'https://github.com', + }; + + const mockGetCredentials = jest.fn().mockReturnValue({ + headers: { token: 'blah' }, + type: 'app', + }); + + jest.spyOn(GithubCredentialsProvider, 'create').mockReturnValue({ + getCredentials: mockGetCredentials, + } as any); + + const entityProvider = new GitHubOrgEntityProvider({ + id: 'my-id', + orgUrl: 'https://github.com/backstage', + gitHubConfig, + logger, + }); + + entityProvider.connect(entityProviderConnection); + + await entityProvider.read(); + + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + }, + }, + locationKey: 'github-org-provider:my-id', + }, + ], + type: 'full', + }); + }); + }); + describe('withLocations', () => { it('should set location for user', () => { const entity: UserEntity = { diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts index c2b38142ea..ecd0d153d1 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -27,21 +27,22 @@ import { import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; import { Logger } from 'winston'; +import { + EntityProvider, + EntityProviderConnection, +} from '../../providers/types'; import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, } from '../processors/github'; -import { buildOrgHierarchy } from '../processors/util/org'; -import { - EntityProvider, - EntityProviderConnection, -} from '../../providers/types'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org'; // TODO: Consider supporting an (optional) webhook that reacts on org changes export class GitHubOrgEntityProvider implements EntityProvider { private connection?: EntityProviderConnection; + private readonly credentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, @@ -75,7 +76,11 @@ export class GitHubOrgEntityProvider implements EntityProvider { gitHubConfig: GitHubIntegrationConfig; logger: Logger; }, - ) {} + ) { + this.credentialsProvider = GithubCredentialsProvider.create( + options.gitHubConfig, + ); + } getProviderName() { return `GitHubOrgEntityProvider:${this.options.id}`; @@ -92,11 +97,8 @@ export class GitHubOrgEntityProvider implements EntityProvider { const { markReadComplete } = trackProgress(this.options.logger); - const credentialsProvider = GithubCredentialsProvider.create( - this.options.gitHubConfig, - ); const { headers, type: tokenType } = - await credentialsProvider.getCredentials({ + await this.credentialsProvider.getCredentials({ url: this.options.orgUrl, }); const client = graphql.defaults({ @@ -110,16 +112,7 @@ export class GitHubOrgEntityProvider implements EntityProvider { client, org, ); - // Fill out the hierarchy - const usersByName = new Map(users.map(u => [u.metadata.name, u])); - for (const [groupName, userNames] of groupMemberUsers.entries()) { - for (const userName of userNames) { - const user = usersByName.get(userName); - if (user && !user.spec.memberOf.includes(groupName)) { - user.spec.memberOf.push(groupName); - } - } - } + assignGroupsToUsers(users, groupMemberUsers); buildOrgHierarchy(groups); const { markCommitComplete } = markReadComplete({ users, groups });