From 3ba87f514e224393c94cd71f9d8ed674fc3b206f Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 15 Oct 2021 11:16:21 +0200 Subject: [PATCH] Add `GitHubOrgEntityProvider` Signed-off-by: Oliver Sand --- .changeset/sharp-crabs-stare.md | 5 + plugins/catalog-backend/api-report.md | 28 +++ .../catalog-backend/src/ingestion/index.ts | 11 +- .../GithubOrgReaderProcessor.test.ts | 21 +- .../processors/GithubOrgReaderProcessor.ts | 23 +-- .../src/ingestion/processors/github/index.ts | 3 +- .../ingestion/processors/github/util.test.ts | 29 +++ .../src/ingestion/processors/github/util.ts | 25 +++ .../providers/GitHubOrgEntityProvider.test.ts | 83 ++++++++ .../providers/GitHubOrgEntityProvider.ts | 187 ++++++++++++++++++ .../src/ingestion/providers/index.ts | 17 ++ 11 files changed, 394 insertions(+), 38 deletions(-) create mode 100644 .changeset/sharp-crabs-stare.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/util.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/util.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts create mode 100644 plugins/catalog-backend/src/ingestion/providers/index.ts diff --git a/.changeset/sharp-crabs-stare.md b/.changeset/sharp-crabs-stare.md new file mode 100644 index 0000000000..a81e83686f --- /dev/null +++ b/.changeset/sharp-crabs-stare.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add a `GitHubOrgEntityProvider` that can be used instead of the `GithubOrgReaderProcessor`. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6b7d966b28..fadc2993ff 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import { EntityRelationSpec } from '@backstage/catalog-model'; import express from 'express'; +import { GitHubIntegrationConfig } from '@backstage/integration'; import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; @@ -1019,6 +1020,33 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "GitHubOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class GitHubOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger_2; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + config: Config, + options: { + id: string; + orgUrl: string; + logger: Logger_2; + }, + ): GitHubOrgEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + read(): Promise; +} + // Warning: (ae-missing-release-tag) "GithubOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index 33c81c56cc..fcb634914b 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -14,14 +14,15 @@ * limitations under the License. */ -export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; export { DefaultCatalogRulesEnforcer } from './CatalogRules'; +export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; +export * from './processors'; +export * from './providers'; export type { - AnalyzeLocationRequest, - AnalyzeLocationResponse, - LocationAnalyzer, AnalyzeLocationEntityField, AnalyzeLocationExistingEntity, AnalyzeLocationGenerateEntity, + AnalyzeLocationRequest, + AnalyzeLocationResponse, + LocationAnalyzer, } from './types'; -export * from './processors'; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index 8894482484..c05d3c03ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -16,24 +16,15 @@ jest.mock('@octokit/graphql'); import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - ScmIntegrations, - GithubCredentialsProvider, -} from '@backstage/integration'; -import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor'; -import { graphql } from '@octokit/graphql'; import { ConfigReader } from '@backstage/config'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; describe('GithubOrgReaderProcessor', () => { - describe('parseUrl', () => { - it('only supports clean org urls, and decodes them', () => { - expect(() => parseUrl('https://github.com')).toThrow(); - expect(() => parseUrl('https://github.com/org/foo')).toThrow(); - expect(() => parseUrl('https://github.com/org/foo/teams')).toThrow(); - expect(parseUrl('https://github.com/foo%32')).toEqual({ org: 'foo2' }); - }); - }); - describe('implementation', () => { const logger = getVoidLogger(); const integrations = ScmIntegrations.fromConfig( diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 3a82ff6516..9b102601ca 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -23,7 +23,11 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationTeams, getOrganizationUsers } from './github'; +import { + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { buildOrgHierarchy } from './util/org'; @@ -61,7 +65,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { } const { client, tokenType } = await this.createClient(location.target); - const { org } = parseUrl(location.target); + const { org } = parseGitHubOrgUrl(location.target); // Read out all of the raw data const startTimestamp = Date.now(); @@ -126,18 +130,3 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { return { client, tokenType }; } } - -/* - * Helpers - */ - -export function parseUrl(urlString: string): { org: string } { - const path = new URL(urlString).pathname.substr(1).split('/'); - - // /backstage - if (path.length === 1 && path[0].length) { - return { org: decodeURIComponent(path[0]) }; - } - - throw new Error(`Expected a URL pointing to /`); -} diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts index 7c14e21108..3d9e881ddd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -17,7 +17,8 @@ export { readGithubConfig, readGithubMultiOrgConfig } from './config'; export type { GithubMultiOrgConfig, ProviderConfig } from './config'; export { + getOrganizationRepositories, getOrganizationTeams, getOrganizationUsers, - getOrganizationRepositories, } from './github'; +export { parseGitHubOrgUrl } from './util'; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts new file mode 100644 index 0000000000..02f54e0675 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { parseGitHubOrgUrl } from './util'; + +describe('parseGitHubOrgUrl', () => { + it('only supports clean org urls, and decodes them', () => { + expect(() => parseGitHubOrgUrl('https://github.com')).toThrow(); + expect(() => parseGitHubOrgUrl('https://github.com/org/foo')).toThrow(); + expect(() => + parseGitHubOrgUrl('https://github.com/org/foo/teams'), + ).toThrow(); + expect(parseGitHubOrgUrl('https://github.com/foo%32')).toEqual({ + org: 'foo2', + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.ts b/plugins/catalog-backend/src/ingestion/processors/github/util.ts new file mode 100644 index 0000000000..d8df376038 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/util.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function parseGitHubOrgUrl(urlString: string): { org: string } { + const path = new URL(urlString).pathname.substr(1).split('/'); + + // /backstage + if (path.length === 1 && path[0].length) { + return { org: decodeURIComponent(path[0]) }; + } + + throw new Error(`Expected a URL pointing to /`); +} diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts new file mode 100644 index 0000000000..955116f457 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { withLocations } from './GitHubOrgEntityProvider'; + +describe('GitHubOrgEntityProvider', () => { + describe('withLocations', () => { + it('should set location for user', () => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + }, + spec: { + memberOf: [], + }, + }; + + expect(withLocations('https://github.com', 'backstage', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: 'githubuser', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/githubuser', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/githubuser', + }, + }, + spec: { + memberOf: [], + }, + }); + }); + + it('should set location for group', () => { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'mygroup', + }, + spec: { + type: 'team', + children: [], + }, + }; + + expect(withLocations('https://github.com', 'backstage', entity)).toEqual({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'mygroup', + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/mygroup', + }, + }, + spec: { + type: 'team', + children: [], + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts new file mode 100644 index 0000000000..c2b38142ea --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts @@ -0,0 +1,187 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + GithubCredentialsProvider, + GitHubIntegrationConfig, + ScmIntegrations, +} from '@backstage/integration'; +import { graphql } from '@octokit/graphql'; +import { merge } from 'lodash'; +import { Logger } from 'winston'; +import { + getOrganizationTeams, + getOrganizationUsers, + parseGitHubOrgUrl, +} from '../processors/github'; +import { buildOrgHierarchy } from '../processors/util/org'; +import { + EntityProvider, + EntityProviderConnection, +} from '../../providers/types'; + +// TODO: Consider supporting an (optional) webhook that reacts on org changes + +export class GitHubOrgEntityProvider implements EntityProvider { + private connection?: EntityProviderConnection; + + static fromConfig( + config: Config, + options: { id: string; orgUrl: string; logger: Logger }, + ) { + const integrations = ScmIntegrations.fromConfig(config); + const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; + + if (!gitHubConfig) { + throw new Error( + `There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`, + ); + } + + const logger = options.logger.child({ + target: options.orgUrl, + }); + + return new GitHubOrgEntityProvider({ + id: options.id, + orgUrl: options.orgUrl, + logger, + gitHubConfig, + }); + } + + constructor( + private options: { + id: string; + orgUrl: string; + gitHubConfig: GitHubIntegrationConfig; + logger: Logger; + }, + ) {} + + getProviderName() { + return `GitHubOrgEntityProvider:${this.options.id}`; + } + + async connect(connection: EntityProviderConnection) { + this.connection = connection; + } + + async read() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { markReadComplete } = trackProgress(this.options.logger); + + const credentialsProvider = GithubCredentialsProvider.create( + this.options.gitHubConfig, + ); + const { headers, type: tokenType } = + await credentialsProvider.getCredentials({ + url: this.options.orgUrl, + }); + const client = graphql.defaults({ + baseUrl: this.options.gitHubConfig.apiBaseUrl, + headers, + }); + + const { org } = parseGitHubOrgUrl(this.options.orgUrl); + const { users } = await getOrganizationUsers(client, org, tokenType); + const { groups, groupMemberUsers } = await getOrganizationTeams( + client, + org, + ); + // Fill out the hierarchy + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + buildOrgHierarchy(groups); + + const { markCommitComplete } = markReadComplete({ users, groups }); + + await this.connection.applyMutation({ + type: 'full', + entities: [...users, ...groups].map(entity => ({ + locationKey: `github-org-provider:${this.options.id}`, + entity: withLocations( + `https://${this.options.gitHubConfig.host}`, + org, + entity, + ), + })), + }); + + markCommitComplete(); + } +} + +// Helps wrap the timing and logging behaviors +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + let summary: string; + + logger.info('Reading GitHub users and groups'); + + function markReadComplete(read: { users: unknown[]; groups: unknown[] }) { + summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`; + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); + return { markCommitComplete }; + } + + function markCommitComplete() { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info(`Committed ${summary} in ${commitDuration} seconds.`); + } + + return { markReadComplete }; +} + +// Makes sure that emitted entities have a proper location +export function withLocations( + baseUrl: string, + org: string, + entity: Entity, +): Entity { + const location = + entity.kind === 'Group' + ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}` + : `url:${baseUrl}/${entity.metadata.name}`; + return merge( + { + metadata: { + annotations: { + [LOCATION_ANNOTATION]: location, + [ORIGIN_LOCATION_ANNOTATION]: location, + }, + }, + }, + entity, + ) as Entity; +} diff --git a/plugins/catalog-backend/src/ingestion/providers/index.ts b/plugins/catalog-backend/src/ingestion/providers/index.ts new file mode 100644 index 0000000000..cd1bc5cb36 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';