Add GitHubOrgEntityProvider

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>

.changeset/sharp-crabs-stare.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add a `GitHubOrgEntityProvider` that can be used instead of the `GithubOrgReaderProcessor`.

plugins/catalog-backend/api-report.md

@@ -16,6 +16,7 @@ import { EntityName } from '@backstage/catalog-model';
 import { EntityPolicy } from '@backstage/catalog-model';
 import { EntityRelationSpec } from '@backstage/catalog-model';
 import express from 'express';
+import { GitHubIntegrationConfig } from '@backstage/integration';
 import { IndexableDocument } from '@backstage/search-common';
 import { JsonObject } from '@backstage/config';
 import { JsonValue } from '@backstage/config';
@@ -1019,6 +1020,33 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
+// Warning: (ae-missing-release-tag) "GitHubOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class GitHubOrgEntityProvider implements EntityProvider {
+  constructor(options: {
+    id: string;
+    orgUrl: string;
+    gitHubConfig: GitHubIntegrationConfig;
+    logger: Logger_2;
+  });
+  // (undocumented)
+  connect(connection: EntityProviderConnection): Promise<void>;
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      id: string;
+      orgUrl: string;
+      logger: Logger_2;
+    },
+  ): GitHubOrgEntityProvider;
+  // (undocumented)
+  getProviderName(): string;
+  // (undocumented)
+  read(): Promise<void>;
+}
+
 // Warning: (ae-missing-release-tag) "GithubOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public

plugins/catalog-backend/src/ingestion/index.ts

@@ -14,14 +14,15 @@
  * limitations under the License.
  */
 
-export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules';
 export { DefaultCatalogRulesEnforcer } from './CatalogRules';
+export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules';
+export * from './processors';
+export * from './providers';
 export type {
-  AnalyzeLocationRequest,
-  AnalyzeLocationResponse,
-  LocationAnalyzer,
   AnalyzeLocationEntityField,
   AnalyzeLocationExistingEntity,
   AnalyzeLocationGenerateEntity,
+  AnalyzeLocationRequest,
+  AnalyzeLocationResponse,
+  LocationAnalyzer,
 } from './types';
-export * from './processors';

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts

@@ -16,24 +16,15 @@
 jest.mock('@octokit/graphql');
 import { getVoidLogger } from '@backstage/backend-common';
 import { LocationSpec } from '@backstage/catalog-model';
-import {
-  ScmIntegrations,
-  GithubCredentialsProvider,
-} from '@backstage/integration';
-import { GithubOrgReaderProcessor, parseUrl } from './GithubOrgReaderProcessor';
-import { graphql } from '@octokit/graphql';
 import { ConfigReader } from '@backstage/config';
+import {
+  GithubCredentialsProvider,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 
 describe('GithubOrgReaderProcessor', () => {
-  describe('parseUrl', () => {
-    it('only supports clean org urls, and decodes them', () => {
-      expect(() => parseUrl('https://github.com')).toThrow();
-      expect(() => parseUrl('https://github.com/org/foo')).toThrow();
-      expect(() => parseUrl('https://github.com/org/foo/teams')).toThrow();
-      expect(parseUrl('https://github.com/foo%32')).toEqual({ org: 'foo2' });
-    });
-  });
-
   describe('implementation', () => {
     const logger = getVoidLogger();
     const integrations = ScmIntegrations.fromConfig(

plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts

@@ -23,7 +23,11 @@ import {
 } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
 import { Logger } from 'winston';
-import { getOrganizationTeams, getOrganizationUsers } from './github';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  parseGitHubOrgUrl,
+} from './github';
 import * as results from './results';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
 import { buildOrgHierarchy } from './util/org';
@@ -61,7 +65,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     }
 
     const { client, tokenType } = await this.createClient(location.target);
-    const { org } = parseUrl(location.target);
+    const { org } = parseGitHubOrgUrl(location.target);
 
     // Read out all of the raw data
     const startTimestamp = Date.now();
@@ -126,18 +130,3 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
     return { client, tokenType };
   }
 }
-
-/*
- * Helpers
- */
-
-export function parseUrl(urlString: string): { org: string } {
-  const path = new URL(urlString).pathname.substr(1).split('/');
-
-  // /backstage
-  if (path.length === 1 && path[0].length) {
-    return { org: decodeURIComponent(path[0]) };
-  }
-
-  throw new Error(`Expected a URL pointing to /<org>`);
-}

plugins/catalog-backend/src/ingestion/processors/github/index.ts

@@ -17,7 +17,8 @@
 export { readGithubConfig, readGithubMultiOrgConfig } from './config';
 export type { GithubMultiOrgConfig, ProviderConfig } from './config';
 export {
+  getOrganizationRepositories,
   getOrganizationTeams,
   getOrganizationUsers,
-  getOrganizationRepositories,
 } from './github';
+export { parseGitHubOrgUrl } from './util';

plugins/catalog-backend/src/ingestion/processors/github/util.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { parseGitHubOrgUrl } from './util';
+
+describe('parseGitHubOrgUrl', () => {
+  it('only supports clean org urls, and decodes them', () => {
+    expect(() => parseGitHubOrgUrl('https://github.com')).toThrow();
+    expect(() => parseGitHubOrgUrl('https://github.com/org/foo')).toThrow();
+    expect(() =>
+      parseGitHubOrgUrl('https://github.com/org/foo/teams'),
+    ).toThrow();
+    expect(parseGitHubOrgUrl('https://github.com/foo%32')).toEqual({
+      org: 'foo2',
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/github/util.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function parseGitHubOrgUrl(urlString: string): { org: string } {
+  const path = new URL(urlString).pathname.substr(1).split('/');
+
+  // /backstage
+  if (path.length === 1 && path[0].length) {
+    return { org: decodeURIComponent(path[0]) };
+  }
+
+  throw new Error(`Expected a URL pointing to /<org>`);
+}

plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { withLocations } from './GitHubOrgEntityProvider';
+
+describe('GitHubOrgEntityProvider', () => {
+  describe('withLocations', () => {
+    it('should set location for user', () => {
+      const entity: UserEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+        },
+        spec: {
+          memberOf: [],
+        },
+      };
+
+      expect(withLocations('https://github.com', 'backstage', entity)).toEqual({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'User',
+        metadata: {
+          name: 'githubuser',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/githubuser',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/githubuser',
+          },
+        },
+        spec: {
+          memberOf: [],
+        },
+      });
+    });
+
+    it('should set location for group', () => {
+      const entity: GroupEntity = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'mygroup',
+        },
+        spec: {
+          type: 'team',
+          children: [],
+        },
+      };
+
+      expect(withLocations('https://github.com', 'backstage', entity)).toEqual({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Group',
+        metadata: {
+          name: 'mygroup',
+          annotations: {
+            'backstage.io/managed-by-location':
+              'url:https://github.com/orgs/backstage/teams/mygroup',
+            'backstage.io/managed-by-origin-location':
+              'url:https://github.com/orgs/backstage/teams/mygroup',
+          },
+        },
+        spec: {
+          type: 'team',
+          children: [],
+        },
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts

@@ -0,0 +1,187 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  Entity,
+  LOCATION_ANNOTATION,
+  ORIGIN_LOCATION_ANNOTATION,
+} from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  GithubCredentialsProvider,
+  GitHubIntegrationConfig,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { merge } from 'lodash';
+import { Logger } from 'winston';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  parseGitHubOrgUrl,
+} from '../processors/github';
+import { buildOrgHierarchy } from '../processors/util/org';
+import {
+  EntityProvider,
+  EntityProviderConnection,
+} from '../../providers/types';
+
+// TODO: Consider supporting an (optional) webhook that reacts on org changes
+
+export class GitHubOrgEntityProvider implements EntityProvider {
+  private connection?: EntityProviderConnection;
+
+  static fromConfig(
+    config: Config,
+    options: { id: string; orgUrl: string; logger: Logger },
+  ) {
+    const integrations = ScmIntegrations.fromConfig(config);
+    const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config;
+
+    if (!gitHubConfig) {
+      throw new Error(
+        `There is no GitHub Org provider that matches ${options.orgUrl}. Please add a configuration for an integration.`,
+      );
+    }
+
+    const logger = options.logger.child({
+      target: options.orgUrl,
+    });
+
+    return new GitHubOrgEntityProvider({
+      id: options.id,
+      orgUrl: options.orgUrl,
+      logger,
+      gitHubConfig,
+    });
+  }
+
+  constructor(
+    private options: {
+      id: string;
+      orgUrl: string;
+      gitHubConfig: GitHubIntegrationConfig;
+      logger: Logger;
+    },
+  ) {}
+
+  getProviderName() {
+    return `GitHubOrgEntityProvider:${this.options.id}`;
+  }
+
+  async connect(connection: EntityProviderConnection) {
+    this.connection = connection;
+  }
+
+  async read() {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const { markReadComplete } = trackProgress(this.options.logger);
+
+    const credentialsProvider = GithubCredentialsProvider.create(
+      this.options.gitHubConfig,
+    );
+    const { headers, type: tokenType } =
+      await credentialsProvider.getCredentials({
+        url: this.options.orgUrl,
+      });
+    const client = graphql.defaults({
+      baseUrl: this.options.gitHubConfig.apiBaseUrl,
+      headers,
+    });
+
+    const { org } = parseGitHubOrgUrl(this.options.orgUrl);
+    const { users } = await getOrganizationUsers(client, org, tokenType);
+    const { groups, groupMemberUsers } = await getOrganizationTeams(
+      client,
+      org,
+    );
+    // Fill out the hierarchy
+    const usersByName = new Map(users.map(u => [u.metadata.name, u]));
+    for (const [groupName, userNames] of groupMemberUsers.entries()) {
+      for (const userName of userNames) {
+        const user = usersByName.get(userName);
+        if (user && !user.spec.memberOf.includes(groupName)) {
+          user.spec.memberOf.push(groupName);
+        }
+      }
+    }
+    buildOrgHierarchy(groups);
+
+    const { markCommitComplete } = markReadComplete({ users, groups });
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: [...users, ...groups].map(entity => ({
+        locationKey: `github-org-provider:${this.options.id}`,
+        entity: withLocations(
+          `https://${this.options.gitHubConfig.host}`,
+          org,
+          entity,
+        ),
+      })),
+    });
+
+    markCommitComplete();
+  }
+}
+
+// Helps wrap the timing and logging behaviors
+function trackProgress(logger: Logger) {
+  let timestamp = Date.now();
+  let summary: string;
+
+  logger.info('Reading GitHub users and groups');
+
+  function markReadComplete(read: { users: unknown[]; groups: unknown[] }) {
+    summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`;
+    const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
+    timestamp = Date.now();
+    logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);
+    return { markCommitComplete };
+  }
+
+  function markCommitComplete() {
+    const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
+    logger.info(`Committed ${summary} in ${commitDuration} seconds.`);
+  }
+
+  return { markReadComplete };
+}
+
+// Makes sure that emitted entities have a proper location
+export function withLocations(
+  baseUrl: string,
+  org: string,
+  entity: Entity,
+): Entity {
+  const location =
+    entity.kind === 'Group'
+      ? `url:${baseUrl}/orgs/${org}/teams/${entity.metadata.name}`
+      : `url:${baseUrl}/${entity.metadata.name}`;
+  return merge(
+    {
+      metadata: {
+        annotations: {
+          [LOCATION_ANNOTATION]: location,
+          [ORIGIN_LOCATION_ANNOTATION]: location,
+        },
+      },
+    },
+    entity,
+  ) as Entity;
+}

plugins/catalog-backend/src/ingestion/providers/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';