Merge pull request #5173 from kuangp/feat/githubMultiOrgReaderProcessor

feat(GithubMultiOrgReaderProcessor): implement processor to handle multi-github org ingestion

plugins/catalog-backend/config.d.ts

@@ -145,6 +145,27 @@ export interface Config {
         }>;
       };
 
+      /**
+       * GithubMultiOrgReaderProcessor configuration
+       */
+      githubMultiOrg?: {
+        /**
+         * The configuration parameters for each GitHub org to process.
+         */
+        orgs: Array<{
+          /**
+           * The name of the GitHub org to process.
+           */
+          name: string;
+          /**
+           * The namespace of the group created for this org.
+           *
+           * Defaults to org name if omitted.
+           */
+          groupNamespace?: string;
+        }>;
+      };
+
       /**
        * AwsOrganizationCloudAccountProcessor configuration
        */

plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts

@@ -0,0 +1,181 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  GithubAppCredentialsMux,
+  GithubCredentialsProvider,
+  GitHubIntegrationConfig,
+  ScmIntegrations,
+} from '@backstage/integration';
+import { graphql } from '@octokit/graphql';
+import { Logger } from 'winston';
+import {
+  getOrganizationTeams,
+  getOrganizationUsers,
+  GithubMultiOrgConfig,
+  readGithubMultiOrgConfig,
+} from './github';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+import { buildOrgHierarchy } from './util/org';
+
+/**
+ * @alpha
+ * Extracts teams and users out of a multiple GitHub orgs namespaced per org.
+ *
+ * Be aware that this processor may not be compatible with future org structures in the catalog.
+ */
+export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrations;
+  private readonly orgs: GithubMultiOrgConfig;
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const c = config.getOptionalConfig('catalog.processors.githubMultiOrg');
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new GithubMultiOrgReaderProcessor({
+      ...options,
+      integrations,
+      orgs: c ? readGithubMultiOrgConfig(c) : [],
+    });
+  }
+
+  constructor(options: {
+    integrations: ScmIntegrations;
+    logger: Logger;
+    orgs: GithubMultiOrgConfig;
+  }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+    this.orgs = options.orgs;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'github-multi-org') {
+      return false;
+    }
+
+    const gitHubConfig = this.integrations.github.byUrl(location.target)
+      ?.config;
+    if (!gitHubConfig) {
+      throw new Error(
+        `There is no GitHub integration that matches ${location.target}. Please add a configuration entry for it under integrations.github`,
+      );
+    }
+
+    const allUsersMap = new Map();
+    const baseUrl = new URL(location.target).origin;
+    const credentialsProvider = GithubCredentialsProvider.create(gitHubConfig);
+
+    const orgsToProcess = this.orgs.length
+      ? this.orgs
+      : await this.getAllOrgs(gitHubConfig);
+
+    for (const orgConfig of orgsToProcess) {
+      try {
+        const {
+          headers,
+          type: tokenType,
+        } = await credentialsProvider.getCredentials({
+          url: `${baseUrl}/${orgConfig.name}`,
+        });
+        const client = graphql.defaults({
+          baseUrl: gitHubConfig.apiBaseUrl,
+          headers,
+        });
+
+        const startTimestamp = Date.now();
+        this.logger.info(
+          `Reading GitHub users and teams for org: ${orgConfig.name}`,
+        );
+        const { users } = await getOrganizationUsers(
+          client,
+          orgConfig.name,
+          tokenType,
+        );
+        const { groups, groupMemberUsers } = await getOrganizationTeams(
+          client,
+          orgConfig.name,
+          orgConfig.groupNamespace,
+        );
+
+        const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+        this.logger.debug(
+          `Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`,
+        );
+
+        users.forEach(u => {
+          if (!allUsersMap.has(u.metadata.name)) {
+            allUsersMap.set(u.metadata.name, u);
+          }
+        });
+
+        for (const [groupName, userNames] of groupMemberUsers.entries()) {
+          for (const userName of userNames) {
+            const user = allUsersMap.get(userName);
+            if (user && !user.spec.memberOf.includes(groupName)) {
+              user.spec.memberOf.push(groupName);
+            }
+          }
+        }
+        buildOrgHierarchy(groups);
+
+        for (const group of groups) {
+          emit(results.entity(location, group));
+        }
+      } catch (e) {
+        this.logger.error(
+          `Failed to read GitHub org data for ${orgConfig.name}: ${e}`,
+        );
+      }
+    }
+
+    const allUsers = Array.from(allUsersMap.values());
+    for (const user of allUsers) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+
+  // Note: Does not support usage of PATs
+  private async getAllOrgs(
+    gitHubConfig: GitHubIntegrationConfig,
+  ): Promise<GithubMultiOrgConfig> {
+    const githubAppMux = new GithubAppCredentialsMux(gitHubConfig);
+    const installs = await githubAppMux.getAllInstallations();
+
+    return installs
+      .map(install =>
+        install.target_type === 'Organization' &&
+        install.account &&
+        install.account.login
+          ? {
+              name: install.account.login,
+              groupNamespace: install.account.login.toLowerCase(),
+            }
+          : undefined,
+      )
+      .filter(Boolean) as GithubMultiOrgConfig;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/github/config.test.ts

@@ -15,7 +15,7 @@
  */
 
 import { ConfigReader } from '@backstage/config';
-import { readGithubConfig } from './config';
+import { readGithubConfig, readGithubMultiOrgConfig } from './config';
 
 describe('config', () => {
   describe('readGithubConfig', () => {
@@ -76,4 +76,35 @@ describe('config', () => {
       ).toThrow(/token/);
     });
   });
+
+  describe('readGithubMultiOrgConfig', () => {
+    function config(orgs: { name: string; groupNamespace?: string }[]) {
+      return new ConfigReader({ orgs });
+    }
+
+    it('reads org configs', () => {
+      const output = readGithubMultiOrgConfig(
+        config([
+          { name: 'foo', groupNamespace: 'apple' },
+          { name: 'bar', groupNamespace: 'Orange' },
+        ]),
+      );
+
+      expect(output).toEqual([
+        { name: 'foo', groupNamespace: 'apple' },
+        { name: 'bar', groupNamespace: 'orange' },
+      ]);
+    });
+
+    it('defaults groupNamespace to org name if undefined', () => {
+      const output = readGithubMultiOrgConfig(
+        config([{ name: 'foo' }, { name: 'bar' }]),
+      );
+
+      expect(output).toEqual([
+        { name: 'foo', groupNamespace: 'foo' },
+        { name: 'bar', groupNamespace: 'bar' },
+      ]);
+    });
+  });
 });

plugins/catalog-backend/src/ingestion/processors/github/config.ts

@@ -82,3 +82,27 @@ export function readGithubConfig(config: Config): ProviderConfig[] {
 
   return providers;
 }
+
+/**
+ * The configuration parameters for a multi-org GitHub processor.
+ */
+export type GithubMultiOrgConfig = Array<{
+  /**
+   * The name of the GitHub org to process.
+   */
+  name: string;
+  /**
+   * The namespace of the group created for this org.
+   */
+  groupNamespace: string;
+}>;
+
+export function readGithubMultiOrgConfig(config: Config): GithubMultiOrgConfig {
+  const orgConfigs = config.getOptionalConfigArray('orgs') ?? [];
+  return orgConfigs.map(c => ({
+    name: c.getString('name'),
+    groupNamespace: (
+      c.getOptionalString('groupNamespace') ?? c.getString('name')
+    ).toLowerCase(),
+  }));
+}

plugins/catalog-backend/src/ingestion/processors/github/github.test.ts

@@ -72,8 +72,10 @@ describe('github', () => {
   });
 
   describe('getOrganizationTeams', () => {
-    it('reads teams', async () => {
-      const input: QueryResponse = {
+    let input: QueryResponse;
+
+    beforeEach(() => {
+      input = {
         organization: {
           teams: {
             pageInfo: { hasNextPage: false },
@@ -98,7 +100,9 @@ describe('github', () => {
           },
         },
       };
+    });
 
+    it('reads teams', async () => {
       const output = {
         groups: [
           expect.objectContaining({
@@ -126,6 +130,38 @@ describe('github', () => {
 
       await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
     });
+
+    it('applies namespaces', async () => {
+      const output = {
+        groups: [
+          expect.objectContaining({
+            metadata: expect.objectContaining({
+              name: 'team',
+              namespace: 'foo',
+              description: 'The one and only team',
+            }),
+            spec: {
+              type: 'team',
+              profile: {
+                displayName: 'Team',
+                picture: 'http://example.com/team.jpeg',
+              },
+              parent: 'parent',
+              children: [],
+            },
+          }),
+        ],
+        groupMemberUsers: new Map([['foo/team', ['user']]]),
+      };
+
+      server.use(
+        graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
+      );
+
+      await expect(getOrganizationTeams(graphql, 'a', 'foo')).resolves.toEqual(
+        output,
+      );
+    });
   });
 
   describe('getTeamMembers', () => {

plugins/catalog-backend/src/ingestion/processors/github/github.ts

@@ -144,6 +144,7 @@ export async function getOrganizationUsers(
 export async function getOrganizationTeams(
   client: typeof graphql,
   org: string,
+  orgNamespace?: string,
 ): Promise<{
   groups: GroupEntity[];
   groupMemberUsers: Map<string, string[]>;
@@ -189,6 +190,10 @@ export async function getOrganizationTeams(
       },
     };
 
+    if (orgNamespace) {
+      entity.metadata.namespace = orgNamespace;
+    }
+
     if (team.description) {
       entity.metadata.description = team.description;
     }
@@ -203,7 +208,8 @@ export async function getOrganizationTeams(
     }
 
     const memberNames: string[] = [];
-    groupMemberUsers.set(team.slug, memberNames);
+    const groupKey = orgNamespace ? `${orgNamespace}/${team.slug}` : team.slug;
+    groupMemberUsers.set(groupKey, memberNames);
 
     if (!team.members.pageInfo.hasNextPage) {
       // We got all the members in one go, run the fast path

plugins/catalog-backend/src/ingestion/processors/github/index.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-export { readGithubConfig } from './config';
-export type { ProviderConfig } from './config';
+export { readGithubConfig, readGithubMultiOrgConfig } from './config';
+export type { GithubMultiOrgConfig, ProviderConfig } from './config';
 export {
   getOrganizationTeams,
   getOrganizationUsers,

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -25,6 +25,7 @@ export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
+export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
 export { LocationEntityProcessor } from './LocationEntityProcessor';
 export { PlaceholderProcessor } from './PlaceholderProcessor';
 export type { PlaceholderResolver } from './PlaceholderProcessor';