feat(catalog): implement discovery by GitHub app

Signed-off-by: Benjamin Janssens <benji.janssens@gmail.com>

packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts

@@ -201,9 +201,11 @@ class GithubAppManager {
 export class GithubAppCredentialsMux {
   private readonly apps: GithubAppManager[];
 
-  constructor(config: GithubIntegrationConfig) {
+  constructor(config: GithubIntegrationConfig, appIds: number[] = []) {
     this.apps =
-      config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? [];
+      config.apps
+        ?.filter(app => (appIds.length ? appIds.includes(app.appId) : true))
+        .map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? [];
   }
 
   async getAllInstallations(): Promise<

plugins/catalog-backend-module-github/config.d.ts

@@ -62,9 +62,13 @@ export interface Config {
              */
             host?: string;
             /**
-             * (Required) Name of your organization account/workspace.
+             * (Required, unless `app` is set) Name of your organization account/workspace.
              */
-            organization: string;
+            organization?: string;
+            /**
+             * (Required, unless `organization` is set) Name of your GitHub App.
+             */
+            app?: string;
             /**
              * (Optional) Path where to look for `catalog-info.yaml` files.
              * You can use wildcards - `*` or `**` - to search the path and/or the filename
@@ -131,9 +135,9 @@ export interface Config {
                */
               host?: string;
               /**
-               * (Required) Name of your organization account/workspace.
+               * (Optional) Name of your organization account/workspace.
                */
-              organization: string;
+              organization?: string;
               /**
                * (Optional) Path where to look for `catalog-info.yaml` files.
                * You can use wildcards - `*` or `**` - to search the path and/or the filename

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts

@@ -16,6 +16,7 @@
 
 import { Config } from '@backstage/config';
 import {
+  GithubAppCredentialsMux,
   GithubCredentialsProvider,
   GithubIntegration,
   GithubIntegrationConfig,
@@ -82,6 +83,7 @@ type Repository = {
   defaultBranchRef?: string;
   isCatalogInfoFilePresent: boolean;
   visibility: string;
+  organization: string;
 };
 
 /**
@@ -219,8 +221,25 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
     );
   }
 
-  private async createGraphqlClient() {
-    const organization = this.config.organization;
+  private async getOrganizations(): Promise<string[]> {
+    if (this.config.organization) return [this.config.organization];
+
+    const githubAppMux = new GithubAppCredentialsMux(this.integration, [
+      this.config.app!,
+    ]);
+    const installs = await githubAppMux.getAllInstallations();
+    return installs
+      .map(install =>
+        install.target_type === 'Organization' &&
+        install.account &&
+        'login' in install.account
+          ? install.account.login
+          : undefined,
+      )
+      .filter(Boolean) as string[];
+  }
+
+  private async createGraphqlClient(organization: string) {
     const host = this.integration.host;
     const orgUrl = `https://${host}/${organization}`;
 
@@ -236,15 +255,21 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
 
   // go to the server and get all repositories
   private async findCatalogFiles(): Promise<Repository[]> {
-    const organization = this.config.organization;
+    const organizations = await this.getOrganizations();
     const catalogPath = this.config.catalogPath;
-    const client = await this.createGraphqlClient();
 
-    const { repositories: repositoriesFromGithub } =
-      await getOrganizationRepositories(client, organization, catalogPath);
-    const repositories = repositoriesFromGithub.map(
-      this.createRepoFromGithubResponse,
-    );
+    let repositories: Repository[] = [];
+    for (const organization of organizations) {
+      const client = await this.createGraphqlClient(organization);
+
+      const { repositories: repositoriesFromGithub } =
+        await getOrganizationRepositories(client, organization, catalogPath);
+      repositories = repositories.concat(
+        repositoriesFromGithub.map(r =>
+          this.createRepoFromGithubResponse(r, organization),
+        ),
+      );
+    }
 
     if (this.config.validateLocationsExist) {
       return repositories.filter(
@@ -322,7 +347,12 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
   }
 
   private async onPush(event: PushEvent) {
-    if (this.config.organization !== event.organization?.login) {
+    const organizations = await this.getOrganizations();
+
+    if (
+      !event.organization?.login ||
+      !organizations.includes(event.organization.login)
+    ) {
       this.logger.debug(
         `skipping push event from organization ${event.organization?.login}`,
       );
@@ -405,7 +435,12 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
   }
 
   private async onRepoChange(event: RepositoryEvent) {
-    if (this.config.organization !== event.organization?.login) {
+    const organizations = await this.getOrganizations();
+
+    if (
+      !event.organization?.login ||
+      !organizations.includes(event.organization.login)
+    ) {
       this.logger.debug(
         `skipping repository event from organization ${event.organization?.login}`,
       );
@@ -597,16 +632,19 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
 
   private async addEntitiesForRepo(repository: Repository) {
     if (this.config.validateLocationsExist) {
-      const organization = this.config.organization;
       const catalogPath = this.config.catalogPath;
-      const client = await this.createGraphqlClient();
+      const client = await this.createGraphqlClient(repository.organization);
 
       const repositoryFromGithub = await getOrganizationRepository(
         client,
-        organization,
+        repository.organization,
         repository.name,
         catalogPath,
-      ).then(r => (r ? this.createRepoFromGithubResponse(r) : null));
+      ).then(r =>
+        r
+          ? this.createRepoFromGithubResponse(r, repository.organization)
+          : null,
+      );
 
       if (!repositoryFromGithub?.isCatalogInfoFilePresent) {
         return;
@@ -636,11 +674,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
       // only the catalog file will be recovered from the commits
       isCatalogInfoFilePresent: true,
       visibility: event.repository.visibility,
+      organization: event.repository.owner.login,
     };
   }
 
   private createRepoFromGithubResponse(
     repositoryResponse: RepositoryResponse,
+    organization: string,
   ): Repository {
     return {
       url: repositoryResponse.url,
@@ -655,6 +695,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
         repositoryResponse.catalogInfoFile?.__typename === 'Blob' &&
         repositoryResponse.catalogInfoFile.text !== '',
       visibility: repositoryResponse.visibility,
+      organization,
     };
   }
 

plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts

@@ -35,7 +35,8 @@ export const DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE = {
 export type GithubEntityProviderConfig = {
   id: string;
   catalogPath: string;
-  organization: string;
+  organization?: string;
+  app?: number;
   host: string;
   filters?: {
     repository?: RegExp;
@@ -61,7 +62,7 @@ export function readProviderConfigs(
     return [];
   }
 
-  if (providersConfig.has('organization')) {
+  if (providersConfig.has('organization') || providersConfig.has('app')) {
     // simple/single config variant
     return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
   }
@@ -77,7 +78,15 @@ function readProviderConfig(
   id: string,
   config: Config,
 ): GithubEntityProviderConfig {
-  const organization = config.getString('organization');
+  const organization = config.getOptionalString('organization');
+  const app = config.getOptionalNumber('app');
+
+  if (!organization && !app) {
+    throw new Error(
+      'Error while processing GitHub provider config. Either organization or app must be set.',
+    );
+  }
+
   const catalogPath =
     config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH;
   const host = config.getOptionalString('host') ?? 'github.com';
@@ -120,6 +129,7 @@ function readProviderConfig(
     id,
     catalogPath,
     organization,
+    app,
     host,
     filters: {
       repository: repositoryPattern