From 7810e8d998a1b27f4f71a2d22f14db9d09533c40 Mon Sep 17 00:00:00 2001 From: Benjamin Janssens Date: Thu, 8 May 2025 16:33:25 +0200 Subject: [PATCH] feat(catalog): implement discovery by GitHub app Signed-off-by: Benjamin Janssens --- ...SingleInstanceGithubCredentialsProvider.ts | 6 +- .../catalog-backend-module-github/config.d.ts | 12 ++-- .../src/providers/GithubEntityProvider.ts | 71 +++++++++++++++---- .../providers/GithubEntityProviderConfig.ts | 16 ++++- 4 files changed, 81 insertions(+), 24 deletions(-) diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index dc3a86b72b..fd3c02486f 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -201,9 +201,11 @@ class GithubAppManager { export class GithubAppCredentialsMux { private readonly apps: GithubAppManager[]; - constructor(config: GithubIntegrationConfig) { + constructor(config: GithubIntegrationConfig, appIds: number[] = []) { this.apps = - config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; + config.apps + ?.filter(app => (appIds.length ? appIds.includes(app.appId) : true)) + .map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; } async getAllInstallations(): Promise< diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index c94593b985..1ba0605de0 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -62,9 +62,13 @@ export interface Config { */ host?: string; /** - * (Required) Name of your organization account/workspace. + * (Required, unless `app` is set) Name of your organization account/workspace. */ - organization: string; + organization?: string; + /** + * (Required, unless `organization` is set) Name of your GitHub App. + */ + app?: string; /** * (Optional) Path where to look for `catalog-info.yaml` files. * You can use wildcards - `*` or `**` - to search the path and/or the filename @@ -131,9 +135,9 @@ export interface Config { */ host?: string; /** - * (Required) Name of your organization account/workspace. + * (Optional) Name of your organization account/workspace. */ - organization: string; + organization?: string; /** * (Optional) Path where to look for `catalog-info.yaml` files. * You can use wildcards - `*` or `**` - to search the path and/or the filename diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index e0837de54e..8087149e28 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -16,6 +16,7 @@ import { Config } from '@backstage/config'; import { + GithubAppCredentialsMux, GithubCredentialsProvider, GithubIntegration, GithubIntegrationConfig, @@ -82,6 +83,7 @@ type Repository = { defaultBranchRef?: string; isCatalogInfoFilePresent: boolean; visibility: string; + organization: string; }; /** @@ -219,8 +221,25 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { ); } - private async createGraphqlClient() { - const organization = this.config.organization; + private async getOrganizations(): Promise { + if (this.config.organization) return [this.config.organization]; + + const githubAppMux = new GithubAppCredentialsMux(this.integration, [ + this.config.app!, + ]); + const installs = await githubAppMux.getAllInstallations(); + return installs + .map(install => + install.target_type === 'Organization' && + install.account && + 'login' in install.account + ? install.account.login + : undefined, + ) + .filter(Boolean) as string[]; + } + + private async createGraphqlClient(organization: string) { const host = this.integration.host; const orgUrl = `https://${host}/${organization}`; @@ -236,15 +255,21 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { // go to the server and get all repositories private async findCatalogFiles(): Promise { - const organization = this.config.organization; + const organizations = await this.getOrganizations(); const catalogPath = this.config.catalogPath; - const client = await this.createGraphqlClient(); - const { repositories: repositoriesFromGithub } = - await getOrganizationRepositories(client, organization, catalogPath); - const repositories = repositoriesFromGithub.map( - this.createRepoFromGithubResponse, - ); + let repositories: Repository[] = []; + for (const organization of organizations) { + const client = await this.createGraphqlClient(organization); + + const { repositories: repositoriesFromGithub } = + await getOrganizationRepositories(client, organization, catalogPath); + repositories = repositories.concat( + repositoriesFromGithub.map(r => + this.createRepoFromGithubResponse(r, organization), + ), + ); + } if (this.config.validateLocationsExist) { return repositories.filter( @@ -322,7 +347,12 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private async onPush(event: PushEvent) { - if (this.config.organization !== event.organization?.login) { + const organizations = await this.getOrganizations(); + + if ( + !event.organization?.login || + !organizations.includes(event.organization.login) + ) { this.logger.debug( `skipping push event from organization ${event.organization?.login}`, ); @@ -405,7 +435,12 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private async onRepoChange(event: RepositoryEvent) { - if (this.config.organization !== event.organization?.login) { + const organizations = await this.getOrganizations(); + + if ( + !event.organization?.login || + !organizations.includes(event.organization.login) + ) { this.logger.debug( `skipping repository event from organization ${event.organization?.login}`, ); @@ -597,16 +632,19 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { private async addEntitiesForRepo(repository: Repository) { if (this.config.validateLocationsExist) { - const organization = this.config.organization; const catalogPath = this.config.catalogPath; - const client = await this.createGraphqlClient(); + const client = await this.createGraphqlClient(repository.organization); const repositoryFromGithub = await getOrganizationRepository( client, - organization, + repository.organization, repository.name, catalogPath, - ).then(r => (r ? this.createRepoFromGithubResponse(r) : null)); + ).then(r => + r + ? this.createRepoFromGithubResponse(r, repository.organization) + : null, + ); if (!repositoryFromGithub?.isCatalogInfoFilePresent) { return; @@ -636,11 +674,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { // only the catalog file will be recovered from the commits isCatalogInfoFilePresent: true, visibility: event.repository.visibility, + organization: event.repository.owner.login, }; } private createRepoFromGithubResponse( repositoryResponse: RepositoryResponse, + organization: string, ): Repository { return { url: repositoryResponse.url, @@ -655,6 +695,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { repositoryResponse.catalogInfoFile?.__typename === 'Blob' && repositoryResponse.catalogInfoFile.text !== '', visibility: repositoryResponse.visibility, + organization, }; } diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index 781239978f..eefbcff5d7 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -35,7 +35,8 @@ export const DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE = { export type GithubEntityProviderConfig = { id: string; catalogPath: string; - organization: string; + organization?: string; + app?: number; host: string; filters?: { repository?: RegExp; @@ -61,7 +62,7 @@ export function readProviderConfigs( return []; } - if (providersConfig.has('organization')) { + if (providersConfig.has('organization') || providersConfig.has('app')) { // simple/single config variant return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; } @@ -77,7 +78,15 @@ function readProviderConfig( id: string, config: Config, ): GithubEntityProviderConfig { - const organization = config.getString('organization'); + const organization = config.getOptionalString('organization'); + const app = config.getOptionalNumber('app'); + + if (!organization && !app) { + throw new Error( + 'Error while processing GitHub provider config. Either organization or app must be set.', + ); + } + const catalogPath = config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; const host = config.getOptionalString('host') ?? 'github.com'; @@ -120,6 +129,7 @@ function readProviderConfig( id, catalogPath, organization, + app, host, filters: { repository: repositoryPattern