diff --git a/.changeset/legal-lemons-attend.md b/.changeset/legal-lemons-attend.md new file mode 100644 index 0000000000..4b497d8620 --- /dev/null +++ b/.changeset/legal-lemons-attend.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': minor +--- + +Added support for limiting GithubAppCredentialsMux to specific apps diff --git a/.changeset/sharp-carrots-spend.md b/.changeset/sharp-carrots-spend.md new file mode 100644 index 0000000000..fc846ce13f --- /dev/null +++ b/.changeset/sharp-carrots-spend.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': minor +--- + +Added support for discovery by app diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index c56aed054b..4a40b25361 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -2,7 +2,7 @@ id: discovery title: GitHub Discovery sidebar_label: Discovery -description: Automatically discovering catalog entities from repositories in a GitHub organization +description: Automatically discovering catalog entities from repositories in a GitHub organization or App --- :::info @@ -12,8 +12,8 @@ This documentation is written for [the new backend system](../../backend-system/ ## GitHub Provider The GitHub integration has a discovery provider for discovering catalog -entities within a GitHub organization. The provider will crawl the GitHub -organization and register entities matching the configured path. This can be +entities within a GitHub organization or App. The provider will crawl the GitHub +organization or App and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. This is the preferred method for ingesting entities into the catalog. @@ -252,7 +252,7 @@ catalog: catalogPath: '/catalog-info.yaml' # string ``` -This provider supports multiple organizations via unique provider IDs. +This provider supports multiple organizations and apps via unique provider IDs. :::note Note @@ -288,9 +288,11 @@ If you do so, `default` will be used as provider ID. Whether to include archived repositories. Defaults to `false`. - **`host`** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). -- **`organization`**: +- **`organization`** _(required, unless `app` is set)_: Name of your organization account/workspace. - If you want to add multiple organizations, you need to add one provider config each. + If you want to add multiple organizations, you need to add one provider config each or specify `app` instead. +- **`app`** _(required, unless `organization` is set)_: + ID of your GitHub App. - **`validateLocationsExist`** _(optional)_: Whether to validate locations that exist before emitting them. This option avoids generating locations for catalog info files that do not exist in the source repository. diff --git a/packages/integration/report.api.md b/packages/integration/report.api.md index f07249bc13..f642058ccd 100644 --- a/packages/integration/report.api.md +++ b/packages/integration/report.api.md @@ -669,7 +669,7 @@ export type GithubAppConfig = { // @public export class GithubAppCredentialsMux { - constructor(config: GithubIntegrationConfig); + constructor(config: GithubIntegrationConfig, appIds?: number[]); // (undocumented) getAllInstallations(): Promise< RestEndpointMethodTypes['apps']['listInstallations']['response']['data'] diff --git a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts index dc3a86b72b..fd3c02486f 100644 --- a/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts +++ b/packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts @@ -201,9 +201,11 @@ class GithubAppManager { export class GithubAppCredentialsMux { private readonly apps: GithubAppManager[]; - constructor(config: GithubIntegrationConfig) { + constructor(config: GithubIntegrationConfig, appIds: number[] = []) { this.apps = - config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; + config.apps + ?.filter(app => (appIds.length ? appIds.includes(app.appId) : true)) + .map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? []; } async getAllInstallations(): Promise< diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index f7c8781c30..1865df7753 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -62,9 +62,13 @@ export interface Config { */ host?: string; /** - * (Required) Name of your organization account/workspace. + * (Required, unless `app` is set) Name of your organization account/workspace. */ - organization: string; + organization?: string; + /** + * (Required, unless `organization` is set) ID of your GitHub App. + */ + app?: number; /** * (Optional) Path where to look for `catalog-info.yaml` files. * You can use wildcards - `*` or `**` - to search the path and/or the filename @@ -136,9 +140,13 @@ export interface Config { */ host?: string; /** - * (Required) Name of your organization account/workspace. + * (Required, unless `app` is set) Name of your organization account/workspace. */ - organization: string; + organization?: string; + /** + * (Required, unless `organization` is set) ID of your GitHub App. + */ + app?: number; /** * (Optional) Path where to look for `catalog-info.yaml` files. * You can use wildcards - `*` or `**` - to search the path and/or the filename diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index e87650a5fb..78c83b8b87 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -74,7 +74,8 @@ "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "luxon": "^3.0.0", - "msw": "^2.0.0" + "msw": "^2.0.0", + "type-fest": "^4.41.0" }, "configSchema": "config.d.ts" } diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index b9f69b5754..19042c0cd9 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -35,6 +35,7 @@ import { RepositoryEvent, RepositoryRenamedEvent, } from '@octokit/webhooks-types'; +import type { PartialDeep } from 'type-fest'; jest.mock('../lib/github', () => { return { @@ -673,7 +674,8 @@ describe('GithubEntityProvider', () => { topics: [], html_url: `https://github.com/${organization}/test-repo`, url: `https://github.com/${organization}/test-repo`, - } as Partial; + owner: { login: 'test-org' }, + } as PartialDeep; const catalogCommit = { added: [] as string[], @@ -1003,7 +1005,10 @@ describe('GithubEntityProvider', () => { topics: [], archived: action === 'archived', private: action !== 'publicized', - } as Partial; + owner: { + login: 'test-org', + }, + } as PartialDeep; const event = { action, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index dd8c16ef34..40ccf2aaaa 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -16,6 +16,7 @@ import { Config } from '@backstage/config'; import { + GithubAppCredentialsMux, GithubCredentialsProvider, GithubIntegration, GithubIntegrationConfig, @@ -82,6 +83,7 @@ type Repository = { defaultBranchRef?: string; isCatalogInfoFilePresent: boolean; visibility: string; + organization: string; }; /** @@ -219,8 +221,25 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { ); } - private async createGraphqlClient() { - const organization = this.config.organization; + private async getOrganizations(): Promise { + if (this.config.organization) return [this.config.organization]; + + const githubAppMux = new GithubAppCredentialsMux(this.integration, [ + this.config.app!, + ]); + const installs = await githubAppMux.getAllInstallations(); + return installs + .map(install => + install.target_type === 'Organization' && + install.account && + 'login' in install.account + ? install.account.login + : undefined, + ) + .filter(Boolean) as string[]; + } + + private async createGraphqlClient(organization: string) { const host = this.integration.host; const orgUrl = `https://${host}/${organization}`; @@ -236,15 +255,21 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { // go to the server and get all repositories private async findCatalogFiles(): Promise { - const organization = this.config.organization; + const organizations = await this.getOrganizations(); const catalogPath = this.config.catalogPath; - const client = await this.createGraphqlClient(); - const { repositories: repositoriesFromGithub } = - await getOrganizationRepositories(client, organization, catalogPath); - const repositories = repositoriesFromGithub.map( - this.createRepoFromGithubResponse, - ); + let repositories: Repository[] = []; + for (const organization of organizations) { + const client = await this.createGraphqlClient(organization); + + const { repositories: repositoriesFromGithub } = + await getOrganizationRepositories(client, organization, catalogPath); + repositories = repositories.concat( + repositoriesFromGithub.map(r => + this.createRepoFromGithubResponse(r, organization), + ), + ); + } if (this.config.validateLocationsExist) { return repositories.filter( @@ -323,12 +348,15 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private async onPush(event: PushEvent) { - const configOrganization = this.config.organization; - const eventOrganization = event.organization?.login; + const configOrganizations = (await this.getOrganizations()).map(org => + org.toLocaleLowerCase('en-US'), + ); + const eventOrganization = + event.organization?.login.toLocaleLowerCase('en-US'); if ( - configOrganization.toLocaleLowerCase('en-US') !== - eventOrganization?.toLocaleLowerCase('en-US') + !eventOrganization || + !configOrganizations.includes(eventOrganization) ) { this.logger.debug( `skipping push event from organization ${event.organization?.login}`, @@ -412,12 +440,15 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } private async onRepoChange(event: RepositoryEvent) { - const configOrganization = this.config.organization; - const eventOrganization = event.organization?.login; + const configOrganizations = (await this.getOrganizations()).map(org => + org.toLocaleLowerCase('en-US'), + ); + const eventOrganization = + event.organization?.login.toLocaleLowerCase('en-US'); if ( - configOrganization.toLocaleLowerCase('en-US') !== - eventOrganization?.toLocaleLowerCase('en-US') + !eventOrganization || + !configOrganizations.includes(eventOrganization) ) { this.logger.debug( `skipping repository event from organization ${event.organization?.login}`, @@ -612,16 +643,18 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { private async addEntitiesForRepo(repository: Repository) { if (this.config.validateLocationsExist) { - const organization = this.config.organization; + const organization = repository.organization; const catalogPath = this.config.catalogPath; - const client = await this.createGraphqlClient(); + const client = await this.createGraphqlClient(organization); const repositoryFromGithub = await getOrganizationRepository( client, organization, repository.name, catalogPath, - ).then(r => (r ? this.createRepoFromGithubResponse(r) : null)); + ).then(r => + r ? this.createRepoFromGithubResponse(r, organization) : null, + ); if (!repositoryFromGithub?.isCatalogInfoFilePresent) { return; @@ -651,11 +684,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { // only the catalog file will be recovered from the commits isCatalogInfoFilePresent: true, visibility: event.repository.visibility, + organization: event.repository.owner.login, }; } private createRepoFromGithubResponse( repositoryResponse: RepositoryResponse, + organization: string, ): Repository { return { url: repositoryResponse.url, @@ -670,6 +705,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { repositoryResponse.catalogInfoFile?.__typename === 'Blob' && repositoryResponse.catalogInfoFile.text !== '', visibility: repositoryResponse.visibility, + organization, }; } diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index 28a3f5e7dd..3b1743e404 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -111,13 +111,20 @@ describe('readProviderConfigs', () => { }, }, }, + providerAppOnly: { + app: '1234', + }, + providerAppAndOrganization: { + app: '1234', + organization: 'test-org1', + }, }, }, }, }); const providerConfigs = readProviderConfigs(config); - expect(providerConfigs).toHaveLength(10); + expect(providerConfigs).toHaveLength(12); expect(providerConfigs[0]).toEqual({ id: 'providerOrganizationOnly', organization: 'test-org1', @@ -313,6 +320,45 @@ describe('readProviderConfigs', () => { }, validateLocationsExist: false, }); + expect(providerConfigs[10]).toEqual({ + id: 'providerAppOnly', + app: 1234, + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + allowForks: true, + topic: { + include: undefined, + exclude: undefined, + }, + visibility: undefined, + allowArchived: false, + }, + schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, + validateLocationsExist: false, + }); + expect(providerConfigs[11]).toEqual({ + id: 'providerAppAndOrganization', + app: 1234, + organization: 'test-org1', + catalogPath: '/catalog-info.yaml', + host: 'github.com', + filters: { + repository: undefined, + branch: undefined, + allowForks: true, + topic: { + include: undefined, + exclude: undefined, + }, + visibility: undefined, + allowArchived: false, + }, + schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE, + validateLocationsExist: false, + }); }); it('defaults validateLocationsExist to false', () => { @@ -365,4 +411,18 @@ describe('readProviderConfigs', () => { expect(() => readProviderConfigs(config)).toThrow(); }); + + it('throws an error when no organization or app is configured', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + catalogPath: '/*/catalog-info.yaml', + }, + }, + }, + }); + + expect(() => readProviderConfigs(config)).toThrow(); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index e33e7adfdd..64fac09f6e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -35,7 +35,8 @@ export const DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE = { export type GithubEntityProviderConfig = { id: string; catalogPath: string; - organization: string; + organization?: string; + app?: number; host: string; filters: { repository?: RegExp; @@ -62,7 +63,7 @@ export function readProviderConfigs( return []; } - if (providersConfig.has('organization')) { + if (providersConfig.has('organization') || providersConfig.has('app')) { // simple/single config variant return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; } @@ -78,7 +79,15 @@ function readProviderConfig( id: string, config: Config, ): GithubEntityProviderConfig { - const organization = config.getString('organization'); + const organization = config.getOptionalString('organization'); + const app = config.getOptionalNumber('app'); + + if (!organization && !app) { + throw new Error( + 'Error while processing GitHub provider config. Either organization or app must be specified.', + ); + } + const catalogPath = config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH; const host = config.getOptionalString('host') ?? 'github.com'; @@ -123,6 +132,7 @@ function readProviderConfig( id, catalogPath, organization, + app, host, filters: { repository: repositoryPattern diff --git a/yarn.lock b/yarn.lock index 5487f23662..e9467c9bfe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4485,6 +4485,7 @@ __metadata: luxon: "npm:^3.0.0" minimatch: "npm:^9.0.0" msw: "npm:^2.0.0" + type-fest: "npm:^4.41.0" uuid: "npm:^11.0.0" languageName: unknown linkType: soft @@ -47725,7 +47726,7 @@ __metadata: languageName: node linkType: hard -"type-fest@npm:^4.26.1, type-fest@npm:^4.3.1": +"type-fest@npm:^4.26.1, type-fest@npm:^4.3.1, type-fest@npm:^4.41.0": version: 4.41.0 resolution: "type-fest@npm:4.41.0" checksum: 10/617ace794ac0893c2986912d28b3065ad1afb484cad59297835a0807dc63286c39e8675d65f7de08fafa339afcb8fe06a36e9a188b9857756ae1e92ee8bda212