paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

Merge pull request #29887 from benjidotsh/catalog/github-app-discovery

Browse Source 
feat(catalog): implement discovery by GitHub app
This commit is contained in:
Fredrik Adelöw
2025-09-01 12:42:20 +02:00
committed by GitHub
parent 6bf7c028a8 198d170443
commit 66ee6e2a33
12 changed files with 176 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/legal-lemons-attend.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/integration': minor
---
Added support for limiting GithubAppCredentialsMux to specific apps
.changeset/sharp-carrots-spend.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github': minor
---
Added support for discovery by app
docs/integrations/github/discovery.md
+8 -6
View File
@@ -2,7 +2,7 @@
id: discovery
title: GitHub Discovery
sidebar_label: Discovery
description: Automatically discovering catalog entities from repositories in a GitHub organization
description: Automatically discovering catalog entities from repositories in a GitHub organization or App
---
:::info
@@ -12,8 +12,8 @@ This documentation is written for [the new backend system](../../backend-system/
## GitHub Provider
The GitHub integration has a discovery provider for discovering catalog
entities within a GitHub organization. The provider will crawl the GitHub
organization and register entities matching the configured path. This can be
entities within a GitHub organization or App. The provider will crawl the GitHub
organization or App and register entities matching the configured path. This can be
useful as an alternative to static locations or manually adding things to the
catalog. This is the preferred method for ingesting entities into the catalog.
@@ -252,7 +252,7 @@ catalog:
catalogPath: '/catalog-info.yaml' # string
```
This provider supports multiple organizations via unique provider IDs.
This provider supports multiple organizations and apps via unique provider IDs.
:::note Note
@@ -288,9 +288,11 @@ If you do so, `default` will be used as provider ID.
Whether to include archived repositories. Defaults to `false`.
- **`host`** _(optional)_:
The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md).
- **`organization`**:
- **`organization`** _(required, unless `app` is set)_:
Name of your organization account/workspace.
If you want to add multiple organizations, you need to add one provider config each.
If you want to add multiple organizations, you need to add one provider config each or specify `app` instead.
- **`app`** _(required, unless `organization` is set)_:
ID of your GitHub App.
- **`validateLocationsExist`** _(optional)_:
Whether to validate locations that exist before emitting them.
This option avoids generating locations for catalog info files that do not exist in the source repository.
packages/integration/report.api.md
+1 -1
View File
@@ -669,7 +669,7 @@ export type GithubAppConfig = {
// @public
export class GithubAppCredentialsMux {
constructor(config: GithubIntegrationConfig);
constructor(config: GithubIntegrationConfig, appIds?: number[]);
// (undocumented)
getAllInstallations(): Promise<
RestEndpointMethodTypes['apps']['listInstallations']['response']['data']
packages/integration/src/github/SingleInstanceGithubCredentialsProvider.ts
+4 -2
View File
@@ -201,9 +201,11 @@ class GithubAppManager {
export class GithubAppCredentialsMux {
private readonly apps: GithubAppManager[];
constructor(config: GithubIntegrationConfig) {
constructor(config: GithubIntegrationConfig, appIds: number[] = []) {
this.apps =
config.apps?.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? [];
config.apps
?.filter(app => (appIds.length ? appIds.includes(app.appId) : true))
.map(ac => new GithubAppManager(ac, config.apiBaseUrl)) ?? [];
}
async getAllInstallations(): Promise<
plugins/catalog-backend-module-github/config.d.ts
Vendored
+12 -4
View File
@@ -62,9 +62,13 @@ export interface Config {
*/
host?: string;
/**
* (Required) Name of your organization account/workspace.
* (Required, unless `app` is set) Name of your organization account/workspace.
*/
organization: string;
organization?: string;
/**
* (Required, unless `organization` is set) ID of your GitHub App.
*/
app?: number;
/**
* (Optional) Path where to look for `catalog-info.yaml` files.
* You can use wildcards - `*` or `**` - to search the path and/or the filename
@@ -136,9 +140,13 @@ export interface Config {
*/
host?: string;
/**
* (Required) Name of your organization account/workspace.
* (Required, unless `app` is set) Name of your organization account/workspace.
*/
organization: string;
organization?: string;
/**
* (Required, unless `organization` is set) ID of your GitHub App.
*/
app?: number;
/**
* (Optional) Path where to look for `catalog-info.yaml` files.
* You can use wildcards - `*` or `**` - to search the path and/or the filename
plugins/catalog-backend-module-github/package.json
+2 -1
View File
@@ -74,7 +74,8 @@
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151",
"luxon": "^3.0.0",
"msw": "^2.0.0"
"msw": "^2.0.0",
"type-fest": "^4.41.0"
},
"configSchema": "config.d.ts"
}
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts
+7 -2
View File
@@ -35,6 +35,7 @@ import {
RepositoryEvent,
RepositoryRenamedEvent,
} from '@octokit/webhooks-types';
import type { PartialDeep } from 'type-fest';
jest.mock('../lib/github', () => {
return {
@@ -673,7 +674,8 @@ describe('GithubEntityProvider', () => {
topics: [],
html_url: `https://github.com/${organization}/test-repo`,
url: `https://github.com/${organization}/test-repo`,
} as Partial<PushEvent['repository']>;
owner: { login: 'test-org' },
} as PartialDeep<PushEvent['repository']>;
const catalogCommit = {
added: [] as string[],
@@ -1003,7 +1005,10 @@ describe('GithubEntityProvider', () => {
topics: [],
archived: action === 'archived',
private: action !== 'publicized',
} as Partial<RepositoryEvent['repository']>;
owner: {
login: 'test-org',
},
} as PartialDeep<RepositoryEvent['repository']>;
const event = {
action,
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts
+56 -20
View File
@@ -16,6 +16,7 @@
import { Config } from '@backstage/config';
import {
GithubAppCredentialsMux,
GithubCredentialsProvider,
GithubIntegration,
GithubIntegrationConfig,
@@ -82,6 +83,7 @@ type Repository = {
defaultBranchRef?: string;
isCatalogInfoFilePresent: boolean;
visibility: string;
organization: string;
};
/**
@@ -219,8 +221,25 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
);
}
private async createGraphqlClient() {
const organization = this.config.organization;
private async getOrganizations(): Promise<string[]> {
if (this.config.organization) return [this.config.organization];
const githubAppMux = new GithubAppCredentialsMux(this.integration, [
this.config.app!,
]);
const installs = await githubAppMux.getAllInstallations();
return installs
.map(install =>
install.target_type === 'Organization' &&
install.account &&
'login' in install.account
? install.account.login
: undefined,
)
.filter(Boolean) as string[];
}
private async createGraphqlClient(organization: string) {
const host = this.integration.host;
const orgUrl = `https://${host}/${organization}`;
@@ -236,15 +255,21 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
// go to the server and get all repositories
private async findCatalogFiles(): Promise<Repository[]> {
const organization = this.config.organization;
const organizations = await this.getOrganizations();
const catalogPath = this.config.catalogPath;
const client = await this.createGraphqlClient();
const { repositories: repositoriesFromGithub } =
await getOrganizationRepositories(client, organization, catalogPath);
const repositories = repositoriesFromGithub.map(
this.createRepoFromGithubResponse,
);
let repositories: Repository[] = [];
for (const organization of organizations) {
const client = await this.createGraphqlClient(organization);
const { repositories: repositoriesFromGithub } =
await getOrganizationRepositories(client, organization, catalogPath);
repositories = repositories.concat(
repositoriesFromGithub.map(r =>
this.createRepoFromGithubResponse(r, organization),
),
);
}
if (this.config.validateLocationsExist) {
return repositories.filter(
@@ -323,12 +348,15 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
}
private async onPush(event: PushEvent) {
const configOrganization = this.config.organization;
const eventOrganization = event.organization?.login;
const configOrganizations = (await this.getOrganizations()).map(org =>
org.toLocaleLowerCase('en-US'),
);
const eventOrganization =
event.organization?.login.toLocaleLowerCase('en-US');
if (
configOrganization.toLocaleLowerCase('en-US') !==
eventOrganization?.toLocaleLowerCase('en-US')
!eventOrganization ||
!configOrganizations.includes(eventOrganization)
) {
this.logger.debug(
`skipping push event from organization ${event.organization?.login}`,
@@ -412,12 +440,15 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
}
private async onRepoChange(event: RepositoryEvent) {
const configOrganization = this.config.organization;
const eventOrganization = event.organization?.login;
const configOrganizations = (await this.getOrganizations()).map(org =>
org.toLocaleLowerCase('en-US'),
);
const eventOrganization =
event.organization?.login.toLocaleLowerCase('en-US');
if (
configOrganization.toLocaleLowerCase('en-US') !==
eventOrganization?.toLocaleLowerCase('en-US')
!eventOrganization ||
!configOrganizations.includes(eventOrganization)
) {
this.logger.debug(
`skipping repository event from organization ${event.organization?.login}`,
@@ -612,16 +643,18 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
private async addEntitiesForRepo(repository: Repository) {
if (this.config.validateLocationsExist) {
const organization = this.config.organization;
const organization = repository.organization;
const catalogPath = this.config.catalogPath;
const client = await this.createGraphqlClient();
const client = await this.createGraphqlClient(organization);
const repositoryFromGithub = await getOrganizationRepository(
client,
organization,
repository.name,
catalogPath,
).then(r => (r ? this.createRepoFromGithubResponse(r) : null));
).then(r =>
r ? this.createRepoFromGithubResponse(r, organization) : null,
);
if (!repositoryFromGithub?.isCatalogInfoFilePresent) {
return;
@@ -651,11 +684,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
// only the catalog file will be recovered from the commits
isCatalogInfoFilePresent: true,
visibility: event.repository.visibility,
organization: event.repository.owner.login,
};
}
private createRepoFromGithubResponse(
repositoryResponse: RepositoryResponse,
organization: string,
): Repository {
return {
url: repositoryResponse.url,
@@ -670,6 +705,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
repositoryResponse.catalogInfoFile?.__typename === 'Blob' &&
repositoryResponse.catalogInfoFile.text !== '',
visibility: repositoryResponse.visibility,
organization,
};
}
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts
+61 -1
View File
@@ -111,13 +111,20 @@ describe('readProviderConfigs', () => {
},
},
},
providerAppOnly: {
app: '1234',
},
providerAppAndOrganization: {
app: '1234',
organization: 'test-org1',
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(10);
expect(providerConfigs).toHaveLength(12);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
@@ -313,6 +320,45 @@ describe('readProviderConfigs', () => {
},
validateLocationsExist: false,
});
expect(providerConfigs[10]).toEqual({
id: 'providerAppOnly',
app: 1234,
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
allowForks: true,
topic: {
include: undefined,
exclude: undefined,
},
visibility: undefined,
allowArchived: false,
},
schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE,
validateLocationsExist: false,
});
expect(providerConfigs[11]).toEqual({
id: 'providerAppAndOrganization',
app: 1234,
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
allowForks: true,
topic: {
include: undefined,
exclude: undefined,
},
visibility: undefined,
allowArchived: false,
},
schedule: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE,
validateLocationsExist: false,
});
});
it('defaults validateLocationsExist to false', () => {
@@ -365,4 +411,18 @@ describe('readProviderConfigs', () => {
expect(() => readProviderConfigs(config)).toThrow();
});
it('throws an error when no organization or app is configured', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
catalogPath: '/*/catalog-info.yaml',
},
},
},
});
expect(() => readProviderConfigs(config)).toThrow();
});
});
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts
+13 -3
View File
@@ -35,7 +35,8 @@ export const DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE = {
export type GithubEntityProviderConfig = {
id: string;
catalogPath: string;
organization: string;
organization?: string;
app?: number;
host: string;
filters: {
repository?: RegExp;
@@ -62,7 +63,7 @@ export function readProviderConfigs(
return [];
}
if (providersConfig.has('organization')) {
if (providersConfig.has('organization') || providersConfig.has('app')) {
// simple/single config variant
return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
}
@@ -78,7 +79,15 @@ function readProviderConfig(
id: string,
config: Config,
): GithubEntityProviderConfig {
const organization = config.getString('organization');
const organization = config.getOptionalString('organization');
const app = config.getOptionalNumber('app');
if (!organization && !app) {
throw new Error(
'Error while processing GitHub provider config. Either organization or app must be specified.',
);
}
const catalogPath =
config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH;
const host = config.getOptionalString('host') ?? 'github.com';
@@ -123,6 +132,7 @@ function readProviderConfig(
id,
catalogPath,
organization,
app,
host,
filters: {
repository: repositoryPattern
yarn.lock
+2 -1
View File
@@ -4485,6 +4485,7 @@ __metadata:
luxon: "npm:^3.0.0"
minimatch: "npm:^9.0.0"
msw: "npm:^2.0.0"
type-fest: "npm:^4.41.0"
uuid: "npm:^11.0.0"
languageName: unknown
linkType: soft
@@ -47725,7 +47726,7 @@ __metadata:
languageName: node
linkType: hard
"type-fest@npm:^4.26.1, type-fest@npm:^4.3.1":
"type-fest@npm:^4.26.1, type-fest@npm:^4.3.1, type-fest@npm:^4.41.0":
version: 4.41.0
resolution: "type-fest@npm:4.41.0"
checksum: 10/617ace794ac0893c2986912d28b3065ad1afb484cad59297835a0807dc63286c39e8675d65f7de08fafa339afcb8fe06a36e9a188b9857756ae1e92ee8bda212