From 999d1c17479122f847ff9338c5bdd1bf792c48b1 Mon Sep 17 00:00:00 2001 From: abhishekbvs Date: Fri, 24 Oct 2025 00:32:24 +0530 Subject: [PATCH] Update docs and code based on review comments Signed-off-by: abhishekbvs --- .changeset/github-api-page-sizes.md | 83 ------------------- .changeset/pretty-kids-allow.md | 5 ++ .changeset/tough-sloths-spend.md | 5 ++ docs/integrations/github/discovery.md | 22 +---- docs/integrations/github/org.md | 38 ++------- .../src/module.test.ts | 1 - .../src/module.ts | 2 - .../catalog-backend-module-github/config.d.ts | 18 +--- .../src/index.ts | 2 - .../src/lib/github.test.ts | 2 +- .../src/lib/github.ts | 6 +- .../providers/GithubMultiOrgEntityProvider.ts | 2 +- 12 files changed, 32 insertions(+), 154 deletions(-) delete mode 100644 .changeset/github-api-page-sizes.md create mode 100644 .changeset/pretty-kids-allow.md create mode 100644 .changeset/tough-sloths-spend.md diff --git a/.changeset/github-api-page-sizes.md b/.changeset/github-api-page-sizes.md deleted file mode 100644 index 724adbd764..0000000000 --- a/.changeset/github-api-page-sizes.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -'@backstage/plugin-catalog-backend-module-github': minor -'@backstage/plugin-catalog-backend-module-github-org': minor ---- - -Added configurable page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with large GitHub organizations. - -**Default Values Changed:** - -To prevent `RESOURCE_LIMITS_EXCEEDED` errors by default, the page sizes have been reduced to 50% of previous values: - -- `teams`: 50 → **25** -- `teamMembers`: 100 → **50** -- `organizationMembers`: 100 → **50** -- `repositories`: 50 → **25** - -**New Configuration:** - -You can now configure page sizes in `app-config.yaml` to customize GitHub API resource consumption: - -**For `githubOrg` provider (users and teams):** - -```yaml -catalog: - providers: - githubOrg: - - id: production - githubUrl: https://github.com - orgs: ['your-org'] - schedule: - frequency: { minutes: 30 } - timeout: { minutes: 3 } - # Optional: Customize page sizes (defaults shown below) - pageSizes: - teams: 25 # Default: 25 - teamMembers: 50 # Default: 50 - organizationMembers: 50 # Default: 50 - repositories: 25 # Default: 25 -``` - -**For `github` provider (repositories):** - -```yaml -catalog: - providers: - github: - myorg: - organization: 'your-org' - catalogPath: '/catalog-info.yaml' - schedule: - frequency: { minutes: 30 } - timeout: { minutes: 3 } - # Optional: Customize page sizes (defaults shown below) - pageSizes: - repositories: 25 # Default: 25 -``` - -**Breaking Changes:** - -The default page sizes have been reduced by 50% to prevent `RESOURCE_LIMITS_EXCEEDED` errors with large organizations. This may result in: - -- ✅ **More stable syncs** for large organizations (200+ teams) -- ⚠️ **Slightly more API calls** due to additional pagination -- ⚠️ **Slightly slower sync times** (typically 10-20% slower) - -If you need the previous behavior, you can restore the old values in your configuration: - -```yaml -pageSizes: - teams: 50 - teamMembers: 100 - organizationMembers: 100 - repositories: 50 -``` - -**Benefits:** - -- Prevents `RESOURCE_LIMITS_EXCEEDED` errors for large GitHub organizations (200+ teams) -- Configurable per provider instance -- No performance impact for smaller organizations -- All data still synced through pagination - -Resolves GitHub issue #31437 diff --git a/.changeset/pretty-kids-allow.md b/.changeset/pretty-kids-allow.md new file mode 100644 index 0000000000..acb258c467 --- /dev/null +++ b/.changeset/pretty-kids-allow.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added configurable pageSizes for GitHub GraphQL API queries to prevent RESOURCE_LIMITS_EXCEEDED errors with organizations with large number of teams, members and repositories. Default page sizes reduced by 50% to improve stability. diff --git a/.changeset/tough-sloths-spend.md b/.changeset/tough-sloths-spend.md new file mode 100644 index 0000000000..f88c3e14c3 --- /dev/null +++ b/.changeset/tough-sloths-spend.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github-org': patch +--- + +Added pageSizes configuration schema to support configurable page sizes for GitHub GraphQL API queries. This enables the configuration to be defined in catalogModuleGithubOrgEntityProvider. diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 8546e297c6..9724ef3177 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -210,6 +210,8 @@ catalog: filters: # optional filters branch: 'develop' # optional string repository: '.*' # optional Regex + pageSizes: + repositories: 25 wildcardProviderId: organization: 'new-org' # string catalogPath: '/groups/**/*.yaml' # this will search all folders for files that end in .yaml @@ -309,25 +311,9 @@ If you do so, `default` will be used as provider ID. - **`scope`** _(optional)_: `'global'` or `'local'`. Sets the scope of concurrency control. - **`pageSizes`** _(optional)_: - Configure page sizes for GitHub GraphQL API queries. This can help prevent `RESOURCE_LIMITS_EXCEEDED` errors with large organizations. + Configure page sizes for GitHub GraphQL API queries. This can help prevent `RESOURCE_LIMITS_EXCEEDED` errors. - **`repositories`** _(optional)_: - Number of repositories to fetch per page. Defaults to `25`. - -Example with page sizes configuration: - -```yaml -catalog: - providers: - github: - myOrganization: - organization: 'my-large-org' - catalogPath: '/catalog-info.yaml' - schedule: - frequency: { minutes: 30 } - timeout: { minutes: 3 } - pageSizes: - repositories: 15 # Reduce if hitting API limits -``` + Number of repositories to fetch per page. Defaults to `25`. Reduce this value if hitting API resource limits. ## GitHub API Rate Limits diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 9962ae641e..6526183f2d 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -79,6 +79,10 @@ catalog: initialDelay: { seconds: 30 } frequency: { hours: 1 } timeout: { minutes: 50 } + pageSizes: + teams: 25 + teamMembers: 50 + organizationMembers: 50 - id: ghe githubUrl: https://ghe.mycompany.com orgs: ['internal-1', 'internal-2', 'internal-3'] @@ -94,37 +98,13 @@ Directly under the `githubOrg` is a list of configurations, each entry is a stru - `githubUrl`: The target that this provider should consume - `orgs` (optional): The list of the GitHub orgs to consume. If you only list a single org the generated group entities will use the `default` namespace, otherwise they will use the org name as the namespace. By default the provider will consume all accessible orgs on the given GitHub instance (support for GitHub App integration only). - `schedule`: The refresh schedule to use, matches the structure of [`SchedulerServiceTaskScheduleDefinitionConfig`](https://backstage.io/docs/reference/backend-plugin-api.schedulerservicetaskscheduledefinitionconfig/) -- `pageSizes` (optional): Configure page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with large organizations. See [Page Sizes Configuration](#page-sizes-configuration) below for details. +- `pageSizes` (optional): Configure page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors. You can configure the following page sizes: -### Page Sizes Configuration + - `teams`: Number of teams to fetch per page when querying organization teams (default: 25) + - `teamMembers`: Number of team members to fetch per page when querying team members (default: 50) + - `organizationMembers`: Number of organization members to fetch per page (default: 50) -For large GitHub organizations (200+ teams), you may encounter `RESOURCE_LIMITS_EXCEEDED` errors due to GitHub's GraphQL API resource limits. You can configure page sizes to reduce the number of records fetched per API request: - -```yaml title="app-config.yaml" -catalog: - providers: - githubOrg: - - id: production - githubUrl: https://github.com - orgs: ['large-org'] - schedule: - frequency: { hours: 1 } - timeout: { minutes: 50 } - pageSizes: - teams: 25 # Default: 25 - teamMembers: 50 # Default: 50 - organizationMembers: 50 # Default: 50 - repositories: 25 # Default: 25 -``` - -**Configuration Options:** - -- `teams`: Number of teams to fetch per page when querying organization teams (default: 25) -- `teamMembers`: Number of team members to fetch per page when querying team members (default: 50) -- `organizationMembers`: Number of organization members to fetch per page (default: 50) -- `repositories`: Number of repositories to fetch per page (default: 25) - -**Note:** Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent resource limit errors for large organizations. + Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent API resource limits for organizations with large number of teams and members. ### Events Support diff --git a/plugins/catalog-backend-module-github-org/src/module.test.ts b/plugins/catalog-backend-module-github-org/src/module.test.ts index da48f013cb..1fcff306fe 100644 --- a/plugins/catalog-backend-module-github-org/src/module.test.ts +++ b/plugins/catalog-backend-module-github-org/src/module.test.ts @@ -105,7 +105,6 @@ describe('catalogModuleGithubOrgEntityProvider', () => { teams: 10, teamMembers: 25, organizationMembers: 30, - repositories: 15, }, }, ], diff --git a/plugins/catalog-backend-module-github-org/src/module.ts b/plugins/catalog-backend-module-github-org/src/module.ts index b9cd227abf..2d349cb873 100644 --- a/plugins/catalog-backend-module-github-org/src/module.ts +++ b/plugins/catalog-backend-module-github-org/src/module.ts @@ -138,7 +138,6 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{ teams?: number; teamMembers?: number; organizationMembers?: number; - repositories?: number; }; }> { const baseKey = 'catalog.providers.githubOrg'; @@ -165,7 +164,6 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{ organizationMembers: c.getOptionalNumber( 'pageSizes.organizationMembers', ), - repositories: c.getOptionalNumber('pageSizes.repositories'), } : undefined, })); diff --git a/plugins/catalog-backend-module-github/config.d.ts b/plugins/catalog-backend-module-github/config.d.ts index d795af2774..f3cbb15e06 100644 --- a/plugins/catalog-backend-module-github/config.d.ts +++ b/plugins/catalog-backend-module-github/config.d.ts @@ -134,7 +134,7 @@ export interface Config { /** * (Optional) Page sizes for GitHub GraphQL API queries. - * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs. + * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. */ pageSizes?: { /** @@ -224,7 +224,7 @@ export interface Config { /** * (Optional) Page sizes for GitHub GraphQL API queries. - * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs. + * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. */ pageSizes?: { /** @@ -271,7 +271,7 @@ export interface Config { /** * (Optional) Page sizes for GitHub GraphQL API queries. - * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs. + * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. */ pageSizes?: { /** @@ -289,11 +289,6 @@ export interface Config { * Default: `50`. */ organizationMembers?: number; - /** - * (Optional) Number of repositories to fetch per page when querying repositories. - * Default: `25`. - */ - repositories?: number; }; } | Array<{ @@ -327,7 +322,7 @@ export interface Config { /** * (Optional) Page sizes for GitHub GraphQL API queries. - * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs. + * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. */ pageSizes?: { /** @@ -345,11 +340,6 @@ export interface Config { * Default: `50`. */ organizationMembers?: number; - /** - * (Optional) Number of repositories to fetch per page when querying repositories. - * Default: `25`. - */ - repositories?: number; }; }>; }; diff --git a/plugins/catalog-backend-module-github/src/index.ts b/plugins/catalog-backend-module-github/src/index.ts index 6f14bb25ef..539e4ae14f 100644 --- a/plugins/catalog-backend-module-github/src/index.ts +++ b/plugins/catalog-backend-module-github/src/index.ts @@ -40,8 +40,6 @@ export { type TeamTransformer, defaultOrganizationTeamTransformer, type TransformerContext, - type GithubPageSizes, - DEFAULT_PAGE_SIZES, } from './lib'; export * from './deprecated'; diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index f8a4fd6201..ca9b3eb692 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -910,7 +910,7 @@ describe('github', () => { it('uses custom page sizes for getOrganizationUsers', async () => { server.use( graphqlMsw.query('users', ({ variables }) => { - expect(variables.pageSize).toBe(30); + expect(variables.organizationMembersPageSize).toBe(30); return HttpResponse.json({ data: { organization: { diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index a3a88b1e97..625fb57bba 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -190,9 +190,9 @@ export async function getOrganizationUsers( pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES, ): Promise<{ users: Entity[] }> { const query = ` - query users($org: String!, $email: Boolean!, $cursor: String, $pageSize: Int!) { + query users($org: String!, $email: Boolean!, $cursor: String, $organizationMembersPageSize: Int!) { organization(login: $org) { - membersWithRole(first: $pageSize, after: $cursor) { + membersWithRole(first: $organizationMembersPageSize, after: $cursor) { pageInfo { hasNextPage, endCursor } nodes { avatarUrl, @@ -218,7 +218,7 @@ export async function getOrganizationUsers( { org, email: tokenType === 'token', - pageSize: pageSizes.organizationMembers, + organizationMembersPageSize: pageSizes.organizationMembers, }, ); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index bb0cc98c40..839ca66e40 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -171,7 +171,7 @@ export interface GithubMultiOrgEntityProviderOptions { /** * Optionally configure page sizes for GitHub GraphQL API queries. - * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs. + * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. */ pageSizes?: Partial; }