Update docs and code based on review comments
Signed-off-by: abhishekbvs <bvsabhishek@gmail.com>
This commit is contained in:
abhishekbvs
@@ -1,83 +0,0 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-github': minor
|
||||
'@backstage/plugin-catalog-backend-module-github-org': minor
|
||||
---
|
||||
|
||||
Added configurable page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with large GitHub organizations.
|
||||
|
||||
**Default Values Changed:**
|
||||
|
||||
To prevent `RESOURCE_LIMITS_EXCEEDED` errors by default, the page sizes have been reduced to 50% of previous values:
|
||||
|
||||
- `teams`: 50 → **25**
|
||||
- `teamMembers`: 100 → **50**
|
||||
- `organizationMembers`: 100 → **50**
|
||||
- `repositories`: 50 → **25**
|
||||
|
||||
**New Configuration:**
|
||||
|
||||
You can now configure page sizes in `app-config.yaml` to customize GitHub API resource consumption:
|
||||
|
||||
**For `githubOrg` provider (users and teams):**
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
githubOrg:
|
||||
- id: production
|
||||
githubUrl: https://github.com
|
||||
orgs: ['your-org']
|
||||
schedule:
|
||||
frequency: { minutes: 30 }
|
||||
timeout: { minutes: 3 }
|
||||
# Optional: Customize page sizes (defaults shown below)
|
||||
pageSizes:
|
||||
teams: 25 # Default: 25
|
||||
teamMembers: 50 # Default: 50
|
||||
organizationMembers: 50 # Default: 50
|
||||
repositories: 25 # Default: 25
|
||||
```
|
||||
|
||||
**For `github` provider (repositories):**
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
github:
|
||||
myorg:
|
||||
organization: 'your-org'
|
||||
catalogPath: '/catalog-info.yaml'
|
||||
schedule:
|
||||
frequency: { minutes: 30 }
|
||||
timeout: { minutes: 3 }
|
||||
# Optional: Customize page sizes (defaults shown below)
|
||||
pageSizes:
|
||||
repositories: 25 # Default: 25
|
||||
```
|
||||
|
||||
**Breaking Changes:**
|
||||
|
||||
The default page sizes have been reduced by 50% to prevent `RESOURCE_LIMITS_EXCEEDED` errors with large organizations. This may result in:
|
||||
|
||||
- ✅ **More stable syncs** for large organizations (200+ teams)
|
||||
- ⚠️ **Slightly more API calls** due to additional pagination
|
||||
- ⚠️ **Slightly slower sync times** (typically 10-20% slower)
|
||||
|
||||
If you need the previous behavior, you can restore the old values in your configuration:
|
||||
|
||||
```yaml
|
||||
pageSizes:
|
||||
teams: 50
|
||||
teamMembers: 100
|
||||
organizationMembers: 100
|
||||
repositories: 50
|
||||
```
|
||||
|
||||
**Benefits:**
|
||||
|
||||
- Prevents `RESOURCE_LIMITS_EXCEEDED` errors for large GitHub organizations (200+ teams)
|
||||
- Configurable per provider instance
|
||||
- No performance impact for smaller organizations
|
||||
- All data still synced through pagination
|
||||
|
||||
Resolves GitHub issue #31437
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-github': patch
|
||||
---
|
||||
|
||||
Added configurable pageSizes for GitHub GraphQL API queries to prevent RESOURCE_LIMITS_EXCEEDED errors with organizations with large number of teams, members and repositories. Default page sizes reduced by 50% to improve stability.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-github-org': patch
|
||||
---
|
||||
|
||||
Added pageSizes configuration schema to support configurable page sizes for GitHub GraphQL API queries. This enables the configuration to be defined in catalogModuleGithubOrgEntityProvider.
|
||||
@@ -210,6 +210,8 @@ catalog:
|
||||
filters: # optional filters
|
||||
branch: 'develop' # optional string
|
||||
repository: '.*' # optional Regex
|
||||
pageSizes:
|
||||
repositories: 25
|
||||
wildcardProviderId:
|
||||
organization: 'new-org' # string
|
||||
catalogPath: '/groups/**/*.yaml' # this will search all folders for files that end in .yaml
|
||||
@@ -309,25 +311,9 @@ If you do so, `default` will be used as provider ID.
|
||||
- **`scope`** _(optional)_:
|
||||
`'global'` or `'local'`. Sets the scope of concurrency control.
|
||||
- **`pageSizes`** _(optional)_:
|
||||
Configure page sizes for GitHub GraphQL API queries. This can help prevent `RESOURCE_LIMITS_EXCEEDED` errors with large organizations.
|
||||
Configure page sizes for GitHub GraphQL API queries. This can help prevent `RESOURCE_LIMITS_EXCEEDED` errors.
|
||||
- **`repositories`** _(optional)_:
|
||||
Number of repositories to fetch per page. Defaults to `25`.
|
||||
|
||||
Example with page sizes configuration:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
github:
|
||||
myOrganization:
|
||||
organization: 'my-large-org'
|
||||
catalogPath: '/catalog-info.yaml'
|
||||
schedule:
|
||||
frequency: { minutes: 30 }
|
||||
timeout: { minutes: 3 }
|
||||
pageSizes:
|
||||
repositories: 15 # Reduce if hitting API limits
|
||||
```
|
||||
Number of repositories to fetch per page. Defaults to `25`. Reduce this value if hitting API resource limits.
|
||||
|
||||
## GitHub API Rate Limits
|
||||
|
||||
|
||||
@@ -79,6 +79,10 @@ catalog:
|
||||
initialDelay: { seconds: 30 }
|
||||
frequency: { hours: 1 }
|
||||
timeout: { minutes: 50 }
|
||||
pageSizes:
|
||||
teams: 25
|
||||
teamMembers: 50
|
||||
organizationMembers: 50
|
||||
- id: ghe
|
||||
githubUrl: https://ghe.mycompany.com
|
||||
orgs: ['internal-1', 'internal-2', 'internal-3']
|
||||
@@ -94,37 +98,13 @@ Directly under the `githubOrg` is a list of configurations, each entry is a stru
|
||||
- `githubUrl`: The target that this provider should consume
|
||||
- `orgs` (optional): The list of the GitHub orgs to consume. If you only list a single org the generated group entities will use the `default` namespace, otherwise they will use the org name as the namespace. By default the provider will consume all accessible orgs on the given GitHub instance (support for GitHub App integration only).
|
||||
- `schedule`: The refresh schedule to use, matches the structure of [`SchedulerServiceTaskScheduleDefinitionConfig`](https://backstage.io/docs/reference/backend-plugin-api.schedulerservicetaskscheduledefinitionconfig/)
|
||||
- `pageSizes` (optional): Configure page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with large organizations. See [Page Sizes Configuration](#page-sizes-configuration) below for details.
|
||||
- `pageSizes` (optional): Configure page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors. You can configure the following page sizes:
|
||||
|
||||
### Page Sizes Configuration
|
||||
- `teams`: Number of teams to fetch per page when querying organization teams (default: 25)
|
||||
- `teamMembers`: Number of team members to fetch per page when querying team members (default: 50)
|
||||
- `organizationMembers`: Number of organization members to fetch per page (default: 50)
|
||||
|
||||
For large GitHub organizations (200+ teams), you may encounter `RESOURCE_LIMITS_EXCEEDED` errors due to GitHub's GraphQL API resource limits. You can configure page sizes to reduce the number of records fetched per API request:
|
||||
|
||||
```yaml title="app-config.yaml"
|
||||
catalog:
|
||||
providers:
|
||||
githubOrg:
|
||||
- id: production
|
||||
githubUrl: https://github.com
|
||||
orgs: ['large-org']
|
||||
schedule:
|
||||
frequency: { hours: 1 }
|
||||
timeout: { minutes: 50 }
|
||||
pageSizes:
|
||||
teams: 25 # Default: 25
|
||||
teamMembers: 50 # Default: 50
|
||||
organizationMembers: 50 # Default: 50
|
||||
repositories: 25 # Default: 25
|
||||
```
|
||||
|
||||
**Configuration Options:**
|
||||
|
||||
- `teams`: Number of teams to fetch per page when querying organization teams (default: 25)
|
||||
- `teamMembers`: Number of team members to fetch per page when querying team members (default: 50)
|
||||
- `organizationMembers`: Number of organization members to fetch per page (default: 50)
|
||||
- `repositories`: Number of repositories to fetch per page (default: 25)
|
||||
|
||||
**Note:** Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent resource limit errors for large organizations.
|
||||
Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent API resource limits for organizations with large number of teams and members.
|
||||
|
||||
### Events Support
|
||||
|
||||
|
||||
@@ -105,7 +105,6 @@ describe('catalogModuleGithubOrgEntityProvider', () => {
|
||||
teams: 10,
|
||||
teamMembers: 25,
|
||||
organizationMembers: 30,
|
||||
repositories: 15,
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
@@ -138,7 +138,6 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{
|
||||
teams?: number;
|
||||
teamMembers?: number;
|
||||
organizationMembers?: number;
|
||||
repositories?: number;
|
||||
};
|
||||
}> {
|
||||
const baseKey = 'catalog.providers.githubOrg';
|
||||
@@ -165,7 +164,6 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{
|
||||
organizationMembers: c.getOptionalNumber(
|
||||
'pageSizes.organizationMembers',
|
||||
),
|
||||
repositories: c.getOptionalNumber('pageSizes.repositories'),
|
||||
}
|
||||
: undefined,
|
||||
}));
|
||||
|
||||
+4
-14
@@ -134,7 +134,7 @@ export interface Config {
|
||||
|
||||
/**
|
||||
* (Optional) Page sizes for GitHub GraphQL API queries.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
|
||||
*/
|
||||
pageSizes?: {
|
||||
/**
|
||||
@@ -224,7 +224,7 @@ export interface Config {
|
||||
|
||||
/**
|
||||
* (Optional) Page sizes for GitHub GraphQL API queries.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
|
||||
*/
|
||||
pageSizes?: {
|
||||
/**
|
||||
@@ -271,7 +271,7 @@ export interface Config {
|
||||
|
||||
/**
|
||||
* (Optional) Page sizes for GitHub GraphQL API queries.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
|
||||
*/
|
||||
pageSizes?: {
|
||||
/**
|
||||
@@ -289,11 +289,6 @@ export interface Config {
|
||||
* Default: `50`.
|
||||
*/
|
||||
organizationMembers?: number;
|
||||
/**
|
||||
* (Optional) Number of repositories to fetch per page when querying repositories.
|
||||
* Default: `25`.
|
||||
*/
|
||||
repositories?: number;
|
||||
};
|
||||
}
|
||||
| Array<{
|
||||
@@ -327,7 +322,7 @@ export interface Config {
|
||||
|
||||
/**
|
||||
* (Optional) Page sizes for GitHub GraphQL API queries.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
|
||||
*/
|
||||
pageSizes?: {
|
||||
/**
|
||||
@@ -345,11 +340,6 @@ export interface Config {
|
||||
* Default: `50`.
|
||||
*/
|
||||
organizationMembers?: number;
|
||||
/**
|
||||
* (Optional) Number of repositories to fetch per page when querying repositories.
|
||||
* Default: `25`.
|
||||
*/
|
||||
repositories?: number;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
|
||||
@@ -40,8 +40,6 @@ export {
|
||||
type TeamTransformer,
|
||||
defaultOrganizationTeamTransformer,
|
||||
type TransformerContext,
|
||||
type GithubPageSizes,
|
||||
DEFAULT_PAGE_SIZES,
|
||||
} from './lib';
|
||||
|
||||
export * from './deprecated';
|
||||
|
||||
@@ -910,7 +910,7 @@ describe('github', () => {
|
||||
it('uses custom page sizes for getOrganizationUsers', async () => {
|
||||
server.use(
|
||||
graphqlMsw.query('users', ({ variables }) => {
|
||||
expect(variables.pageSize).toBe(30);
|
||||
expect(variables.organizationMembersPageSize).toBe(30);
|
||||
return HttpResponse.json({
|
||||
data: {
|
||||
organization: {
|
||||
|
||||
@@ -190,9 +190,9 @@ export async function getOrganizationUsers(
|
||||
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
|
||||
): Promise<{ users: Entity[] }> {
|
||||
const query = `
|
||||
query users($org: String!, $email: Boolean!, $cursor: String, $pageSize: Int!) {
|
||||
query users($org: String!, $email: Boolean!, $cursor: String, $organizationMembersPageSize: Int!) {
|
||||
organization(login: $org) {
|
||||
membersWithRole(first: $pageSize, after: $cursor) {
|
||||
membersWithRole(first: $organizationMembersPageSize, after: $cursor) {
|
||||
pageInfo { hasNextPage, endCursor }
|
||||
nodes {
|
||||
avatarUrl,
|
||||
@@ -218,7 +218,7 @@ export async function getOrganizationUsers(
|
||||
{
|
||||
org,
|
||||
email: tokenType === 'token',
|
||||
pageSize: pageSizes.organizationMembers,
|
||||
organizationMembersPageSize: pageSizes.organizationMembers,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ export interface GithubMultiOrgEntityProviderOptions {
|
||||
|
||||
/**
|
||||
* Optionally configure page sizes for GitHub GraphQL API queries.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors with large orgs.
|
||||
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
|
||||
*/
|
||||
pageSizes?: Partial<GithubPageSizes>;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user