Merge pull request #31474 from abhishekbvs/feat/github-plugin-api-page-sizes

feat: add configurable GitHub API page sizes

docs/integrations/github/discovery.md

@@ -210,6 +210,8 @@ catalog:
         filters: # optional filters
           branch: 'develop' # optional string
           repository: '.*' # optional Regex
+        pageSizes:
+          repositories: 25
       wildcardProviderId:
         organization: 'new-org' # string
         catalogPath: '/groups/**/*.yaml' # this will search all folders for files that end in .yaml
@@ -308,6 +310,10 @@ If you do so, `default` will be used as provider ID.
     The amount of time that should pass before the first invocation happens.
   - **`scope`** _(optional)_:
     `'global'` or `'local'`. Sets the scope of concurrency control.
+- **`pageSizes`** _(optional)_:
+  Configure page sizes for GitHub GraphQL API queries. This can help prevent `RESOURCE_LIMITS_EXCEEDED` errors.
+  - **`repositories`** _(optional)_:
+    Number of repositories to fetch per page. Defaults to `25`. Reduce this value if hitting API resource limits.
 
 ## GitHub API Rate Limits
 

docs/integrations/github/org.md

@@ -79,6 +79,10 @@ catalog:
           initialDelay: { seconds: 30 }
           frequency: { hours: 1 }
           timeout: { minutes: 50 }
+        pageSizes:
+          teams: 25
+          teamMembers: 50
+          organizationMembers: 50
       - id: ghe
         githubUrl: https://ghe.mycompany.com
         orgs: ['internal-1', 'internal-2', 'internal-3']
@@ -94,6 +98,13 @@ Directly under the `githubOrg` is a list of configurations, each entry is a stru
 - `githubUrl`: The target that this provider should consume
 - `orgs` (optional): The list of the GitHub orgs to consume. If you only list a single org the generated group entities will use the `default` namespace, otherwise they will use the org name as the namespace. By default the provider will consume all accessible orgs on the given GitHub instance (support for GitHub App integration only).
 - `schedule`: The refresh schedule to use, matches the structure of [`SchedulerServiceTaskScheduleDefinitionConfig`](https://backstage.io/docs/reference/backend-plugin-api.schedulerservicetaskscheduledefinitionconfig/)
+- `pageSizes` (optional): Configure page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors. You can configure the following page sizes:
+
+  - `teams`: Number of teams to fetch per page when querying organization teams (default: 25)
+  - `teamMembers`: Number of team members to fetch per page when querying team members (default: 50)
+  - `organizationMembers`: Number of organization members to fetch per page (default: 50)
+
+  Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent API resource limits for organizations with large number of teams and members.
 
 ### Events Support