chore: add page size options to GithubOrgEntityProvider

Signed-off-by: Rogerio Angeliski <angeliski@hotmail.com>

.changeset/swift-moments-think.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-github': patch
+---
+
+Added configurable `pageSizes` option to `GithubOrgEntityProvider` for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with organizations with large number of teams and members. This aligns the configuration options with `GithubMultiOrgEntityProvider`.

plugins/catalog-backend-module-github/report.api.md

@@ -229,6 +229,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
     githubCredentialsProvider?: GithubCredentialsProvider;
     userTransformer?: UserTransformer;
     teamTransformer?: TeamTransformer;
+    pageSizes?: Partial<GithubPageSizes>;
     excludeSuspendedUsers?: boolean;
   });
   connect(connection: EntityProviderConnection): Promise<void>;
@@ -252,6 +253,7 @@ export interface GithubOrgEntityProviderOptions {
   id: string;
   logger: LoggerService;
   orgUrl: string;
+  pageSizes?: Partial<GithubPageSizes>;
   schedule?: 'manual' | SchedulerServiceTaskRunner;
   teamTransformer?: TeamTransformer;
   userTransformer?: UserTransformer;

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts

@@ -59,6 +59,7 @@ import {
   getOrganizationTeams,
   getOrganizationTeamsFromUsers,
   getOrganizationUsers,
+  GithubPageSizes,
   GithubTeam,
 } from '../lib/github';
 import { areGroupEntities, areUserEntities } from '../lib/guards';
@@ -132,6 +133,12 @@ export interface GithubOrgEntityProviderOptions {
    */
   teamTransformer?: TeamTransformer;
 
+  /**
+   * Optionally configure page sizes for GitHub GraphQL API queries.
+   * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
+   */
+  pageSizes?: Partial<GithubPageSizes>;
+
   /**
    * Optionally exclude suspended users when querying organization users.
    * @defaultValue false
@@ -176,6 +183,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
       userTransformer: options.userTransformer,
       teamTransformer: options.teamTransformer,
       events: options.events,
+      pageSizes: options.pageSizes,
       excludeSuspendedUsers: options.excludeSuspendedUsers,
     });
 
@@ -194,6 +202,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
       githubCredentialsProvider?: GithubCredentialsProvider;
       userTransformer?: UserTransformer;
       teamTransformer?: TeamTransformer;
+      pageSizes?: Partial<GithubPageSizes>;
       excludeSuspendedUsers?: boolean;
     },
   ) {
@@ -207,6 +216,13 @@ export class GithubOrgEntityProvider implements EntityProvider {
     return `GithubOrgEntityProvider:${this.options.id}`;
   }
 
+  private getPageSizes(): GithubPageSizes {
+    return {
+      ...DEFAULT_PAGE_SIZES,
+      ...this.options.pageSizes,
+    };
+  }
+
   /** {@inheritdoc @backstage/plugin-catalog-node#EntityProvider.connect} */
   async connect(connection: EntityProviderConnection) {
     this.connection = connection;
@@ -242,18 +258,20 @@ export class GithubOrgEntityProvider implements EntityProvider {
     });
 
     const { org } = parseGithubOrgUrl(this.options.orgUrl);
+    const pageSizes = this.getPageSizes();
     const { users } = await getOrganizationUsers(
       client,
       org,
       tokenType,
       this.options.userTransformer,
-      DEFAULT_PAGE_SIZES,
+      pageSizes,
       this.options.excludeSuspendedUsers,
     );
     const { teams } = await getOrganizationTeams(
       client,
       org,
       this.options.teamTransformer,
+      pageSizes,
     );
 
     if (areGroupEntities(teams)) {
@@ -365,6 +383,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
     });
 
     const { org } = parseGithubOrgUrl(this.options.orgUrl);
+    const pageSizes = this.getPageSizes();
     const { team } = await getOrganizationTeam(
       client,
       org,
@@ -377,7 +396,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
       org,
       tokenType,
       this.options.userTransformer,
-      DEFAULT_PAGE_SIZES,
+      pageSizes,
       this.options.excludeSuspendedUsers,
     );
 
@@ -395,6 +414,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
       org,
       usersToRebuild.map(u => u.metadata.name),
       this.options.teamTransformer,
+      pageSizes,
     );
 
     if (areGroupEntities(teams)) {
@@ -458,6 +478,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
     });
 
     const { org } = parseGithubOrgUrl(this.options.orgUrl);
+    const pageSizes = this.getPageSizes();
     const { team } = await getOrganizationTeam(
       client,
       org,
@@ -470,7 +491,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
       org,
       tokenType,
       this.options.userTransformer,
-      DEFAULT_PAGE_SIZES,
+      pageSizes,
       this.options.excludeSuspendedUsers,
     );
 
@@ -481,6 +502,7 @@ export class GithubOrgEntityProvider implements EntityProvider {
       org,
       [userLogin],
       this.options.teamTransformer,
+      pageSizes,
     );
 
     // we include group because the removed event need to update the old group too