From a413977c7c4078a34d49cc649cf007017bca818e Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Thu, 11 Dec 2025 16:12:30 -0300 Subject: [PATCH] chore: add page size options to GithubOrgEntityProvider Signed-off-by: Rogerio Angeliski --- .changeset/swift-moments-think.md | 5 ++++ .../report.api.md | 2 ++ .../src/providers/GithubOrgEntityProvider.ts | 28 +++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 .changeset/swift-moments-think.md diff --git a/.changeset/swift-moments-think.md b/.changeset/swift-moments-think.md new file mode 100644 index 0000000000..185fd4ff48 --- /dev/null +++ b/.changeset/swift-moments-think.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added configurable `pageSizes` option to `GithubOrgEntityProvider` for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with organizations with large number of teams and members. This aligns the configuration options with `GithubMultiOrgEntityProvider`. diff --git a/plugins/catalog-backend-module-github/report.api.md b/plugins/catalog-backend-module-github/report.api.md index 5f140a0d5e..aa275cfc9e 100644 --- a/plugins/catalog-backend-module-github/report.api.md +++ b/plugins/catalog-backend-module-github/report.api.md @@ -229,6 +229,7 @@ export class GithubOrgEntityProvider implements EntityProvider { githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + pageSizes?: Partial; excludeSuspendedUsers?: boolean; }); connect(connection: EntityProviderConnection): Promise; @@ -252,6 +253,7 @@ export interface GithubOrgEntityProviderOptions { id: string; logger: LoggerService; orgUrl: string; + pageSizes?: Partial; schedule?: 'manual' | SchedulerServiceTaskRunner; teamTransformer?: TeamTransformer; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 4758311e83..35ec849e12 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -59,6 +59,7 @@ import { getOrganizationTeams, getOrganizationTeamsFromUsers, getOrganizationUsers, + GithubPageSizes, GithubTeam, } from '../lib/github'; import { areGroupEntities, areUserEntities } from '../lib/guards'; @@ -132,6 +133,12 @@ export interface GithubOrgEntityProviderOptions { */ teamTransformer?: TeamTransformer; + /** + * Optionally configure page sizes for GitHub GraphQL API queries. + * Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors. + */ + pageSizes?: Partial; + /** * Optionally exclude suspended users when querying organization users. * @defaultValue false @@ -176,6 +183,7 @@ export class GithubOrgEntityProvider implements EntityProvider { userTransformer: options.userTransformer, teamTransformer: options.teamTransformer, events: options.events, + pageSizes: options.pageSizes, excludeSuspendedUsers: options.excludeSuspendedUsers, }); @@ -194,6 +202,7 @@ export class GithubOrgEntityProvider implements EntityProvider { githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + pageSizes?: Partial; excludeSuspendedUsers?: boolean; }, ) { @@ -207,6 +216,13 @@ export class GithubOrgEntityProvider implements EntityProvider { return `GithubOrgEntityProvider:${this.options.id}`; } + private getPageSizes(): GithubPageSizes { + return { + ...DEFAULT_PAGE_SIZES, + ...this.options.pageSizes, + }; + } + /** {@inheritdoc @backstage/plugin-catalog-node#EntityProvider.connect} */ async connect(connection: EntityProviderConnection) { this.connection = connection; @@ -242,18 +258,20 @@ export class GithubOrgEntityProvider implements EntityProvider { }); const { org } = parseGithubOrgUrl(this.options.orgUrl); + const pageSizes = this.getPageSizes(); const { users } = await getOrganizationUsers( client, org, tokenType, this.options.userTransformer, - DEFAULT_PAGE_SIZES, + pageSizes, this.options.excludeSuspendedUsers, ); const { teams } = await getOrganizationTeams( client, org, this.options.teamTransformer, + pageSizes, ); if (areGroupEntities(teams)) { @@ -365,6 +383,7 @@ export class GithubOrgEntityProvider implements EntityProvider { }); const { org } = parseGithubOrgUrl(this.options.orgUrl); + const pageSizes = this.getPageSizes(); const { team } = await getOrganizationTeam( client, org, @@ -377,7 +396,7 @@ export class GithubOrgEntityProvider implements EntityProvider { org, tokenType, this.options.userTransformer, - DEFAULT_PAGE_SIZES, + pageSizes, this.options.excludeSuspendedUsers, ); @@ -395,6 +414,7 @@ export class GithubOrgEntityProvider implements EntityProvider { org, usersToRebuild.map(u => u.metadata.name), this.options.teamTransformer, + pageSizes, ); if (areGroupEntities(teams)) { @@ -458,6 +478,7 @@ export class GithubOrgEntityProvider implements EntityProvider { }); const { org } = parseGithubOrgUrl(this.options.orgUrl); + const pageSizes = this.getPageSizes(); const { team } = await getOrganizationTeam( client, org, @@ -470,7 +491,7 @@ export class GithubOrgEntityProvider implements EntityProvider { org, tokenType, this.options.userTransformer, - DEFAULT_PAGE_SIZES, + pageSizes, this.options.excludeSuspendedUsers, ); @@ -481,6 +502,7 @@ export class GithubOrgEntityProvider implements EntityProvider { org, [userLogin], this.options.teamTransformer, + pageSizes, ); // we include group because the removed event need to update the old group too