paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 34 Packages Projects Releases Wiki Activity

Merge pull request #31474 from abhishekbvs/feat/github-plugin-api-page-sizes

Browse Source 
feat: add configurable GitHub API page sizes
This commit is contained in:
Fredrik Adelöw
2025-11-04 15:22:17 +01:00
committed by GitHub
parent 7a26c0947e ce2ad7f718
commit 037536c627
16 changed files with 595 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/pretty-kids-allow.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Added configurable `pageSizes` for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with organizations with large number of repositories. Please see the [GitHub Discovery documentation](https://backstage.io/docs/integrations/github/discovery#configuration) for new configuration options.
.changeset/tough-sloths-spend.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github-org': patch
---
Added configurable `pageSizes` for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors with organizations with large number of teams and members. Please see the [GitHub Org Data documentation](https://backstage.io/docs/integrations/github/org#configuration-details) for new configuration options.
docs/integrations/github/discovery.md
+6
View File
@@ -210,6 +210,8 @@ catalog:
filters: # optional filters
branch: 'develop' # optional string
repository: '.*' # optional Regex
pageSizes:
repositories: 25
wildcardProviderId:
organization: 'new-org' # string
catalogPath: '/groups/**/*.yaml' # this will search all folders for files that end in .yaml
@@ -308,6 +310,10 @@ If you do so, `default` will be used as provider ID.
The amount of time that should pass before the first invocation happens.
- **`scope`** _(optional)_:
`'global'` or `'local'`. Sets the scope of concurrency control.
- **`pageSizes`** _(optional)_:
Configure page sizes for GitHub GraphQL API queries. This can help prevent `RESOURCE_LIMITS_EXCEEDED` errors.
- **`repositories`** _(optional)_:
Number of repositories to fetch per page. Defaults to `25`. Reduce this value if hitting API resource limits.
## GitHub API Rate Limits
docs/integrations/github/org.md
+11
View File
@@ -79,6 +79,10 @@ catalog:
initialDelay: { seconds: 30 }
frequency: { hours: 1 }
timeout: { minutes: 50 }
pageSizes:
teams: 25
teamMembers: 50
organizationMembers: 50
- id: ghe
githubUrl: https://ghe.mycompany.com
orgs: ['internal-1', 'internal-2', 'internal-3']
@@ -94,6 +98,13 @@ Directly under the `githubOrg` is a list of configurations, each entry is a stru
- `githubUrl`: The target that this provider should consume
- `orgs` (optional): The list of the GitHub orgs to consume. If you only list a single org the generated group entities will use the `default` namespace, otherwise they will use the org name as the namespace. By default the provider will consume all accessible orgs on the given GitHub instance (support for GitHub App integration only).
- `schedule`: The refresh schedule to use, matches the structure of [`SchedulerServiceTaskScheduleDefinitionConfig`](https://backstage.io/docs/reference/backend-plugin-api.schedulerservicetaskscheduledefinitionconfig/)
- `pageSizes` (optional): Configure page sizes for GitHub GraphQL API queries to prevent `RESOURCE_LIMITS_EXCEEDED` errors. You can configure the following page sizes:
- `teams`: Number of teams to fetch per page when querying organization teams (default: 25)
- `teamMembers`: Number of team members to fetch per page when querying team members (default: 50)
- `organizationMembers`: Number of organization members to fetch per page (default: 50)
Reducing page sizes will result in more API calls and slightly longer sync times, but will prevent API resource limits for organizations with large number of teams and members.
### Events Support
plugins/catalog-backend-module-github-org/src/module.test.ts
+101
View File
@@ -73,4 +73,105 @@ describe('catalogModuleGithubOrgEntityProvider', () => {
);
expect(runner).not.toHaveBeenCalled();
});
it('should register provider with custom page sizes', async () => {
let addedProviders: Array<EntityProvider> | undefined;
const extensionPoint = {
addEntityProvider: (...providers: any) => {
addedProviders = providers;
},
};
const runner = jest.fn();
const scheduler = mockServices.scheduler.mock({
createScheduledTaskRunner() {
return { run: runner };
},
});
const config = {
catalog: {
providers: {
githubOrg: [
{
id: 'default',
githubUrl: 'https://github.com',
orgs: ['backstage'],
schedule: {
frequency: 'P1M',
timeout: 'PT3M',
},
pageSizes: {
teams: 10,
teamMembers: 25,
organizationMembers: 30,
},
},
],
},
},
};
await startTestBackend({
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
features: [
catalogModuleGithubOrgEntityProvider,
mockServices.rootConfig.factory({ data: config }),
scheduler.factory,
],
});
expect(addedProviders?.length).toEqual(1);
expect(addedProviders![0].getProviderName()).toEqual(
'GithubMultiOrgEntityProvider:default',
);
});
it('should register provider without page sizes configuration', async () => {
let addedProviders: Array<EntityProvider> | undefined;
const extensionPoint = {
addEntityProvider: (...providers: any) => {
addedProviders = providers;
},
};
const runner = jest.fn();
const scheduler = mockServices.scheduler.mock({
createScheduledTaskRunner() {
return { run: runner };
},
});
const config = {
catalog: {
providers: {
githubOrg: [
{
id: 'default',
githubUrl: 'https://github.com',
orgs: ['backstage'],
schedule: {
frequency: 'P1M',
timeout: 'PT3M',
},
},
],
},
},
};
await startTestBackend({
extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
features: [
catalogModuleGithubOrgEntityProvider,
mockServices.rootConfig.factory({ data: config }),
scheduler.factory,
],
});
expect(addedProviders?.length).toEqual(1);
expect(addedProviders![0].getProviderName()).toEqual(
'GithubMultiOrgEntityProvider:default',
);
});
});
plugins/catalog-backend-module-github-org/src/module.ts
+15
View File
@@ -120,6 +120,7 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({
teamTransformer,
alwaysUseDefaultNamespace:
definitions.length === 1 && definition.orgs?.length === 1,
pageSizes: definition.pageSizes,
}),
);
}
@@ -133,6 +134,11 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{
githubUrl: string;
orgs?: string[];
schedule: SchedulerServiceTaskScheduleDefinition;
pageSizes?: {
teams?: number;
teamMembers?: number;
organizationMembers?: number;
};
}> {
const baseKey = 'catalog.providers.githubOrg';
const baseConfig = rootConfig.getOptional(baseKey);
@@ -151,5 +157,14 @@ function readDefinitionsFromConfig(rootConfig: Config): Array<{
schedule: readSchedulerServiceTaskScheduleDefinitionFromConfig(
c.getConfig('schedule'),
),
pageSizes: c.has('pageSizes')
? {
teams: c.getOptionalNumber('pageSizes.teams'),
teamMembers: c.getOptionalNumber('pageSizes.teamMembers'),
organizationMembers: c.getOptionalNumber(
'pageSizes.organizationMembers',
),
}
: undefined,
}));
}
plugins/catalog-backend-module-github/config.d.ts
Vendored
+68
View File
@@ -131,6 +131,18 @@ export interface Config {
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
/**
* (Optional) Page sizes for GitHub GraphQL API queries.
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
*/
pageSizes?: {
/**
* (Optional) Number of repositories to fetch per page when querying repositories.
* Default: `25`.
*/
repositories?: number;
};
}
| {
[name: string]: {
@@ -209,6 +221,18 @@ export interface Config {
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
/**
* (Optional) Page sizes for GitHub GraphQL API queries.
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
*/
pageSizes?: {
/**
* (Optional) Number of repositories to fetch per page when querying repositories.
* Default: `25`.
*/
repositories?: number;
};
};
};
@@ -244,6 +268,28 @@ export interface Config {
* The refresh schedule to use.
*/
schedule: SchedulerServiceTaskScheduleDefinitionConfig;
/**
* (Optional) Page sizes for GitHub GraphQL API queries.
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
*/
pageSizes?: {
/**
* (Optional) Number of teams to fetch per page when querying organization teams.
* Default: `25`.
*/
teams?: number;
/**
* (Optional) Number of team members to fetch per page when querying team members.
* Default: `50`.
*/
teamMembers?: number;
/**
* (Optional) Number of organization members to fetch per page when querying org members.
* Default: `50`.
*/
organizationMembers?: number;
};
}
| Array<{
/**
@@ -273,6 +319,28 @@ export interface Config {
* The refresh schedule to use.
*/
schedule: SchedulerServiceTaskScheduleDefinitionConfig;
/**
* (Optional) Page sizes for GitHub GraphQL API queries.
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
*/
pageSizes?: {
/**
* (Optional) Number of teams to fetch per page when querying organization teams.
* Default: `25`.
*/
teams?: number;
/**
* (Optional) Number of team members to fetch per page when querying team members.
* Default: `50`.
*/
teamMembers?: number;
/**
* (Optional) Number of organization members to fetch per page when querying org members.
* Default: `50`.
*/
organizationMembers?: number;
};
}>;
};
};
plugins/catalog-backend-module-github/report.api.md
+10
View File
@@ -150,6 +150,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
userTransformer?: UserTransformer;
teamTransformer?: TeamTransformer;
alwaysUseDefaultNamespace?: boolean;
pageSizes?: Partial<GithubPageSizes>;
});
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
@@ -170,6 +171,7 @@ export interface GithubMultiOrgEntityProviderOptions {
id: string;
logger: LoggerService;
orgs?: string[];
pageSizes?: Partial<GithubPageSizes>;
schedule?: 'manual' | SchedulerServiceTaskRunner;
teamTransformer?: TeamTransformer;
userTransformer?: UserTransformer;
@@ -276,6 +278,14 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
): Promise<boolean>;
}
// @public
export type GithubPageSizes = {
teams: number;
teamMembers: number;
organizationMembers: number;
repositories: number;
};
// @public
export type GithubTeam = {
slug: string;
plugins/catalog-backend-module-github/src/index.ts
+1
View File
@@ -40,6 +40,7 @@ export {
type TeamTransformer,
defaultOrganizationTeamTransformer,
type TransformerContext,
type GithubPageSizes,
} from './lib';
export * from './deprecated';
plugins/catalog-backend-module-github/src/lib/github.test.ts
+140
View File
@@ -863,4 +863,144 @@ describe('github', () => {
});
});
});
describe('Page sizes configuration', () => {
const org = 'my-org';
it('uses custom page sizes for getOrganizationTeams', async () => {
server.use(
graphqlMsw.query('teams', ({ variables }) => {
expect(variables.teamsPageSize).toBe(10);
expect(variables.membersPageSize).toBe(20);
return HttpResponse.json({
data: {
organization: {
teams: {
pageInfo: { hasNextPage: false, endCursor: null },
nodes: [
{
slug: 'team1',
combinedSlug: 'my-org/team1',
name: 'Team 1',
description: 'desc',
avatarUrl: '',
editTeamUrl: '',
parentTeam: null,
members: {
pageInfo: { hasNextPage: false },
nodes: [{ login: 'user1' }],
},
},
],
},
},
},
});
}),
);
await getOrganizationTeams(graphql as any, org, undefined, {
teams: 10,
teamMembers: 20,
organizationMembers: 20,
repositories: 10,
});
});
it('uses custom page sizes for getOrganizationUsers', async () => {
server.use(
graphqlMsw.query('users', ({ variables }) => {
expect(variables.organizationMembersPageSize).toBe(30);
return HttpResponse.json({
data: {
organization: {
membersWithRole: {
pageInfo: { hasNextPage: false, endCursor: null },
nodes: [
{
login: 'user1',
name: 'User 1',
bio: '',
avatarUrl: '',
email: 'user1@example.com',
organizationVerifiedDomainEmails: [],
},
],
},
},
},
});
}),
);
await getOrganizationUsers(graphql as any, org, 'token', undefined, {
teams: 10,
teamMembers: 20,
organizationMembers: 30,
repositories: 10,
});
});
it('uses custom page sizes for getOrganizationRepositories', async () => {
server.use(
graphqlMsw.query('repositories', ({ variables }) => {
expect(variables.repositoriesPageSize).toBe(15);
return HttpResponse.json({
data: {
repositoryOwner: {
repositories: {
pageInfo: { hasNextPage: false, endCursor: null },
nodes: [
{
name: 'repo1',
url: 'https://github.com/my-org/repo1',
isArchived: false,
isFork: false,
visibility: 'public',
defaultBranchRef: { name: 'main' },
catalogInfoFile: null,
repositoryTopics: { nodes: [] },
},
],
},
},
},
});
}),
);
await getOrganizationRepositories(
graphql as any,
org,
'/catalog-info.yaml',
{
teams: 10,
teamMembers: 20,
organizationMembers: 30,
repositories: 15,
},
);
});
it('uses default page sizes when not specified', async () => {
server.use(
graphqlMsw.query('teams', ({ variables }) => {
expect(variables.teamsPageSize).toBe(25);
expect(variables.membersPageSize).toBe(50);
return HttpResponse.json({
data: {
organization: {
teams: {
pageInfo: { hasNextPage: false, endCursor: null },
nodes: [],
},
},
},
});
}),
);
await getOrganizationTeams(graphql as any, org);
});
});
});
plugins/catalog-backend-module-github/src/lib/github.ts
+109 -28
View File
@@ -30,6 +30,48 @@ import { DeferredEntity } from '@backstage/plugin-catalog-node';
import { Octokit } from '@octokit/core';
import { LoggerService } from '@backstage/backend-plugin-api';
import { throttling } from '@octokit/plugin-throttling';
/**
* Configuration for GitHub GraphQL API page sizes.
*
* @public
*/
export type GithubPageSizes = {
/**
* Number of teams to fetch per page when querying organization teams.
* Default: 25
*/
teams: number;
/**
* Number of team members to fetch per page when querying team members.
* Default: 50
*/
teamMembers: number;
/**
* Number of organization members to fetch per page when querying org members.
* Default: 50
*/
organizationMembers: number;
/**
* Number of repositories to fetch per page when querying repositories.
* Default: 25
*/
repositories: number;
};
/**
* Default page sizes for GitHub GraphQL API queries.
* These values are reduced to prevent RESOURCE_LIMITS_EXCEEDED errors with large organizations.
*
* @public
*/
export const DEFAULT_PAGE_SIZES: GithubPageSizes = {
teams: 25,
teamMembers: 50,
organizationMembers: 50,
repositories: 25,
};
// Graphql types
export type QueryResponse = {
@@ -136,17 +178,21 @@ export type Connection<T> = {
*
* @param client - An octokit graphql client
* @param org - The slug of the org to read
* @param tokenType - The type of GitHub credential
* @param userTransformer - Optional transformer for user entities
* @param pageSizes - Optional page sizes configuration
*/
export async function getOrganizationUsers(
client: typeof graphql,
org: string,
tokenType: GithubCredentialType,
userTransformer: UserTransformer = defaultUserTransformer,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{ users: Entity[] }> {
const query = `
query users($org: String!, $email: Boolean!, $cursor: String) {
query users($org: String!, $email: Boolean!, $cursor: String, $organizationMembersPageSize: Int!) {
organization(login: $org) {
membersWithRole(first: 100, after: $cursor) {
membersWithRole(first: $organizationMembersPageSize, after: $cursor) {
pageInfo { hasNextPage, endCursor }
nodes {
avatarUrl,
@@ -172,6 +218,7 @@ export async function getOrganizationUsers(
{
org,
email: tokenType === 'token',
organizationMembersPageSize: pageSizes.organizationMembers,
},
);
@@ -185,18 +232,21 @@ export async function getOrganizationUsers(
*
* @param client - An octokit graphql client
* @param org - The slug of the org to read
* @param teamTransformer - Optional transformer for team entities
* @param pageSizes - Optional page sizes configuration
*/
export async function getOrganizationTeams(
client: typeof graphql,
org: string,
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{
teams: Entity[];
}> {
const query = `
query teams($org: String!, $cursor: String) {
query teams($org: String!, $cursor: String, $teamsPageSize: Int!, $membersPageSize: Int!) {
organization(login: $org) {
teams(first: 50, after: $cursor) {
teams(first: $teamsPageSize, after: $cursor) {
pageInfo { hasNextPage, endCursor }
nodes {
slug
@@ -206,7 +256,7 @@ export async function getOrganizationTeams(
avatarUrl
editTeamUrl
parentTeam { slug }
members(first: 100, membership: IMMEDIATE) {
members(first: $membersPageSize, membership: IMMEDIATE) {
pageInfo { hasNextPage }
nodes {
avatarUrl,
@@ -234,9 +284,14 @@ export async function getOrganizationTeams(
memberNames.push(user);
}
} else {
// There were more than a hundred immediate members - run the slow
// There were more immediate members than page size - run the slow
// path of fetching them explicitly
const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
const { members } = await getTeamMembers(
ctx.client,
ctx.org,
item.slug,
pageSizes,
);
for (const userLogin of members) {
memberNames.push(userLogin);
}
@@ -256,7 +311,11 @@ export async function getOrganizationTeams(
org,
r => r.organization?.teams,
materialisedTeams,
{ org },
{
org,
teamsPageSize: pageSizes.teams,
membersPageSize: pageSizes.teamMembers,
},
);
return { teams };
@@ -267,13 +326,14 @@ export async function getOrganizationTeamsFromUsers(
org: string,
userLogins: string[],
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{
teams: Entity[];
}> {
const query = `
query teams($org: String!, $cursor: String, $userLogins: [String!] = "") {
query teams($org: String!, $cursor: String, $userLogins: [String!] = "", $teamsPageSize: Int!, $membersPageSize: Int!) {
organization(login: $org) {
teams(first: 100, after: $cursor, userLogins: $userLogins) {
teams(first: $teamsPageSize, after: $cursor, userLogins: $userLogins) {
pageInfo {
hasNextPage
endCursor
@@ -288,7 +348,7 @@ export async function getOrganizationTeamsFromUsers(
parentTeam {
slug
}
members(first: 100, membership: IMMEDIATE) {
members(first: $membersPageSize, membership: IMMEDIATE) {
pageInfo {
hasNextPage
}
@@ -318,9 +378,14 @@ export async function getOrganizationTeamsFromUsers(
memberNames.push(user);
}
} else {
// There were more than a hundred immediate members - run the slow
// There were more immediate members than page size - run the slow
// path of fetching them explicitly
const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
const { members } = await getTeamMembers(
ctx.client,
ctx.org,
item.slug,
pageSizes,
);
for (const userLogin of members) {
memberNames.push(userLogin);
}
@@ -340,7 +405,12 @@ export async function getOrganizationTeamsFromUsers(
org,
r => r.organization?.teams,
materialisedTeams,
{ org, userLogins },
{
org,
userLogins,
teamsPageSize: pageSizes.teams,
membersPageSize: pageSizes.teamMembers,
},
);
return { teams };
@@ -351,11 +421,12 @@ export async function getOrganizationTeamsForUser(
org: string,
userLogin: string,
teamTransformer: TeamTransformer,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{ teams: Entity[] }> {
const query = `
query teams($org: String!, $cursor: String, $userLogins: [String!] = "") {
query teams($org: String!, $cursor: String, $userLogins: [String!] = "", $teamsPageSize: Int!) {
organization(login: $org) {
teams(first: 100, after: $cursor, userLogins: $userLogins) {
teams(first: $teamsPageSize, after: $cursor, userLogins: $userLogins) {
pageInfo {
hasNextPage
endCursor
@@ -393,7 +464,7 @@ export async function getOrganizationTeamsForUser(
org,
r => r.organization?.teams,
materialisedTeams,
{ org, userLogins: [userLogin] },
{ org, userLogins: [userLogin], teamsPageSize: pageSizes.teams },
);
return { teams };
@@ -432,11 +503,12 @@ export async function getOrganizationTeam(
org: string,
teamSlug: string,
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{
team: Entity;
}> {
const query = `
query teams($org: String!, $teamSlug: String!) {
query teams($org: String!, $teamSlug: String!, $membersPageSize: Int!) {
organization(login: $org) {
team(slug:$teamSlug) {
slug
@@ -446,7 +518,7 @@ export async function getOrganizationTeam(
avatarUrl
editTeamUrl
parentTeam { slug }
members(first: 100, membership: IMMEDIATE) {
members(first: $membersPageSize, membership: IMMEDIATE) {
pageInfo { hasNextPage }
nodes { login }
}
@@ -466,9 +538,14 @@ export async function getOrganizationTeam(
memberNames.push(user);
}
} else {
// There were more than a hundred immediate members - run the slow
// There were more immediate members than page size - run the slow
// path of fetching them explicitly
const { members } = await getTeamMembers(ctx.client, ctx.org, item.slug);
const { members } = await getTeamMembers(
ctx.client,
ctx.org,
item.slug,
pageSizes,
);
for (const userLogin of members) {
memberNames.push(userLogin);
}
@@ -485,6 +562,7 @@ export async function getOrganizationTeam(
const response: QueryResponse = await client(query, {
org,
teamSlug,
membersPageSize: pageSizes.teamMembers,
});
if (!response.organization?.team)
@@ -505,6 +583,7 @@ export async function getOrganizationRepositories(
client: typeof graphql,
org: string,
catalogPath: string,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{ repositories: RepositoryResponse[] }> {
let relativeCatalogPathRef: string;
// We must strip the leading slash or the query for objects does not work
@@ -515,10 +594,10 @@ export async function getOrganizationRepositories(
}
const catalogPathRef = `HEAD:${relativeCatalogPathRef}`;
const query = `
query repositories($org: String!, $catalogPathRef: String!, $cursor: String) {
query repositories($org: String!, $catalogPathRef: String!, $cursor: String, $repositoriesPageSize: Int!) {
repositoryOwner(login: $org) {
login
repositories(first: 50, after: $cursor) {
repositories(first: $repositoriesPageSize, after: $cursor) {
nodes {
name
catalogInfoFile: object(expression: $catalogPathRef) {
@@ -559,7 +638,7 @@ export async function getOrganizationRepositories(
org,
r => r.repositoryOwner?.repositories,
async x => x,
{ org, catalogPathRef },
{ org, catalogPathRef, repositoriesPageSize: pageSizes.repositories },
);
return { repositories };
@@ -621,24 +700,26 @@ export async function getOrganizationRepository(
}
/**
* Gets all the users out of a Github organization.
* Gets all the users out of a Github organization team.
*
* Note that the users will not have their memberships filled in.
*
* @param client - An octokit graphql client
* @param org - The slug of the org to read
* @param teamSlug - The slug of the team to read
* @param pageSizes - Optional page sizes configuration
*/
export async function getTeamMembers(
client: typeof graphql,
org: string,
teamSlug: string,
pageSizes: GithubPageSizes = DEFAULT_PAGE_SIZES,
): Promise<{ members: GithubUser[] }> {
const query = `
query members($org: String!, $teamSlug: String!, $cursor: String) {
query members($org: String!, $teamSlug: String!, $cursor: String, $membersPageSize: Int!) {
organization(login: $org) {
team(slug: $teamSlug) {
members(first: 100, after: $cursor, membership: IMMEDIATE) {
members(first: $membersPageSize, after: $cursor, membership: IMMEDIATE) {
pageInfo { hasNextPage, endCursor }
nodes { login }
}
@@ -652,7 +733,7 @@ export async function getTeamMembers(
org,
r => r.organization?.team?.members,
async user => user,
{ org, teamSlug },
{ org, teamSlug, membersPageSize: pageSizes.teamMembers },
);
return { members };
plugins/catalog-backend-module-github/src/lib/index.ts
+2
View File
@@ -22,6 +22,8 @@ export {
getOrganizationUsers,
type GithubUser,
type GithubTeam,
type GithubPageSizes,
DEFAULT_PAGE_SIZES,
} from './github';
export {
type UserTransformer,
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts
+13 -1
View File
@@ -42,6 +42,8 @@ import {
getOrganizationRepositories,
getOrganizationRepository,
RepositoryResponse,
GithubPageSizes,
DEFAULT_PAGE_SIZES,
} from '../lib/github';
import {
satisfiesForkFilter,
@@ -262,8 +264,18 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
for (const organization of organizations) {
const client = await this.createGraphqlClient(organization);
const pageSizes: GithubPageSizes = {
...DEFAULT_PAGE_SIZES,
...this.config.pageSizes,
};
const { repositories: repositoriesFromGithub } =
await getOrganizationRepositories(client, organization, catalogPath);
await getOrganizationRepositories(
client,
organization,
catalogPath,
pageSizes,
);
repositories = repositories.concat(
repositoriesFromGithub.map(r =>
this.createRepoFromGithubResponse(r, organization),
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts
+64
View File
@@ -425,4 +425,68 @@ describe('readProviderConfigs', () => {
expect(() => readProviderConfigs(config)).toThrow();
});
it('reads page sizes configuration', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
pageSizes: {
repositories: 10,
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(1);
expect(providerConfigs[0].pageSizes).toEqual({
repositories: 10,
});
});
it('handles missing page sizes configuration', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(1);
expect(providerConfigs[0].pageSizes).toBeUndefined();
});
it('reads multiple providers with different page sizes', () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
providerWithPageSizes: {
organization: 'test-org1',
pageSizes: {
repositories: 15,
},
},
providerWithoutPageSizes: {
organization: 'test-org2',
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(2);
expect(providerConfigs[0].pageSizes).toEqual({
repositories: 15,
});
expect(providerConfigs[1].pageSizes).toBeUndefined();
});
});
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts
+10
View File
@@ -48,6 +48,9 @@ export type GithubEntityProviderConfig = {
};
validateLocationsExist: boolean;
schedule?: SchedulerServiceTaskScheduleDefinition;
pageSizes?: {
repositories?: number;
};
};
export type GithubTopicFilters = {
@@ -128,6 +131,12 @@ function readProviderConfig(
)
: DEFAULT_GITHUB_ENTITY_PROVIDER_CONFIG_SCHEDULE;
const pageSizes = config.has('pageSizes')
? {
repositories: config.getOptionalNumber('pageSizes.repositories'),
}
: undefined;
return {
id,
catalogPath,
@@ -149,6 +158,7 @@ function readProviderConfig(
},
schedule,
validateLocationsExist,
pageSizes,
};
}
plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts
+35
View File
@@ -66,6 +66,8 @@ import {
TeamTransformer,
TransformerContext,
UserTransformer,
GithubPageSizes,
DEFAULT_PAGE_SIZES,
} from '../lib';
import {
ANNOTATION_GITHUB_TEAM_SLUG,
@@ -166,6 +168,12 @@ export interface GithubMultiOrgEntityProviderOptions {
* By default, groups will be namespaced according to their GitHub org.
*/
teamTransformer?: TeamTransformer;
/**
* Optionally configure page sizes for GitHub GraphQL API queries.
* Reduce these values if hitting RESOURCE_LIMITS_EXCEEDED errors.
*/
pageSizes?: Partial<GithubPageSizes>;
}
type CreateDeltaOperation = (entities: Entity[]) => {
@@ -212,6 +220,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
teamTransformer: options.teamTransformer,
events: options.events,
alwaysUseDefaultNamespace: options.alwaysUseDefaultNamespace,
pageSizes: options.pageSizes,
});
provider.schedule(options.schedule);
@@ -231,6 +240,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
userTransformer?: UserTransformer;
teamTransformer?: TeamTransformer;
alwaysUseDefaultNamespace?: boolean;
pageSizes?: Partial<GithubPageSizes>;
},
) {}
@@ -239,6 +249,13 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
return `GithubMultiOrgEntityProvider:${this.options.id}`;
}
private getPageSizes(): GithubPageSizes {
return {
...DEFAULT_PAGE_SIZES,
...this.options.pageSizes,
};
}
/** {@inheritdoc @backstage/plugin-catalog-node#EntityProvider.connect} */
async connect(connection: EntityProviderConnection) {
this.connection = connection;
@@ -281,17 +298,21 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
logger.info(`Reading GitHub users and teams for org: ${org}`);
const pageSizes = this.getPageSizes();
const { users } = await getOrganizationUsers(
client,
org,
tokenType,
this.options.userTransformer,
pageSizes,
);
const { teams } = await getOrganizationTeams(
client,
org,
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
// Grab current users from `allUsersMap` if they already exist in our
@@ -429,17 +450,21 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers,
});
const pageSizes = this.getPageSizes();
const { users } = await getOrganizationUsers(
client,
org,
tokenType,
this.options.userTransformer,
pageSizes,
);
const { teams } = await getOrganizationTeams(
client,
org,
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
if (users.length) {
@@ -464,6 +489,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
u.metadata.name,
),
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
if (areGroupEntities(userTeams) && areUserEntities(users)) {
@@ -548,6 +574,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
}
if (updateMemberships) {
const pageSizes = this.getPageSizes();
for (const userOrg of userApplicableOrgs) {
const { headers: orgHeaders } =
await this.options.githubCredentialsProvider.getCredentials({
@@ -563,6 +590,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
userOrg,
login,
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
if (isUserEntity(user) && areGroupEntities(teams)) {
@@ -648,12 +676,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers,
});
const pageSizes = this.getPageSizes();
const teamSlug = event.team.slug;
const { team } = await getOrganizationTeam(
client,
org,
teamSlug,
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
const { users } = await getOrganizationUsers(
@@ -661,6 +691,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
org,
tokenType,
this.options.userTransformer,
pageSizes,
);
const usersFromChangedGroup = isGroupEntity(team)
@@ -693,6 +724,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
u.metadata.name,
),
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
if (areGroupEntities(teams) && areUserEntities(usersToRebuild)) {
@@ -761,12 +793,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers,
});
const pageSizes = this.getPageSizes();
const teamSlug = event.team.slug;
const { team } = await getOrganizationTeam(
client,
org,
teamSlug,
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
const userTransformer =
@@ -806,6 +840,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
userOrg,
login,
this.defaultMultiOrgTeamTransformer.bind(this),
pageSizes,
);
if (areGroupEntities(teams)) {