diff --git a/.changeset/seven-beans-appear.md b/.changeset/seven-beans-appear.md new file mode 100644 index 0000000000..82fe9f9889 --- /dev/null +++ b/.changeset/seven-beans-appear.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-gitlab': patch +--- + +Implement retry for GitLab API calls to handle rate limiting diff --git a/plugins/catalog-backend-module-gitlab/src/lib/client.ts b/plugins/catalog-backend-module-gitlab/src/lib/client.ts index 9053d68554..e58227839e 100644 --- a/plugins/catalog-backend-module-gitlab/src/lib/client.ts +++ b/plugins/catalog-backend-module-gitlab/src/lib/client.ts @@ -190,9 +190,8 @@ export class GitLabClient { let endCursor: string | null = null; do { - const response: GitLabDescendantGroupsResponse = await fetch( - `${this.config.baseUrl}/api/graphql`, - { + const response: GitLabDescendantGroupsResponse = + await this.fetchWithRetry(`${this.config.baseUrl}/api/graphql`, { method: 'POST', headers: { ...getGitLabRequestOptions(this.config).headers, @@ -223,8 +222,7 @@ export class GitLabClient { } `, }), - }, - ).then(r => r.json()); + }).then(r => r.json()); if (response.errors) { throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); } @@ -266,7 +264,7 @@ export class GitLabClient { let hasNextPage: boolean = false; let endCursor: string | null = null; do { - const response: GitLabGroupMembersResponse = await fetch( + const response: GitLabGroupMembersResponse = await this.fetchWithRetry( `${this.config.baseUrl}/api/graphql`, { method: 'POST', @@ -359,7 +357,7 @@ export class GitLabClient { const request = new URL(`${this.config.apiBaseUrl}${endpoint}`); request.searchParams.append('ref', branch); - const response = await fetch(request.toString(), { + const response = await this.fetchWithRetry(request.toString(), { headers: getGitLabRequestOptions(this.config).headers, method: 'HEAD', }); @@ -406,7 +404,7 @@ export class GitLabClient { } this.logger.debug(`Fetching: ${request.toString()}`); - const response = await fetch( + const response = await this.fetchWithRetry( request.toString(), getGitLabRequestOptions(this.config), ); @@ -444,7 +442,7 @@ export class GitLabClient { } } - const response = await fetch( + const response = await this.fetchWithRetry( request.toString(), getGitLabRequestOptions(this.config), ); @@ -459,6 +457,48 @@ export class GitLabClient { return response.json(); } + + /** + * Performs a fetch request with retry logic for rate limiting (429 errors) + * @param url - The URL to fetch + * @param options - Fetch options + * @param retries - Maximum number of retries + * @param initialBackoff - Initial backoff time in ms + */ + async fetchWithRetry( + url: string, + options: fetch.RequestInit, + retries = 5, + initialBackoff = 100, + ): Promise { + let currentRetry = 0; + let backoff = initialBackoff; + + for (;;) { + const response = await fetch(url, options); + + if (response.status !== 429 || currentRetry >= retries) { + return response; + } + + // Get retry-after header if available, or use exponential backoff + const retryAfter = response.headers.get('Retry-After'); + const waitTime = retryAfter ? parseInt(retryAfter, 10) * 1000 : backoff; + + this.logger.warn( + `GitLab API rate limit exceeded, retrying in ${waitTime}ms (retry ${ + currentRetry + 1 + }/${retries})`, + ); + + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, waitTime)); + + // Exponential backoff with jitter + backoff = backoff * 2 * (0.8 + Math.random() * 0.4); + currentRetry++; + } + } } /**