Merge pull request #30804 from JohannesWill/gitlab/throttle
Gitlab/throttle
This commit is contained in:
Fredrik Adelöw
Vendored
+22
@@ -386,6 +386,28 @@ export interface Config {
|
||||
* @visibility secret
|
||||
*/
|
||||
commitSigningKey?: string;
|
||||
|
||||
/**
|
||||
* Retry configuration for requests.
|
||||
* @visibility frontend
|
||||
*/
|
||||
retry?: {
|
||||
/**
|
||||
* Maximum number of retries for failed requests.
|
||||
* @visibility frontend
|
||||
*/
|
||||
maxRetries?: number;
|
||||
/**
|
||||
* HTTP status codes that should trigger a retry.
|
||||
* @visibility frontend
|
||||
*/
|
||||
retryStatusCodes?: number[];
|
||||
/**
|
||||
* Maximum number of API requests allowed per minute. Set to -1 to disable rate limiting.
|
||||
* @visibility frontend
|
||||
*/
|
||||
maxApiRequestsPerMinute?: number;
|
||||
};
|
||||
}>;
|
||||
|
||||
/** Integration configuration for Google Cloud Storage */
|
||||
|
||||
@@ -46,7 +46,8 @@
|
||||
"cross-fetch": "^4.0.0",
|
||||
"git-url-parse": "^15.0.0",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.0.0"
|
||||
"luxon": "^3.0.0",
|
||||
"p-throttle": "^4.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/cli": "workspace:^",
|
||||
|
||||
@@ -596,7 +596,7 @@ export function getGitilesAuthenticationUrl(
|
||||
export function getGitLabFileFetchUrl(
|
||||
url: string,
|
||||
config: GitLabIntegrationConfig,
|
||||
token?: string,
|
||||
_token?: string,
|
||||
): Promise<string>;
|
||||
|
||||
// @public
|
||||
@@ -759,6 +759,8 @@ export class GitLabIntegration implements ScmIntegration {
|
||||
// (undocumented)
|
||||
static factory: ScmIntegrationsFactory<GitLabIntegration>;
|
||||
// (undocumented)
|
||||
fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
|
||||
// (undocumented)
|
||||
resolveEditUrl(url: string): string;
|
||||
// (undocumented)
|
||||
resolveUrl(options: {
|
||||
@@ -779,6 +781,11 @@ export type GitLabIntegrationConfig = {
|
||||
token?: string;
|
||||
baseUrl: string;
|
||||
commitSigningKey?: string;
|
||||
retry?: {
|
||||
maxRetries?: number;
|
||||
retryStatusCodes?: number[];
|
||||
maxApiRequestsPerMinute?: number;
|
||||
};
|
||||
};
|
||||
|
||||
// @public
|
||||
|
||||
@@ -14,8 +14,20 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { setupServer } from 'msw/node';
|
||||
import { rest } from 'msw';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { GitLabIntegration, replaceGitLabUrlType } from './GitLabIntegration';
|
||||
import {
|
||||
GitLabIntegration,
|
||||
replaceGitLabUrlType,
|
||||
sleep,
|
||||
} from './GitLabIntegration';
|
||||
import { registerMswTestHooks } from '../helpers';
|
||||
|
||||
// Mock pThrottle to make testing easier
|
||||
jest.mock('p-throttle', () => {
|
||||
return jest.fn(() => (fn: any) => fn);
|
||||
});
|
||||
|
||||
describe('GitLabIntegration', () => {
|
||||
it('has a working factory', () => {
|
||||
@@ -45,7 +57,11 @@ describe('GitLabIntegration', () => {
|
||||
});
|
||||
|
||||
it('resolve edit URL', () => {
|
||||
const integration = new GitLabIntegration({ host: 'h.com' } as any);
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
});
|
||||
|
||||
expect(
|
||||
integration.resolveEditUrl(
|
||||
@@ -53,6 +69,349 @@ describe('GitLabIntegration', () => {
|
||||
),
|
||||
).toBe('https://gitlab.com/my-org/my-project/-/edit/develop/README.md');
|
||||
});
|
||||
|
||||
describe('fetch strategy', () => {
|
||||
const worker = setupServer();
|
||||
registerMswTestHooks(worker);
|
||||
|
||||
beforeAll(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
afterAll(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
beforeEach(() => {
|
||||
jest.clearAllTimers();
|
||||
});
|
||||
|
||||
it('uses plain fetch when no throttling or retries configured', async () => {
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
});
|
||||
|
||||
let calledUrl: string | undefined;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (req, res, ctx) => {
|
||||
calledUrl = req.url.href;
|
||||
return res(ctx.status(200));
|
||||
}),
|
||||
);
|
||||
|
||||
await integration.fetch('https://h.com/api/v4');
|
||||
expect(calledUrl).toBe('https://h.com/api/v4');
|
||||
});
|
||||
|
||||
it('applies retry logic when maxRetries > 0', async () => {
|
||||
let callCount = 0;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (_req, res, ctx) => {
|
||||
callCount += 1;
|
||||
if (callCount === 1) {
|
||||
return res(ctx.status(429), ctx.json({}));
|
||||
}
|
||||
return res(ctx.status(200), ctx.json({}));
|
||||
}),
|
||||
);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
retryStatusCodes: [429],
|
||||
},
|
||||
});
|
||||
const responsePromise = integration.fetch('https://h.com/api/v4');
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await jest.advanceTimersByTimeAsync(200);
|
||||
await jest.advanceTimersByTimeAsync(400);
|
||||
const response = await responsePromise;
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(callCount).toBe(2);
|
||||
});
|
||||
|
||||
it('does not retry when status code is not in retryStatusCodes', async () => {
|
||||
let callCount = 0;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (_req, res, ctx) => {
|
||||
callCount += 1;
|
||||
return res(ctx.status(404));
|
||||
}),
|
||||
);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
retryStatusCodes: [429, 500],
|
||||
},
|
||||
});
|
||||
|
||||
const responsePromise = integration.fetch('https://h.com/api/v4');
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
const response = await responsePromise;
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
expect(callCount).toBe(1);
|
||||
});
|
||||
|
||||
it('stops retrying after maxRetries attempts', async () => {
|
||||
let callCount = 0;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (_req, res, ctx) => {
|
||||
callCount += 1;
|
||||
return res(ctx.status(429));
|
||||
}),
|
||||
);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxRetries: 2,
|
||||
retryStatusCodes: [429],
|
||||
},
|
||||
});
|
||||
|
||||
const responsePromise = integration.fetch('https://h.com/api/v4');
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await jest.advanceTimersByTimeAsync(200);
|
||||
await jest.advanceTimersByTimeAsync(400);
|
||||
const response = await responsePromise;
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(callCount).toBe(3); // initial + 2 retries
|
||||
});
|
||||
|
||||
it('applies throttling when limitPerMinute > 0', async () => {
|
||||
const pThrottle = require('p-throttle');
|
||||
const throttleMock = jest.fn(() => (fn: any) => fn);
|
||||
pThrottle.mockReturnValue(throttleMock);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxApiRequestsPerMinute: 60,
|
||||
},
|
||||
});
|
||||
|
||||
await integration.fetch('https://h.com/api/v4');
|
||||
|
||||
expect(pThrottle).toHaveBeenCalledWith({
|
||||
limit: 60,
|
||||
interval: 60_000,
|
||||
});
|
||||
expect(throttleMock).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('applies both throttling and retry when both are configured', async () => {
|
||||
const pThrottle = require('p-throttle');
|
||||
|
||||
const throttleMock = jest.fn((fn: any) => fn);
|
||||
pThrottle.mockReturnValue(throttleMock);
|
||||
|
||||
let callCount = 0;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (_req, res, ctx) => {
|
||||
callCount += 1;
|
||||
if (callCount === 1) {
|
||||
return res(ctx.status(429), ctx.json({}));
|
||||
}
|
||||
return res(ctx.status(200), ctx.json({}));
|
||||
}),
|
||||
);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
host: 'h.com',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
retryStatusCodes: [429],
|
||||
maxApiRequestsPerMinute: 60,
|
||||
},
|
||||
});
|
||||
|
||||
const responsePromise = integration.fetch('https://h.com/api/v4');
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
const response = await responsePromise;
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(pThrottle).toHaveBeenCalledWith({
|
||||
limit: 60,
|
||||
interval: 60_000,
|
||||
});
|
||||
expect(callCount).toBe(2);
|
||||
});
|
||||
|
||||
it('retries based on configured status codes', async () => {
|
||||
let callCount = 0;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (_req, res, ctx) => {
|
||||
callCount += 1;
|
||||
if (callCount === 1) {
|
||||
return res(
|
||||
ctx.status(429),
|
||||
ctx.set('Retry-After', '1'),
|
||||
ctx.json({}),
|
||||
);
|
||||
}
|
||||
return res(ctx.status(200), ctx.json({}));
|
||||
}),
|
||||
);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
retryStatusCodes: [429],
|
||||
},
|
||||
});
|
||||
|
||||
const responsePromise = integration.fetch('https://h.com/api/v4');
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
const response = await responsePromise;
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(callCount).toBe(2);
|
||||
});
|
||||
|
||||
it('retries multiple times for persistent failures', async () => {
|
||||
let callCount = 0;
|
||||
worker.use(
|
||||
rest.get('https://h.com/api/v4', (_req, res, ctx) => {
|
||||
callCount += 1;
|
||||
if (callCount < 3) {
|
||||
return res(ctx.status(500), ctx.json({}));
|
||||
}
|
||||
return res(ctx.status(200), ctx.json({}));
|
||||
}),
|
||||
);
|
||||
|
||||
const integration = new GitLabIntegration({
|
||||
host: 'h.com',
|
||||
apiBaseUrl: 'https://h.com/api/v4',
|
||||
baseUrl: 'https://h.com',
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
retryStatusCodes: [500],
|
||||
},
|
||||
});
|
||||
|
||||
const responsePromise = integration.fetch('https://h.com/api/v4');
|
||||
await jest.advanceTimersByTimeAsync(100);
|
||||
await jest.advanceTimersByTimeAsync(200);
|
||||
await jest.advanceTimersByTimeAsync(400);
|
||||
const response = await responsePromise;
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(callCount).toBe(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('sleep', () => {
|
||||
beforeAll(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
afterAll(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
beforeEach(() => {
|
||||
jest.clearAllTimers();
|
||||
});
|
||||
|
||||
it('should resolve after the specified duration when not aborted', async () => {
|
||||
const duration = 1000;
|
||||
const sleepPromise = sleep(duration, null);
|
||||
|
||||
// Fast-forward timers to trigger the timeout
|
||||
jest.advanceTimersByTimeAsync(duration);
|
||||
|
||||
await expect(sleepPromise).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should resolve immediately if abortSignal is already aborted', async () => {
|
||||
const abortController = new AbortController();
|
||||
abortController.abort();
|
||||
|
||||
const sleepPromise = sleep(5000, abortController.signal);
|
||||
|
||||
// Should resolve immediately without needing to advance timers
|
||||
await expect(sleepPromise).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should resolve when aborted during wait', async () => {
|
||||
const abortController = new AbortController();
|
||||
const duration = 5000;
|
||||
|
||||
const sleepPromise = sleep(duration, abortController.signal);
|
||||
|
||||
// Abort the signal after starting the sleep
|
||||
abortController.abort();
|
||||
|
||||
// Should resolve immediately when aborted
|
||||
await expect(sleepPromise).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should handle undefined abortSignal gracefully', async () => {
|
||||
const duration = 500;
|
||||
const sleepPromise = sleep(duration, undefined);
|
||||
|
||||
// Fast-forward timers to trigger the timeout
|
||||
jest.advanceTimersByTimeAsync(duration);
|
||||
|
||||
await expect(sleepPromise).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('should clean up timeout when aborted', async () => {
|
||||
const abortController = new AbortController();
|
||||
const duration = 10000;
|
||||
|
||||
const sleepPromise = sleep(duration, abortController.signal);
|
||||
|
||||
// Check that a timer was set
|
||||
expect(jest.getTimerCount()).toBe(1);
|
||||
|
||||
// Abort the signal
|
||||
abortController.abort();
|
||||
|
||||
// Wait for the promise to resolve
|
||||
await sleepPromise;
|
||||
|
||||
// Timer should be cleaned up
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('should clean up abort event listener when timeout completes', async () => {
|
||||
const abortController = new AbortController();
|
||||
const duration = 1000;
|
||||
|
||||
const sleepPromise = sleep(duration, abortController.signal);
|
||||
|
||||
// Fast-forward timers to complete the timeout
|
||||
jest.advanceTimersByTimeAsync(duration);
|
||||
|
||||
// Wait for the promise to complete and verify it resolves properly
|
||||
await expect(sleepPromise).resolves.toBeUndefined();
|
||||
|
||||
// Event listener should be cleaned up - aborting after completion should not cause issues
|
||||
abortController.abort(); // This should not affect anything since the sleep is already done
|
||||
|
||||
// Verify the sleep function handled cleanup properly
|
||||
expect(jest.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('replaceGitLabUrlType', () => {
|
||||
|
||||
@@ -13,13 +13,15 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
|
||||
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
|
||||
import {
|
||||
GitLabIntegrationConfig,
|
||||
readGitLabIntegrationConfigs,
|
||||
} from './config';
|
||||
import pThrottle from 'p-throttle';
|
||||
|
||||
type FetchFunction = typeof fetch;
|
||||
|
||||
/**
|
||||
* A GitLab based integration.
|
||||
@@ -37,7 +39,12 @@ export class GitLabIntegration implements ScmIntegration {
|
||||
);
|
||||
};
|
||||
|
||||
constructor(private readonly integrationConfig: GitLabIntegrationConfig) {}
|
||||
private readonly fetchImpl: FetchFunction;
|
||||
|
||||
constructor(private readonly integrationConfig: GitLabIntegrationConfig) {
|
||||
// Configure fetch strategy based on configuration
|
||||
this.fetchImpl = this.createFetchStrategy();
|
||||
}
|
||||
|
||||
get type(): string {
|
||||
return 'gitlab';
|
||||
@@ -62,6 +69,97 @@ export class GitLabIntegration implements ScmIntegration {
|
||||
resolveEditUrl(url: string): string {
|
||||
return replaceGitLabUrlType(url, 'edit');
|
||||
}
|
||||
|
||||
fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
|
||||
return this.fetchImpl(input, init);
|
||||
}
|
||||
|
||||
private createFetchStrategy(): FetchFunction {
|
||||
let fetchFn: FetchFunction = async (url, options) => {
|
||||
return fetch(url, { ...options, mode: 'same-origin' });
|
||||
};
|
||||
|
||||
const retryConfig = this.integrationConfig.retry;
|
||||
if (retryConfig) {
|
||||
// Apply retry wrapper if configured
|
||||
fetchFn = this.withRetry(fetchFn, retryConfig);
|
||||
|
||||
// Apply throttling wrapper if configured
|
||||
if (
|
||||
retryConfig.maxApiRequestsPerMinute &&
|
||||
retryConfig.maxApiRequestsPerMinute > 0
|
||||
) {
|
||||
fetchFn = pThrottle({
|
||||
limit: retryConfig.maxApiRequestsPerMinute,
|
||||
interval: 60_000,
|
||||
})(fetchFn);
|
||||
}
|
||||
}
|
||||
|
||||
return fetchFn;
|
||||
}
|
||||
|
||||
private withRetry(
|
||||
fetchFn: FetchFunction,
|
||||
retryConfig: { maxRetries?: number; retryStatusCodes?: number[] },
|
||||
): FetchFunction {
|
||||
const maxRetries = retryConfig?.maxRetries ?? 0;
|
||||
const retryStatusCodes = retryConfig?.retryStatusCodes ?? [];
|
||||
if (maxRetries <= 0 || retryStatusCodes.length === 0) {
|
||||
return fetchFn;
|
||||
}
|
||||
|
||||
return async (url, options) => {
|
||||
const abortSignal = options?.signal;
|
||||
let response: Response;
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
response = await fetchFn(url, options);
|
||||
// If response is not retryable, return immediately
|
||||
if (!retryStatusCodes.includes(response.status)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// If this was the last allowed attempt, return response
|
||||
if (attempt++ >= maxRetries) {
|
||||
break;
|
||||
}
|
||||
// Determine delay from Retry-After header if present, otherwise exponential backoff
|
||||
const retryAfter = response.headers.get('Retry-After');
|
||||
const delay = retryAfter
|
||||
? parseInt(retryAfter, 10) * 1000
|
||||
: Math.min(100 * Math.pow(2, attempt - 1), 10000); // Exponential backoff, cap at 10 seconds
|
||||
|
||||
await sleep(delay, abortSignal);
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function sleep(
|
||||
durationMs: number,
|
||||
abortSignal: AbortSignal | null | undefined,
|
||||
): Promise<void> {
|
||||
if (abortSignal?.aborted) {
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>(resolve => {
|
||||
let timeoutHandle: NodeJS.Timeout | undefined = undefined;
|
||||
|
||||
const done = () => {
|
||||
if (timeoutHandle) {
|
||||
clearTimeout(timeoutHandle);
|
||||
}
|
||||
abortSignal?.removeEventListener('abort', done);
|
||||
resolve();
|
||||
};
|
||||
|
||||
timeoutHandle = setTimeout(done, durationMs);
|
||||
abortSignal?.addEventListener('abort', done);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -58,6 +58,11 @@ describe('readGitLabIntegrationConfig', () => {
|
||||
token: ' t\n',
|
||||
apiBaseUrl: 'https://a.com',
|
||||
baseUrl: 'https://baseurl.for.me/gitlab',
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
maxApiRequestsPerMinute: 1000,
|
||||
retryStatusCodes: [429],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -66,6 +71,12 @@ describe('readGitLabIntegrationConfig', () => {
|
||||
token: 't',
|
||||
apiBaseUrl: 'https://a.com',
|
||||
baseUrl: 'https://baseurl.for.me/gitlab',
|
||||
commitSigningKey: undefined,
|
||||
retry: {
|
||||
maxRetries: 3,
|
||||
maxApiRequestsPerMinute: 1000,
|
||||
retryStatusCodes: [429],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -77,6 +88,8 @@ describe('readGitLabIntegrationConfig', () => {
|
||||
host: 'gitlab.com',
|
||||
apiBaseUrl: 'https://gitlab.com/api/v4',
|
||||
baseUrl: 'https://gitlab.com',
|
||||
commitSigningKey: undefined,
|
||||
retry: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -89,6 +102,7 @@ describe('readGitLabIntegrationConfig', () => {
|
||||
host: 'gitlab.com',
|
||||
baseUrl: 'https://gitlab.com',
|
||||
apiBaseUrl: 'https://gitlab.com/api/v4',
|
||||
retry: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -119,6 +133,9 @@ describe('readGitLabIntegrationConfig', () => {
|
||||
host: 'a.com',
|
||||
apiBaseUrl: 'https://a.com/api',
|
||||
baseUrl: 'https://a.com',
|
||||
token: undefined, // token is filtered out on frontend
|
||||
commitSigningKey: undefined,
|
||||
retry: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -144,6 +161,7 @@ describe('readGitLabIntegrationConfigs', () => {
|
||||
token: 't',
|
||||
apiBaseUrl: 'https://a.com/api/v4',
|
||||
baseUrl: 'https://a.com',
|
||||
retry: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -21,6 +21,32 @@ import { isValidHost, isValidUrl } from '../helpers';
|
||||
const GITLAB_HOST = 'gitlab.com';
|
||||
const GITLAB_API_BASE_URL = 'https://gitlab.com/api/v4';
|
||||
|
||||
/**
|
||||
* Reads an optional number array from config
|
||||
*/
|
||||
function readOptionalNumberArray(
|
||||
config: Config,
|
||||
key: string,
|
||||
): number[] | undefined {
|
||||
const value = config.getOptional(key);
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (!Array.isArray(value)) {
|
||||
throw new Error(
|
||||
`Invalid ${key} config: expected an array, got ${typeof value}`,
|
||||
);
|
||||
}
|
||||
return value.map((item, index) => {
|
||||
if (typeof item !== 'number') {
|
||||
throw new Error(
|
||||
`Invalid ${key} config: all values must be numbers, got ${typeof item} at index ${index}`,
|
||||
);
|
||||
}
|
||||
return item;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The configuration parameters for a single GitLab integration.
|
||||
*
|
||||
@@ -59,6 +85,29 @@ export type GitLabIntegrationConfig = {
|
||||
* Signing key to sign commits
|
||||
*/
|
||||
commitSigningKey?: string;
|
||||
|
||||
/**
|
||||
* Retry configuration for failed requests.
|
||||
*/
|
||||
retry?: {
|
||||
/**
|
||||
* Maximum number of retries for failed requests
|
||||
* @defaultValue 0
|
||||
*/
|
||||
maxRetries?: number;
|
||||
|
||||
/**
|
||||
* HTTP status codes that should trigger a retry
|
||||
* @defaultValue []
|
||||
*/
|
||||
retryStatusCodes?: number[];
|
||||
|
||||
/**
|
||||
* Rate limit for requests per minute
|
||||
* @defaultValue -1
|
||||
*/
|
||||
maxApiRequestsPerMinute?: number;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -100,12 +149,25 @@ export function readGitLabIntegrationConfig(
|
||||
);
|
||||
}
|
||||
|
||||
const retryConfig = config.getOptionalConfig('retry');
|
||||
|
||||
const retry = retryConfig
|
||||
? {
|
||||
maxRetries: retryConfig.getOptionalNumber('maxRetries') ?? 0,
|
||||
retryStatusCodes:
|
||||
readOptionalNumberArray(retryConfig, 'retryStatusCodes') ?? [],
|
||||
maxApiRequestsPerMinute:
|
||||
retryConfig.getOptionalNumber('maxApiRequestsPerMinute') ?? -1,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
host,
|
||||
token,
|
||||
apiBaseUrl,
|
||||
baseUrl,
|
||||
commitSigningKey: config.getOptionalString('commitSigningKey'),
|
||||
retry,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -14,29 +14,14 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { GitLabIntegrationConfig } from './config';
|
||||
import { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core';
|
||||
|
||||
const worker = setupServer();
|
||||
import {
|
||||
getGitLabFileFetchUrl,
|
||||
getGitLabRequestOptions,
|
||||
extractProjectPath,
|
||||
} from './core';
|
||||
|
||||
describe('gitlab core', () => {
|
||||
beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
|
||||
afterAll(() => worker.close());
|
||||
afterEach(() => worker.resetHandlers());
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/group%2Fproject', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
rest.get('*/api/v4/projects/group%2Fsubgroup%2Fproject', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
const configWithNoToken: GitLabIntegrationConfig = {
|
||||
host: 'gitlab.com',
|
||||
apiBaseUrl: '<ignored>',
|
||||
@@ -63,7 +48,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/-/blob/branch/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -73,7 +58,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/-/blob/branch/blob/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/blob%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fproject/repository/files/blob%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -83,7 +68,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/subgroup/project/-/blob/branch/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fsubgroup%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -93,7 +78,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/-/blob/branch/folder/file.yml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile.yml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile.yml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -103,7 +88,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/-/blob/branch/folder/file with spaces.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile%20with%20spaces.yaml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile%20with%20spaces.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -114,7 +99,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.mycompany.com/group/project/-/blob/branch/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.mycompany.com/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.mycompany.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configSelfHostedWithoutRelativePath),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -124,7 +109,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.mycompany.com/group/project/-/blob/branch/folder/file with spaces.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.mycompany.com/api/v4/projects/12345/repository/files/folder%2Ffile%20with%20spaces.yaml/raw?ref=branch';
|
||||
'https://gitlab.mycompany.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile%20with%20spaces.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configSelfHostedWithoutRelativePath),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -135,7 +120,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.mycompany.com/gitlab/group/project/-/blob/branch/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.mycompany.com/gitlab/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configSelfHosteWithRelativePath),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -145,7 +130,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.mycompany.com/gitlab/group/project/-/blob/branch/folder/file with spaces.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/folder%2Ffile%20with%20spaces.yaml/raw?ref=branch';
|
||||
'https://gitlab.mycompany.com/gitlab/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile%20with%20spaces.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configSelfHosteWithRelativePath),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -159,7 +144,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/blob/branch/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -169,7 +154,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/subgroup/project/blob/branch/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fsubgroup%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=branch';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -179,7 +164,7 @@ describe('gitlab core', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/blob/blob/folder/file.yaml';
|
||||
const fetchUrl =
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/folder%2Ffile.yaml/raw?ref=blob';
|
||||
'https://gitlab.com/api/v4/projects/group%2Fproject/repository/files/folder%2Ffile.yaml/raw?ref=blob';
|
||||
await expect(
|
||||
getGitLabFileFetchUrl(target, configWithNoToken),
|
||||
).resolves.toBe(fetchUrl);
|
||||
@@ -187,8 +172,49 @@ describe('gitlab core', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractProjectPath', () => {
|
||||
it('extracts project path from scoped route', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/-/blob/branch/folder/file.yaml';
|
||||
expect(extractProjectPath(target, configWithNoToken)).toBe(
|
||||
'group/project',
|
||||
);
|
||||
});
|
||||
|
||||
it('extracts project path from subgroup', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/subgroup/project/-/blob/branch/folder/file.yaml';
|
||||
expect(extractProjectPath(target, configWithNoToken)).toBe(
|
||||
'group/subgroup/project',
|
||||
);
|
||||
});
|
||||
|
||||
it('extracts project path from unscoped route', () => {
|
||||
const target =
|
||||
'https://gitlab.com/group/project/blob/branch/folder/file.yaml';
|
||||
expect(extractProjectPath(target, configWithNoToken)).toBe(
|
||||
'group/project',
|
||||
);
|
||||
});
|
||||
|
||||
it('extracts project path from self-hosted gitlab with relative path', () => {
|
||||
const target =
|
||||
'https://gitlab.mycompany.com/gitlab/group/project/-/blob/branch/folder/file.yaml';
|
||||
expect(extractProjectPath(target, configSelfHosteWithRelativePath)).toBe(
|
||||
'group/project',
|
||||
);
|
||||
});
|
||||
|
||||
it('throws error for invalid URLs without blob path', () => {
|
||||
const target = 'https://gitlab.com/some/random/endpoint';
|
||||
expect(() => extractProjectPath(target, configWithNoToken)).toThrow(
|
||||
'Failed extracting project path from /some/random/endpoint. Url path must include /blob/.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGitLabRequestOptions', () => {
|
||||
it('should return Authorization bearer header when a token is provided', () => {
|
||||
it('should return Authorization bearer header when a token is provided', async () => {
|
||||
const token = '1234567890';
|
||||
const result = getGitLabRequestOptions(
|
||||
configSelfHosteWithRelativePath,
|
||||
@@ -202,7 +228,7 @@ describe('gitlab core', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should return Authorization bearer header using the config token when no token is provided', () => {
|
||||
it('should return Authorization bearer header using the config token when no token is provided', async () => {
|
||||
const result = getGitLabRequestOptions(configSelfHosteWithRelativePath);
|
||||
|
||||
expect(result).toEqual({
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import fetch from 'cross-fetch';
|
||||
import {
|
||||
getGitLabIntegrationRelativePath,
|
||||
GitLabIntegrationConfig,
|
||||
@@ -28,22 +27,25 @@ import {
|
||||
*
|
||||
* Converts
|
||||
* from: https://gitlab.example.com/a/b/blob/master/c.yaml
|
||||
* to: https://gitlab.com/api/v4/projects/projectId/repository/c.yaml?ref=master
|
||||
* to: https://gitlab.com/api/v4/projects/a%2Fb/repository/files/c.yaml/raw?ref=master
|
||||
* -or-
|
||||
* from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
* to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch
|
||||
* to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FteamA%2FsubgroupA%2FrepoA/repository/files/filepath/raw?ref=branch
|
||||
*
|
||||
* @param url - A URL pointing to a file
|
||||
* @param config - The relevant provider config
|
||||
* @param token - An optional auth token (not used in path extraction, kept for compatibility)
|
||||
* @public
|
||||
*/
|
||||
export async function getGitLabFileFetchUrl(
|
||||
export function getGitLabFileFetchUrl(
|
||||
url: string,
|
||||
config: GitLabIntegrationConfig,
|
||||
token?: string,
|
||||
_token?: string,
|
||||
): Promise<string> {
|
||||
const projectID = await getProjectId(url, config, token);
|
||||
return buildProjectUrl(url, projectID, config).toString();
|
||||
// Use project path directly instead of making an API call to get project ID
|
||||
// Note: _token parameter kept for backward compatibility but not used for path extraction
|
||||
const projectPath = extractProjectPath(url, config);
|
||||
return Promise.resolve(buildProjectUrl(url, projectPath, config).toString());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -72,10 +74,10 @@ export function getGitLabRequestOptions(
|
||||
|
||||
// Converts
|
||||
// from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
// to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch
|
||||
// to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FteamA%2FsubgroupA%2FrepoA/repository/files/filepath/raw?ref=branch
|
||||
export function buildProjectUrl(
|
||||
target: string,
|
||||
projectID: Number,
|
||||
projectPathOrID: string | Number,
|
||||
config: GitLabIntegrationConfig,
|
||||
): URL {
|
||||
try {
|
||||
@@ -88,10 +90,12 @@ export function buildProjectUrl(
|
||||
const [branch, ...filePath] = branchAndFilePath.split('/');
|
||||
const relativePath = getGitLabIntegrationRelativePath(config);
|
||||
|
||||
const projectIdentifier = encodeURIComponent(String(projectPathOrID));
|
||||
|
||||
url.pathname = [
|
||||
...(relativePath ? [relativePath] : []),
|
||||
'api/v4/projects',
|
||||
projectID,
|
||||
projectIdentifier,
|
||||
'repository/files',
|
||||
encodeURIComponent(decodeURIComponent(filePath.join('/'))),
|
||||
'raw',
|
||||
@@ -105,62 +109,33 @@ export function buildProjectUrl(
|
||||
}
|
||||
}
|
||||
|
||||
// Convert
|
||||
// from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
// to: The project ID that corresponds to the URL
|
||||
export async function getProjectId(
|
||||
/**
|
||||
* Extracts the project path from a GitLab URL
|
||||
* from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
* to: groupA/teams/teamA/subgroupA/repoA
|
||||
*/
|
||||
export function extractProjectPath(
|
||||
target: string,
|
||||
config: GitLabIntegrationConfig,
|
||||
token?: string,
|
||||
): Promise<number> {
|
||||
): string {
|
||||
const url = new URL(target);
|
||||
|
||||
if (!url.pathname.includes('/blob/')) {
|
||||
throw new Error(
|
||||
`Failed converting ${url.pathname} to a project id. Url path must include /blob/.`,
|
||||
`Failed extracting project path from ${url.pathname}. Url path must include /blob/.`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
let repo = url.pathname.split('/-/blob/')[0].split('/blob/')[0];
|
||||
let repo = url.pathname.split('/-/blob/')[0].split('/blob/')[0];
|
||||
|
||||
// Get gitlab relative path
|
||||
const relativePath = getGitLabIntegrationRelativePath(config);
|
||||
// Get gitlab relative path
|
||||
const relativePath = getGitLabIntegrationRelativePath(config);
|
||||
|
||||
// Check relative path exist and replace it if it's the case.
|
||||
if (relativePath) {
|
||||
repo = repo.replace(relativePath, '');
|
||||
}
|
||||
|
||||
// Convert
|
||||
// to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo
|
||||
const repoIDLookup = new URL(
|
||||
`${url.origin}${relativePath}/api/v4/projects/${encodeURIComponent(
|
||||
repo.replace(/^\//, ''),
|
||||
)}`,
|
||||
);
|
||||
|
||||
const response = await fetch(
|
||||
repoIDLookup.toString(),
|
||||
getGitLabRequestOptions(config, token),
|
||||
);
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) {
|
||||
throw new Error(
|
||||
'GitLab Error: 401 - Unauthorized. The access token used is either expired, or does not have permission to read the project',
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`GitLab Error '${data.error}', ${data.error_description}`,
|
||||
);
|
||||
}
|
||||
|
||||
return Number(data.id);
|
||||
} catch (e) {
|
||||
throw new Error(`Could not get GitLab project ID for: ${target}, ${e}`);
|
||||
// Check relative path exist and replace it if it's the case.
|
||||
if (relativePath) {
|
||||
repo = repo.replace(relativePath, '');
|
||||
}
|
||||
|
||||
// Remove leading slash
|
||||
return repo.replace(/^\//, '');
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user