diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index d8240c429c..26417fc7da 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -169,6 +169,7 @@ "node-fetch": "^2.7.0", "node-forge": "^1.3.1", "p-limit": "^3.1.0", + "p-throttle": "^4.1.1", "path-to-regexp": "^8.0.0", "pg": "^8.11.3", "pg-connection-string": "^2.3.0", diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts index c37cb59308..3bc84deeb9 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts @@ -37,6 +37,20 @@ import { Minimatch } from 'minimatch'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; +import pThrottle from 'p-throttle'; + +// 1 per second +const throttle = pThrottle({ + limit: 1, + interval: 1000, +}); + +const throttledFetch = throttle( + async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { + return await fetch(url, options); + }, +); + /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs. * @@ -79,7 +93,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { let response: Response; try { - response = await fetch(bitbucketUrl.toString(), { + response = await throttledFetch(bitbucketUrl.toString(), { headers: { ...requestOptions.headers, ...(etag && { 'If-None-Match': etag }), @@ -129,7 +143,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { url, this.integration.config, ); - const archiveResponse = await fetch( + const archiveResponse = await throttledFetch( downloadUrl, getBitbucketServerRequestOptions(this.integration.config), ); @@ -198,7 +212,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { // https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp211 (branches docs) const branchListUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches${branchParameter}`; - const branchListResponse = await fetch( + const branchListResponse = await throttledFetch( branchListUrl, getBitbucketServerRequestOptions(this.integration.config), ); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts index 9ab116a400..1859acf51d 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -21,14 +21,14 @@ import { import { BitbucketServerProject, BitbucketServerRepository } from './types'; import pThrottle from 'p-throttle'; +// 1 per second const throttle = pThrottle({ limit: 1, - interval: 500, + interval: 1000, }); const throttledFetch = throttle( async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { - console.log(JSON.stringify(url)); return await fetch(url, options); }, ); diff --git a/yarn.lock b/yarn.lock index d315dc0104..2bce4a50eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3646,6 +3646,7 @@ __metadata: node-forge: ^1.3.1 node-mocks-http: ^1.0.0 p-limit: ^3.1.0 + p-throttle: ^4.1.1 path-to-regexp: ^8.0.0 pg: ^8.11.3 pg-connection-string: ^2.3.0