From b3ff40c269e0cb1684082daf1b81c8c71b9780ca Mon Sep 17 00:00:00 2001 From: Kevin Johnson Date: Tue, 26 Nov 2024 13:31:30 -0700 Subject: [PATCH 1/3] Add throttling to bitbucket server client Signed-off-by: Kevin Johnson --- .../package.json | 1 + .../src/lib/BitbucketServerClient.ts | 38 +++++++++++++------ yarn.lock | 1 + 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/plugins/catalog-backend-module-bitbucket-server/package.json b/plugins/catalog-backend-module-bitbucket-server/package.json index ac90aaf86a..f5ef864ef2 100644 --- a/plugins/catalog-backend-module-bitbucket-server/package.json +++ b/plugins/catalog-backend-module-bitbucket-server/package.json @@ -56,6 +56,7 @@ "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "p-throttle": "^4.1.1", "uuid": "^11.0.0" }, "devDependencies": { diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts index ee2caf7ffd..9ab116a400 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -19,6 +19,19 @@ import { getBitbucketServerRequestOptions, } from '@backstage/integration'; import { BitbucketServerProject, BitbucketServerRepository } from './types'; +import pThrottle from 'p-throttle'; + +const throttle = pThrottle({ + limit: 1, + interval: 500, +}); + +const throttledFetch = throttle( + async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { + console.log(JSON.stringify(url)); + return await fetch(url, options); + }, +); /** * A client for interacting with a Bitbucket Server instance @@ -65,7 +78,7 @@ export class BitbucketServerClient { path: string; }): Promise { const base = new URL(this.config.apiBaseUrl); - return fetch( + return throttledFetch( `${base.protocol}//${base.host}/projects/${options.projectKey}/repos/${options.repo}/raw/${options.path}`, getBitbucketServerRequestOptions(this.config), ); @@ -76,7 +89,7 @@ export class BitbucketServerClient { repo: string; }): Promise { const request = `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}`; - const response = await fetch( + const response = await throttledFetch( request, getBitbucketServerRequestOptions(this.config), ); @@ -117,16 +130,17 @@ export class BitbucketServerClient { } private async request(req: Request): Promise { - return fetch(req, getBitbucketServerRequestOptions(this.config)).then( - (response: Response) => { - if (!response.ok) { - throw new Error( - `Unexpected response for ${req.method} ${req.url}. Expected 200 but got ${response.status} - ${response.statusText}`, - ); - } - return response; - }, - ); + return throttledFetch( + req, + getBitbucketServerRequestOptions(this.config), + ).then((response: Response) => { + if (!response.ok) { + throw new Error( + `Unexpected response for ${req.method} ${req.url}. Expected 200 but got ${response.status} - ${response.statusText}`, + ); + } + return response; + }); } } diff --git a/yarn.lock b/yarn.lock index 2c40b37ad1..d315dc0104 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5876,6 +5876,7 @@ __metadata: "@backstage/plugin-catalog-node": "workspace:^" luxon: ^3.0.0 msw: ^1.0.0 + p-throttle: ^4.1.1 uuid: ^11.0.0 languageName: unknown linkType: soft From 8f7a06bcbfce770c0a6be2e923a04e636c559eaa Mon Sep 17 00:00:00 2001 From: Kevin Johnson Date: Tue, 26 Nov 2024 15:51:39 -0700 Subject: [PATCH 2/3] Adding throttling Signed-off-by: Kevin Johnson --- packages/backend-defaults/package.json | 1 + .../urlReader/lib/BitbucketServerUrlReader.ts | 20 ++++++++++++++++--- .../src/lib/BitbucketServerClient.ts | 4 ++-- yarn.lock | 1 + 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index d8240c429c..26417fc7da 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -169,6 +169,7 @@ "node-fetch": "^2.7.0", "node-forge": "^1.3.1", "p-limit": "^3.1.0", + "p-throttle": "^4.1.1", "path-to-regexp": "^8.0.0", "pg": "^8.11.3", "pg-connection-string": "^2.3.0", diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts index c37cb59308..3bc84deeb9 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts @@ -37,6 +37,20 @@ import { Minimatch } from 'minimatch'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; +import pThrottle from 'p-throttle'; + +// 1 per second +const throttle = pThrottle({ + limit: 1, + interval: 1000, +}); + +const throttledFetch = throttle( + async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { + return await fetch(url, options); + }, +); + /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs. * @@ -79,7 +93,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { let response: Response; try { - response = await fetch(bitbucketUrl.toString(), { + response = await throttledFetch(bitbucketUrl.toString(), { headers: { ...requestOptions.headers, ...(etag && { 'If-None-Match': etag }), @@ -129,7 +143,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { url, this.integration.config, ); - const archiveResponse = await fetch( + const archiveResponse = await throttledFetch( downloadUrl, getBitbucketServerRequestOptions(this.integration.config), ); @@ -198,7 +212,7 @@ export class BitbucketServerUrlReader implements UrlReaderService { // https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp211 (branches docs) const branchListUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches${branchParameter}`; - const branchListResponse = await fetch( + const branchListResponse = await throttledFetch( branchListUrl, getBitbucketServerRequestOptions(this.integration.config), ); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts index 9ab116a400..1859acf51d 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -21,14 +21,14 @@ import { import { BitbucketServerProject, BitbucketServerRepository } from './types'; import pThrottle from 'p-throttle'; +// 1 per second const throttle = pThrottle({ limit: 1, - interval: 500, + interval: 1000, }); const throttledFetch = throttle( async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { - console.log(JSON.stringify(url)); return await fetch(url, options); }, ); diff --git a/yarn.lock b/yarn.lock index d315dc0104..2bce4a50eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3646,6 +3646,7 @@ __metadata: node-forge: ^1.3.1 node-mocks-http: ^1.0.0 p-limit: ^3.1.0 + p-throttle: ^4.1.1 path-to-regexp: ^8.0.0 pg: ^8.11.3 pg-connection-string: ^2.3.0 From 3f34ea918d71166c1363218f4f082cad58d23067 Mon Sep 17 00:00:00 2001 From: Kevin Johnson Date: Tue, 26 Nov 2024 16:09:42 -0700 Subject: [PATCH 3/3] chore: adding reports Signed-off-by: Kevin Johnson --- .changeset/cool-days-sparkle.md | 6 ++++++ .../entrypoints/urlReader/lib/BitbucketServerUrlReader.ts | 2 +- .../src/lib/BitbucketServerClient.ts | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changeset/cool-days-sparkle.md diff --git a/.changeset/cool-days-sparkle.md b/.changeset/cool-days-sparkle.md new file mode 100644 index 0000000000..1dc9a76df7 --- /dev/null +++ b/.changeset/cool-days-sparkle.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend-module-bitbucket-server': minor +'@backstage/backend-defaults': minor +--- + +Throttles Bitbucket Server API calls diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts index 3bc84deeb9..d327f80704 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts @@ -46,7 +46,7 @@ const throttle = pThrottle({ }); const throttledFetch = throttle( - async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { + async (url: RequestInfo, options?: RequestInit) => { return await fetch(url, options); }, ); diff --git a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts index 1859acf51d..73373ffff1 100644 --- a/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts +++ b/plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts @@ -28,7 +28,7 @@ const throttle = pThrottle({ }); const throttledFetch = throttle( - async (url: fetch.RequestInfo, options?: fetch.RequestInit) => { + async (url: RequestInfo, options?: RequestInit) => { return await fetch(url, options); }, );