Merge pull request #27850 from kjjuno/feature/bitbucket-server-throttling

feat: Throttle calls to Bitbucket Server API

.changeset/cool-days-sparkle.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-catalog-backend-module-bitbucket-server': minor
+'@backstage/backend-defaults': minor
+---
+
+Throttles Bitbucket Server API calls

packages/backend-defaults/package.json

@@ -169,6 +169,7 @@
     "node-fetch": "^2.7.0",
     "node-forge": "^1.3.1",
     "p-limit": "^3.1.0",
+    "p-throttle": "^4.1.1",
     "path-to-regexp": "^8.0.0",
     "pg": "^8.11.3",
     "pg-connection-string": "^2.3.0",

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts

@@ -37,6 +37,20 @@ import { Minimatch } from 'minimatch';
 import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 
+import pThrottle from 'p-throttle';
+
+// 1 per second
+const throttle = pThrottle({
+  limit: 1,
+  interval: 1000,
+});
+
+const throttledFetch = throttle(
+  async (url: RequestInfo, options?: RequestInit) => {
+    return await fetch(url, options);
+  },
+);
+
 /**
  * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs.
  *
@@ -79,7 +93,7 @@ export class BitbucketServerUrlReader implements UrlReaderService {
 
     let response: Response;
     try {
-      response = await fetch(bitbucketUrl.toString(), {
+      response = await throttledFetch(bitbucketUrl.toString(), {
         headers: {
           ...requestOptions.headers,
           ...(etag && { 'If-None-Match': etag }),
@@ -129,7 +143,7 @@ export class BitbucketServerUrlReader implements UrlReaderService {
       url,
       this.integration.config,
     );
-    const archiveResponse = await fetch(
+    const archiveResponse = await throttledFetch(
       downloadUrl,
       getBitbucketServerRequestOptions(this.integration.config),
     );
@@ -198,7 +212,7 @@ export class BitbucketServerUrlReader implements UrlReaderService {
     // https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp211 (branches docs)
     const branchListUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches${branchParameter}`;
 
-    const branchListResponse = await fetch(
+    const branchListResponse = await throttledFetch(
       branchListUrl,
       getBitbucketServerRequestOptions(this.integration.config),
     );

plugins/catalog-backend-module-bitbucket-server/package.json

@@ -56,6 +56,7 @@
     "@backstage/errors": "workspace:^",
     "@backstage/integration": "workspace:^",
     "@backstage/plugin-catalog-node": "workspace:^",
+    "p-throttle": "^4.1.1",
     "uuid": "^11.0.0"
   },
   "devDependencies": {

plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts

@@ -19,6 +19,19 @@ import {
   getBitbucketServerRequestOptions,
 } from '@backstage/integration';
 import { BitbucketServerProject, BitbucketServerRepository } from './types';
+import pThrottle from 'p-throttle';
+
+// 1 per second
+const throttle = pThrottle({
+  limit: 1,
+  interval: 1000,
+});
+
+const throttledFetch = throttle(
+  async (url: RequestInfo, options?: RequestInit) => {
+    return await fetch(url, options);
+  },
+);
 
 /**
  * A client for interacting with a Bitbucket Server instance
@@ -65,7 +78,7 @@ export class BitbucketServerClient {
     path: string;
   }): Promise<Response> {
     const base = new URL(this.config.apiBaseUrl);
-    return fetch(
+    return throttledFetch(
       `${base.protocol}//${base.host}/projects/${options.projectKey}/repos/${options.repo}/raw/${options.path}`,
       getBitbucketServerRequestOptions(this.config),
     );
@@ -76,7 +89,7 @@ export class BitbucketServerClient {
     repo: string;
   }): Promise<BitbucketServerRepository> {
     const request = `${this.config.apiBaseUrl}/projects/${options.projectKey}/repos/${options.repo}`;
-    const response = await fetch(
+    const response = await throttledFetch(
       request,
       getBitbucketServerRequestOptions(this.config),
     );
@@ -117,16 +130,17 @@ export class BitbucketServerClient {
   }
 
   private async request(req: Request): Promise<Response> {
-    return fetch(req, getBitbucketServerRequestOptions(this.config)).then(
-      (response: Response) => {
-        if (!response.ok) {
-          throw new Error(
-            `Unexpected response for ${req.method} ${req.url}. Expected 200 but got ${response.status} - ${response.statusText}`,
-          );
-        }
-        return response;
-      },
-    );
+    return throttledFetch(
+      req,
+      getBitbucketServerRequestOptions(this.config),
+    ).then((response: Response) => {
+      if (!response.ok) {
+        throw new Error(
+          `Unexpected response for ${req.method} ${req.url}. Expected 200 but got ${response.status} - ${response.statusText}`,
+        );
+      }
+      return response;
+    });
   }
 }
 

yarn.lock

@@ -3646,6 +3646,7 @@ __metadata:
     node-forge: ^1.3.1
     node-mocks-http: ^1.0.0
     p-limit: ^3.1.0
+    p-throttle: ^4.1.1
     path-to-regexp: ^8.0.0
     pg: ^8.11.3
     pg-connection-string: ^2.3.0
@@ -5875,6 +5876,7 @@ __metadata:
     "@backstage/plugin-catalog-node": "workspace:^"
     luxon: ^3.0.0
     msw: ^1.0.0
+    p-throttle: ^4.1.1
     uuid: ^11.0.0
   languageName: unknown
   linkType: soft