Adding throttling

Signed-off-by: Kevin Johnson <kljohnson@verisk.com>

packages/backend-defaults/package.json

@@ -169,6 +169,7 @@
     "node-fetch": "^2.7.0",
     "node-forge": "^1.3.1",
     "p-limit": "^3.1.0",
+    "p-throttle": "^4.1.1",
     "path-to-regexp": "^8.0.0",
     "pg": "^8.11.3",
     "pg-connection-string": "^2.3.0",

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts

@@ -37,6 +37,20 @@ import { Minimatch } from 'minimatch';
 import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 
+import pThrottle from 'p-throttle';
+
+// 1 per second
+const throttle = pThrottle({
+  limit: 1,
+  interval: 1000,
+});
+
+const throttledFetch = throttle(
+  async (url: fetch.RequestInfo, options?: fetch.RequestInit) => {
+    return await fetch(url, options);
+  },
+);
+
 /**
  * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for files from Bitbucket Server APIs.
  *
@@ -79,7 +93,7 @@ export class BitbucketServerUrlReader implements UrlReaderService {
 
     let response: Response;
     try {
-      response = await fetch(bitbucketUrl.toString(), {
+      response = await throttledFetch(bitbucketUrl.toString(), {
         headers: {
           ...requestOptions.headers,
           ...(etag && { 'If-None-Match': etag }),
@@ -129,7 +143,7 @@ export class BitbucketServerUrlReader implements UrlReaderService {
       url,
       this.integration.config,
     );
-    const archiveResponse = await fetch(
+    const archiveResponse = await throttledFetch(
       downloadUrl,
       getBitbucketServerRequestOptions(this.integration.config),
     );
@@ -198,7 +212,7 @@ export class BitbucketServerUrlReader implements UrlReaderService {
     // https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp211 (branches docs)
     const branchListUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches${branchParameter}`;
 
-    const branchListResponse = await fetch(
+    const branchListResponse = await throttledFetch(
       branchListUrl,
       getBitbucketServerRequestOptions(this.integration.config),
     );

plugins/catalog-backend-module-bitbucket-server/src/lib/BitbucketServerClient.ts

@@ -21,14 +21,14 @@ import {
 import { BitbucketServerProject, BitbucketServerRepository } from './types';
 import pThrottle from 'p-throttle';
 
+// 1 per second
 const throttle = pThrottle({
   limit: 1,
-  interval: 500,
+  interval: 1000,
 });
 
 const throttledFetch = throttle(
   async (url: fetch.RequestInfo, options?: fetch.RequestInit) => {
-    console.log(JSON.stringify(url));
     return await fetch(url, options);
   },
 );

yarn.lock

@@ -3646,6 +3646,7 @@ __metadata:
     node-forge: ^1.3.1
     node-mocks-http: ^1.0.0
     p-limit: ^3.1.0
+    p-throttle: ^4.1.1
     path-to-regexp: ^8.0.0
     pg: ^8.11.3
     pg-connection-string: ^2.3.0