Merge pull request #26861 from backstage/freben/reader-limiter

properly honor the reader limiter in the url reader processor

.changeset/quiet-dingos-bathe.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Fixed a bug where the concurrency limiter for URL reading was not honored

plugins/catalog-backend/report.api.md

@@ -543,8 +543,8 @@ export class UrlReaderProcessor implements CatalogProcessor_2 {
 // src/processors/PlaceholderProcessor.d.ts:21:5 - (ae-undocumented) Missing documentation for "getProcessorName".
 // src/processors/PlaceholderProcessor.d.ts:22:5 - (ae-undocumented) Missing documentation for "preProcessEntity".
 // src/processors/UrlReaderProcessor.d.ts:5:1 - (ae-undocumented) Missing documentation for "UrlReaderProcessor".
-// src/processors/UrlReaderProcessor.d.ts:11:5 - (ae-undocumented) Missing documentation for "getProcessorName".
-// src/processors/UrlReaderProcessor.d.ts:12:5 - (ae-undocumented) Missing documentation for "readLocation".
+// src/processors/UrlReaderProcessor.d.ts:12:5 - (ae-undocumented) Missing documentation for "getProcessorName".
+// src/processors/UrlReaderProcessor.d.ts:13:5 - (ae-undocumented) Missing documentation for "readLocation".
 // src/search/DefaultCatalogCollator.d.ts:11:1 - (ae-undocumented) Missing documentation for "DefaultCatalogCollator".
 // src/search/DefaultCatalogCollator.d.ts:12:5 - (ae-undocumented) Missing documentation for "discovery".
 // src/search/DefaultCatalogCollator.d.ts:13:5 - (ae-undocumented) Missing documentation for "locationTemplate".

plugins/catalog-backend/src/processors/UrlReaderProcessor.ts

@@ -16,7 +16,7 @@
 
 import { Entity } from '@backstage/catalog-model';
 import { assertError } from '@backstage/errors';
-import limiterFactory from 'p-limit';
+import limiterFactory, { Limit } from 'p-limit';
 import { LocationSpec } from '@backstage/plugin-catalog-common';
 import parseGitUrl from 'git-url-parse';
 import {
@@ -44,12 +44,18 @@ type CacheItem = {
 
 /** @public */
 export class UrlReaderProcessor implements CatalogProcessor {
+  // This limiter is used for only consuming a limited number of read streams
+  // concurrently.
+  #limiter: Limit;
+
   constructor(
     private readonly options: {
       reader: UrlReaderService;
       logger: LoggerService;
     },
-  ) {}
+  ) {
+    this.#limiter = limiterFactory(5);
+  }
 
   getProcessorName() {
     return 'url-reader';
@@ -126,11 +132,10 @@ export class UrlReaderProcessor implements CatalogProcessor {
 
     const { filepath } = parseGitUrl(location);
     if (filepath?.match(/[*?]/)) {
-      const limiter = limiterFactory(5);
       const response = await this.options.reader.search(location, { etag });
       const output = response.files.map(async file => ({
         url: file.url,
-        data: await limiter(file.content),
+        data: await this.#limiter(file.content),
       }));
       return { response: await Promise.all(output), etag: response.etag };
     }