From 62747f87f2431082941393626ee7c98408ddc319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 26 Sep 2024 14:16:17 +0200 Subject: [PATCH] properly honor the reader limiter in the url reader processor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/quiet-dingos-bathe.md | 5 +++++ plugins/catalog-backend/report.api.md | 4 ++-- .../src/processors/UrlReaderProcessor.ts | 13 +++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 .changeset/quiet-dingos-bathe.md diff --git a/.changeset/quiet-dingos-bathe.md b/.changeset/quiet-dingos-bathe.md new file mode 100644 index 0000000000..b0e41de687 --- /dev/null +++ b/.changeset/quiet-dingos-bathe.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed a bug where the concurrency limiter for URL reading was not honored diff --git a/plugins/catalog-backend/report.api.md b/plugins/catalog-backend/report.api.md index 51dc76f385..9e27995a28 100644 --- a/plugins/catalog-backend/report.api.md +++ b/plugins/catalog-backend/report.api.md @@ -543,8 +543,8 @@ export class UrlReaderProcessor implements CatalogProcessor_2 { // src/processors/PlaceholderProcessor.d.ts:21:5 - (ae-undocumented) Missing documentation for "getProcessorName". // src/processors/PlaceholderProcessor.d.ts:22:5 - (ae-undocumented) Missing documentation for "preProcessEntity". // src/processors/UrlReaderProcessor.d.ts:5:1 - (ae-undocumented) Missing documentation for "UrlReaderProcessor". -// src/processors/UrlReaderProcessor.d.ts:11:5 - (ae-undocumented) Missing documentation for "getProcessorName". -// src/processors/UrlReaderProcessor.d.ts:12:5 - (ae-undocumented) Missing documentation for "readLocation". +// src/processors/UrlReaderProcessor.d.ts:12:5 - (ae-undocumented) Missing documentation for "getProcessorName". +// src/processors/UrlReaderProcessor.d.ts:13:5 - (ae-undocumented) Missing documentation for "readLocation". // src/search/DefaultCatalogCollator.d.ts:11:1 - (ae-undocumented) Missing documentation for "DefaultCatalogCollator". // src/search/DefaultCatalogCollator.d.ts:12:5 - (ae-undocumented) Missing documentation for "discovery". // src/search/DefaultCatalogCollator.d.ts:13:5 - (ae-undocumented) Missing documentation for "locationTemplate". diff --git a/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts index b623098171..9d76f5679f 100644 --- a/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; -import limiterFactory from 'p-limit'; +import limiterFactory, { Limit } from 'p-limit'; import { LocationSpec } from '@backstage/plugin-catalog-common'; import parseGitUrl from 'git-url-parse'; import { @@ -44,12 +44,18 @@ type CacheItem = { /** @public */ export class UrlReaderProcessor implements CatalogProcessor { + // This limiter is used for only consuming a limited number of read streams + // concurrently. + #limiter: Limit; + constructor( private readonly options: { reader: UrlReaderService; logger: LoggerService; }, - ) {} + ) { + this.#limiter = limiterFactory(5); + } getProcessorName() { return 'url-reader'; @@ -126,11 +132,10 @@ export class UrlReaderProcessor implements CatalogProcessor { const { filepath } = parseGitUrl(location); if (filepath?.match(/[*?]/)) { - const limiter = limiterFactory(5); const response = await this.options.reader.search(location, { etag }); const output = response.files.map(async file => ({ url: file.url, - data: await limiter(file.content), + data: await this.#limiter(file.content), })); return { response: await Promise.all(output), etag: response.etag }; }