From c3249d6c11569d086f199db65880f58592cdfb54 Mon Sep 17 00:00:00 2001 From: armandocomellas1 Date: Fri, 12 Jan 2024 11:31:19 -0600 Subject: [PATCH 1/5] Add code to handle URL Reader from GCS with wildcard \* Signed-off-by: armandocomellas1 --- .changeset/thirty-cats-help.md | 5 +++++ .../catalog-backend/src/modules/core/UrlReaderProcessor.ts | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/thirty-cats-help.md diff --git a/.changeset/thirty-cats-help.md b/.changeset/thirty-cats-help.md new file mode 100644 index 0000000000..70ae78e3c1 --- /dev/null +++ b/.changeset/thirty-cats-help.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Change script in **UrlReaderProcessor.ts** Replacing the line code 127 'const { pathname: filepath } = new URL(location)' with to handle URL Reader from GCS with wildcard \* diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index 5e31dcbd1a..c568f03a81 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -17,7 +17,6 @@ import { UrlReader } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; -import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; import { LocationSpec } from '@backstage/plugin-catalog-common'; @@ -124,7 +123,8 @@ export class UrlReaderProcessor implements CatalogProcessor { ): Promise<{ response: { data: Buffer; url: string }[]; etag?: string }> { // Does it contain globs? I.e. does it contain asterisks or question marks // (no curly braces for now) - const { filepath } = parseGitUrl(location); + + const { pathname: filepath } = new URL(location); if (filepath?.match(/[*?]/)) { const limiter = limiterFactory(5); const response = await this.options.reader.search(location, { etag }); From 41b25258bff4869bf6394d7cf353764e1cf7d558 Mon Sep 17 00:00:00 2001 From: armandocomellas1 Date: Fri, 12 Jan 2024 11:36:42 -0600 Subject: [PATCH 2/5] Fixing the linter error with the naming the variable inside the changeset thirty-cats-help.md Signed-off-by: armandocomellas1 --- .changeset/thirty-cats-help.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/thirty-cats-help.md b/.changeset/thirty-cats-help.md index 70ae78e3c1..107fb7584b 100644 --- a/.changeset/thirty-cats-help.md +++ b/.changeset/thirty-cats-help.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Change script in **UrlReaderProcessor.ts** Replacing the line code 127 'const { pathname: filepath } = new URL(location)' with to handle URL Reader from GCS with wildcard \* +Change script in **UrlReaderProcessor.ts** Replacing the line code 127 **const { pathname: filepath } = new URL(location)** with to handle URL Reader from GCS with wildcard \* From 1de52a9c60d6ba373a8c9f1b2cbb05f198f957fa Mon Sep 17 00:00:00 2001 From: armandocomellas1 Date: Fri, 12 Jan 2024 11:40:45 -0600 Subject: [PATCH 3/5] Fixing the linter error with the naming the variable inside the changeset thirty-cats-help.md Signed-off-by: armandocomellas1 --- .changeset/thirty-cats-help.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/thirty-cats-help.md b/.changeset/thirty-cats-help.md index 107fb7584b..bb6011ca9c 100644 --- a/.changeset/thirty-cats-help.md +++ b/.changeset/thirty-cats-help.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Change script in **UrlReaderProcessor.ts** Replacing the line code 127 **const { pathname: filepath } = new URL(location)** with to handle URL Reader from GCS with wildcard \* +Change script in **UrlReaderProcessor.ts** Replacing the line code 127 with method (new URL) to handle URL Reader from GCS with wildcard \* From f5da6523c03fbfc8f6de6e72cfbc47de0a4386da Mon Sep 17 00:00:00 2001 From: armandocomellas1 Date: Mon, 15 Jan 2024 18:01:06 -0600 Subject: [PATCH 4/5] Change the description header of the changeset and also add a unit test for new URL method Signed-off-by: armandocomellas1 --- .changeset/thirty-cats-help.md | 2 +- .../modules/core/UrlReaderProcessor.test.ts | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.changeset/thirty-cats-help.md b/.changeset/thirty-cats-help.md index bb6011ca9c..9c5d24180d 100644 --- a/.changeset/thirty-cats-help.md +++ b/.changeset/thirty-cats-help.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Change script in **UrlReaderProcessor.ts** Replacing the line code 127 with method (new URL) to handle URL Reader from GCS with wildcard \* +Parse the URL using a different method rather than `git-url-parse` to support wildcards for URLs which are not VCS providers diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index c9575ce5cd..0a3bec7570 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -213,4 +213,31 @@ describe('UrlReaderProcessor', () => { expect(reader.search).toHaveBeenCalledTimes(1); }); + + it('parser return valid URL with wildcard *', async () => { + const logger = getVoidLogger(); + + const reader: jest.Mocked = { + readUrl: jest.fn(), + readTree: jest.fn(), + search: jest.fn().mockImplementation(async () => []), + }; + + const processor = new UrlReaderProcessor({ reader, logger }); + + const emit = jest.fn(); + + await processor.readLocation( + { + type: 'url', + target: 'https://storage.cloud.google.com/ah-backstage-poc-catalog/*', + }, + false, + emit, + defaultEntityDataParser, + mockCache, + ); + + expect(reader.search).toHaveReturned(); + }); }); From f6e4c2a9353fc6707e9334608dca5f5ac7a676e5 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 16 Jan 2024 10:24:31 +0100 Subject: [PATCH 5/5] chore: update the test to ensure that it's called with the correct arguments Signed-off-by: blam --- .../src/modules/core/UrlReaderProcessor.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index 0a3bec7570..e2489df4b4 100644 --- a/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -238,6 +238,9 @@ describe('UrlReaderProcessor', () => { mockCache, ); - expect(reader.search).toHaveReturned(); + expect(reader.search).toHaveBeenCalledWith( + 'https://storage.cloud.google.com/ah-backstage-poc-catalog/*', + { etag: undefined }, + ); }); });