diff --git a/.changeset/tiny-spoons-repeat.md b/.changeset/tiny-spoons-repeat.md new file mode 100644 index 0000000000..2977e0dde8 --- /dev/null +++ b/.changeset/tiny-spoons-repeat.md @@ -0,0 +1,11 @@ +--- +'@backstage/backend-defaults': patch +'@backstage/plugin-catalog-backend': minor +--- + +**BREAKING:** The `UrlReaderProccessor` now always calls the `search` method of the `UrlReaders`. Previous behavior was to call the `search` method only if the parsed Git URL's filename contained a wildcard and use `readUrl` otherwise. `UrlReaderService` must implement this logic in the `search` method instead. + +This allows each `UrlReaderService` implementation to check whether it's a search URL (that contains a wildcard pattern) or not using logic that is specific to each provider. + +In the different `UrlReadersService`, the `search` method have been updated to use the `readUrl` if the given URL doesn't contain a pattern. +For `UrlReaders` that didn't implement the `search` method, `readUrl` is called internally. diff --git a/packages/backend-defaults/report-urlReader.api.md b/packages/backend-defaults/report-urlReader.api.md index efe4a9ae47..72e8f0bfbe 100644 --- a/packages/backend-defaults/report-urlReader.api.md +++ b/packages/backend-defaults/report-urlReader.api.md @@ -57,7 +57,10 @@ export class AwsS3UrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -230,7 +233,10 @@ export class FetchUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -265,7 +271,10 @@ export class GerritUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -293,7 +302,10 @@ export class GiteaUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -384,7 +396,10 @@ export class HarnessUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts index a17f0c6c61..74db1b5a71 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts @@ -608,4 +608,48 @@ describe('AwsCodeCommitUrlReader', () => { expect(bodySubfolderFile.toString().trim()).toBe('site_name: Test2'); }); }); + + describe('search', () => { + const [{ reader }] = createReader({ + integrations: { + awsCodeCommit: [ + { + host: AMAZON_AWS_CODECOMMIT_HOST, + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + region: 'fakeregion', + }, + ], + }, + }); + + beforeEach(() => { + codeCommitClient.reset(); + + codeCommitClient.on(GetFileCommand).resolves({ + fileContent: fs.readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml', + ), + ), + commitId: `123abc`, + }); + }); + + it('should return a file when given an exact valid url', async () => { + const data = await reader.search( + 'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml', + ); + + expect(data.etag).toBe('123abc'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'site_name: Test\n', + ); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts index 5a2655ad1c..0759140894 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts @@ -21,6 +21,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { @@ -31,7 +32,11 @@ import { AwsCodeCommitIntegration, ScmIntegrations, } from '@backstage/integration'; -import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + ForwardedError, + NotModifiedError, +} from '@backstage/errors'; import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { CodeCommitClient, @@ -257,7 +262,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService { } return ReadUrlResponseFactory.fromReadable( - Readable.from([response?.fileContent] || []), + Readable.from([response?.fileContent]), { etag: response.commitId, }, @@ -357,7 +362,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService { commitSpecifier: commitSpecifier, }); const response = await codeCommitClient.send(getFileCommand); - const objectData = await Readable.from([response?.fileContent] || []); + const objectData = await Readable.from([response?.fileContent]); responses.push({ data: objectData, @@ -380,8 +385,33 @@ export class AwsCodeCommitUrlReader implements UrlReaderService { } } - async search(): Promise { - throw new Error('AwsCodeCommitReader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts index 15f46199da..ec8ff1694e 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts @@ -527,4 +527,50 @@ describe('AwsS3UrlReader', () => { expect(body.toString().trim()).toBe('site_name: Test'); }); }); + + describe('search', () => { + const [{ reader }] = createReader({ + integrations: { + awsS3: [ + { + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }, + ], + }, + }); + + beforeEach(() => { + s3Client.reset(); + + s3Client.on(GetObjectCommand).resolves({ + Body: sdkStreamMixin( + fs.createReadStream( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), + ), + ETag: '123abc', + LastModified: new Date('2020-01-01T00:00:00Z'), + }); + }); + + it('should return a file when given an exact valid url', async () => { + const data = await reader.search( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + + expect(data.etag).toBe('123abc'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'site_name: Test\n', + ); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts index 324538add4..26861b5273 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; @@ -32,7 +33,11 @@ import { ScmIntegrations, AwsS3IntegrationConfig, } from '@backstage/integration'; -import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + ForwardedError, + NotModifiedError, +} from '@backstage/errors'; import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { @@ -357,8 +362,33 @@ export class AwsS3UrlReader implements UrlReaderService { } } - async search(): Promise { - throw new Error('AwsS3Reader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts index f53a7b6c11..12a7ae8093 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts @@ -351,6 +351,17 @@ describe('AzureUrlReader', () => { ); }); + it('works for exact urls', async () => { + const result = await processor.search( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + ); + expect(result.etag).toBe(''); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + ); + }); + it('works for the naive case', async () => { const result = await processor.search( 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster', diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts index b46e7c20cd..465ec2326c 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts @@ -32,8 +32,13 @@ import { ScmIntegrations, AzureIntegration, } from '@backstage/integration'; +import parseGitUrl from 'git-url-parse'; import { Minimatch } from 'minimatch'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; @@ -181,6 +186,35 @@ export class AzureUrlReader implements UrlReaderService { url: string, options?: UrlReaderServiceSearchOptions, ): Promise { + const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const treeUrl = new URL(url); const path = treeUrl.searchParams.get('path'); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts index f5caade3fa..d2d0d28d76 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts @@ -451,5 +451,31 @@ describe('BitbucketCloudUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('should work for exact URLs', async () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBeNull(); + return res( + ctx.status(200), + ctx.body('foo'), + ctx.set('ETag', 'etag-value'), + ); + }, + ), + ); + + const result = await reader.search( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + expect(result.etag).toBe('etag-value'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + expect((await result.files[0].content()).toString()).toEqual('foo'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts index f6742c7423..c968420a5e 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts @@ -23,7 +23,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { BitbucketCloudIntegration, getBitbucketCloudDefaultBranch, @@ -163,6 +167,34 @@ export class BitbucketCloudUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const matcher = new Minimatch(filepath); // TODO(freben): For now, read the entire repo and filter through that. In diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts index 6af2858cde..5e82246298 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts @@ -30,6 +30,7 @@ import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; createMockDirectory({ mockOsTmpDir: true }); @@ -277,5 +278,23 @@ describe('BitbucketServerUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('should work for exact URLs by using readUrl directly', async () => { + reader.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + + const result = await reader.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml', + ); + expect(reader.readUrl).toHaveBeenCalledTimes(1); + expect(result.etag).toBe('etag'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml', + ); + expect((await result.files[0].content()).toString()).toEqual('content'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts index d327f80704..0a838af68b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts @@ -23,7 +23,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { BitbucketServerIntegration, getBitbucketServerDownloadUrl, @@ -168,6 +172,34 @@ export class BitbucketServerUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const matcher = new Minimatch(filepath); // TODO(freben): For now, read the entire repo and filter through that. In diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts index 54880aaa75..29217ea1f5 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts @@ -32,6 +32,7 @@ import { NotModifiedError } from '@backstage/errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import getRawBody from 'raw-body'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const logger = mockServices.logger.mock(); @@ -613,5 +614,22 @@ describe('BitbucketUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('should work for exact URLs', async () => { + hostedBitbucketProcessor.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + + const result = await hostedBitbucketProcessor.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + expect(result.etag).toBe('etag'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + expect((await result.files[0].content()).toString()).toEqual('content'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts index c12998509e..4c6de31aa4 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts @@ -23,7 +23,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { BitbucketIntegration, getBitbucketDefaultBranch, @@ -174,6 +178,34 @@ export class BitbucketUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const matcher = new Minimatch(filepath); // TODO(freben): For now, read the entire repo and filter through that. In diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts index f84422c564..4aa1429769 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts @@ -266,4 +266,26 @@ describe('FetchUrlReader', () => { ).rejects.toThrow(Error); }); }); + + describe('search', () => { + it('should return a file', async () => { + const data = await fetchUrlReader.search( + `https://backstage.io/some-resource`, + { etag: 'etag' }, + ); + expect(data.etag).toBe('foo'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe(`https://backstage.io/some-resource`); + expect((await data.files[0].content()).toString()).toEqual('content foo'); + }); + + it('should return an empty list of file if not found', async () => { + const data = await fetchUrlReader.search( + `https://backstage.io/not-exists`, + { etag: 'etag' }, + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts index 0e5a279c4d..5fb7f68809 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts @@ -19,9 +19,14 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { ReaderFactory } from './types'; import path from 'path'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; @@ -162,8 +167,33 @@ export class FetchUrlReader implements UrlReaderService { throw new Error('FetchUrlReader does not implement readTree'); } - async search(): Promise { - throw new Error('FetchUrlReader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts index 2fbb85b9b1..face959b24 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts @@ -420,4 +420,51 @@ describe.skip('GerritUrlReader', () => { expect(response.etag).toBe(sha); }); }); + + describe('search', () => { + const responseBuffer = Buffer.from('Apache License'); + + it('should return a single file when given an exact URL', async () => { + worker.use( + rest.get( + 'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content', + (_, res, ctx) => { + return res( + ctx.status(200), + ctx.body(responseBuffer.toString('base64')), + ); + }, + ), + ); + + const data = await gerritProcessor.search( + 'https://gerrit.com/web/project/+/refs/heads/master/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://gerrit.com/web/project/+/refs/heads/master/LICENSE', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'Apache License', + ); + }); + + it('should return empty list of files for not found files.', async () => { + worker.use( + rest.get( + 'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content', + (_, res, ctx) => { + return res(ctx.status(404, 'File not found.')); + }, + ), + ); + + const data = await gerritProcessor.search( + 'https://gerrit.com/web/project/+/refs/heads/master/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts index 835624627b..928ee41b73 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { Base64Decode } from 'base64-stream'; @@ -39,6 +40,7 @@ import { NotFoundError, NotModifiedError, ResponseError, + assertError, } from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; @@ -144,8 +146,33 @@ export class GerritUrlReader implements UrlReaderService { return this.readTreeFromGitiles(url, urlRevision, options); } - async search(): Promise { - throw new Error('GerritReader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts index 77c995c552..d0432a6162 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts @@ -330,4 +330,62 @@ describe('GiteaUrlReader', () => { ).rejects.toThrow(NotFoundError); }); }); + + describe('search', () => { + const responseBuffer = Buffer.from('Apache License'); + const giteaApiResponse = (content: any) => { + return JSON.stringify({ + encoding: 'base64', + content: Buffer.from(content).toString('base64'), + }); + }; + + it('should return a single file when given an exact URL', async () => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE', + (req, res, ctx) => { + // Test utils prefers matching URL directly but it is part of Gitea's API + if (req.url.searchParams.get('ref') === 'branch2') { + return res( + ctx.status(200), + ctx.body(giteaApiResponse(responseBuffer.toString())), + ); + } + + return res(ctx.status(500)); + }, + ), + ); + + const data = await giteaProcessor.search( + 'https://gitea.com/owner/project/src/branch/branch2/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://gitea.com/owner/project/src/branch/branch2/LICENSE', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'Apache License', + ); + }); + + it('should return empty list of files for not found files.', async () => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE', + (_, res, ctx) => { + return res(ctx.status(404, 'File not found.')); + }, + ), + ); + + const data = await giteaProcessor.search( + 'https://gitea.com/owner/project/src/branch/branch2/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts index 502c2d2ff9..920faa386c 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { @@ -34,6 +35,7 @@ import { import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { + assertError, AuthenticationError, NotFoundError, NotModifiedError, @@ -155,8 +157,33 @@ export class GiteaUrlReader implements UrlReaderService { }); } - search(): Promise { - throw new Error('GiteaUrlReader search not implemented.'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts index e8d81046ab..70b18df091 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts @@ -37,6 +37,7 @@ import { GithubUrlReader, } from './GithubUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const mockDir = createMockDirectory({ mockOsTmpDir: true }); @@ -887,9 +888,6 @@ describe('GithubUrlReader', () => { expect(r2.etag).toBe('etag123abc'); expect(r2.files.length).toBe(2); - const r3 = await reader.search(`${baseUrl}/backstage/mock/tree/main/o`); - expect(r3.files.length).toBe(0); - const r4 = await reader.search( `${baseUrl}/backstage/mock/tree/main/*docs*`, ); @@ -1017,6 +1015,23 @@ describe('GithubUrlReader', () => { await runTests(gheProcessor, 'https://ghe.github.com'); }); + it('uses readUrl when searching for an exact file', async () => { + githubProcessor.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + const data = await githubProcessor.search( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect(githubProcessor.readUrl).toHaveBeenCalledTimes(1); + expect(data.etag).toBe('etag'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect((await data.files[0].content()).toString()).toEqual('content'); + }); + it('throws NotModifiedError when same etag', async () => { await expect( githubProcessor.search( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts index 913e382f92..d78475de0b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts @@ -37,7 +37,11 @@ import fetch, { RequestInit, Response } from 'node-fetch'; import parseGitUrl from 'git-url-parse'; import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { parseLastModified } from './util'; @@ -179,6 +183,35 @@ export class GithubUrlReader implements UrlReaderService { url: string, options?: UrlReaderServiceSearchOptions, ): Promise { + const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const repoDetails = await this.getRepoDetails(url); const commitSha = repoDetails.commitSha; @@ -186,7 +219,6 @@ export class GithubUrlReader implements UrlReaderService { throw new NotModifiedError(); } - const { filepath } = parseGitUrl(url); const { headers } = await this.getCredentials(url, options); const files = await this.doSearch( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts index d0607767c2..6274ba8dc3 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts @@ -31,6 +31,7 @@ import { GitLabIntegration, readGitLabIntegrationConfig, } from '@backstage/integration'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const logger = mockServices.logger.mock(); @@ -663,6 +664,23 @@ describe('GitlabUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('returns a single file for exact urls', async () => { + gitlabProcessor.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + const data = await gitlabProcessor.search( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect(gitlabProcessor.readUrl).toHaveBeenCalledTimes(1); + expect(data.etag).toBe('etag'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect((await data.files[0].content()).toString()).toEqual('content'); + }); }); describe('getGitlabFetchUrl', () => { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts index 77a8d31623..1707c3b452 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts @@ -26,7 +26,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { GitLabIntegration, ScmIntegrations, @@ -242,6 +246,34 @@ export class GitlabUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const staticPart = this.getStaticPart(filepath); const matcher = new Minimatch(filepath); const treeUrl = trimEnd(url.replace(filepath, staticPart), `/`); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts index 537b8c375e..7429bae94f 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts @@ -22,6 +22,7 @@ import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; import { UrlReaderPredicateTuple } from './types'; import packageinfo from '../../../../package.json'; import { mockServices } from '@backstage/backend-test-utils'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const bucketGetFilesMock = jest.fn(); class Bucket { @@ -130,7 +131,7 @@ describe('GcsUrlReader', () => { }); it('throws if search url does not end with *', async () => { - const glob = 'https://storage.cloud.google.com/bucket/no-asterisk'; + const glob = 'https://storage.cloud.google.com/bucket/*no-asterisk'; await expect(() => reader.search(glob)).rejects.toThrow( 'GcsUrlReader only supports prefix-based searches', ); @@ -167,5 +168,22 @@ describe('GcsUrlReader', () => { 'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt', ); }); + + it('returns single file if there is no wildcard', async () => { + reader.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + const data = await reader.search( + 'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt', + ); + expect(reader.readUrl).toHaveBeenCalledTimes(1); + expect(data.etag).toBe('etag'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt', + ); + expect((await data.files[0].content()).toString()).toEqual('content'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts index 10440dba20..36f2e183c7 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { ReaderFactory } from './types'; @@ -31,6 +32,7 @@ import { import { Readable } from 'stream'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import packageinfo from '../../../../package.json'; +import { assertError } from '@backstage/errors'; const GOOGLE_GCS_HOST = 'storage.cloud.google.com'; @@ -117,9 +119,39 @@ export class GoogleGcsUrlReader implements UrlReaderService { throw new Error('GcsUrlReader does not implement readTree'); } - async search(url: string): Promise { + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { const { bucket, key: pattern } = parseURL(url); + // If it's a direct URL we use readUrl instead + if (!pattern?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + if (!pattern.endsWith('*') || pattern.indexOf('*') !== pattern.length - 1) { throw new Error('GcsUrlReader only supports prefix-based searches'); } diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts index 004e1931d0..b3f851a894 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts @@ -257,4 +257,28 @@ describe('HarnessUrlReader', () => { ).rejects.toThrow(NotFoundError); }); }); + + describe('search', () => { + it('should return a single file when given an exact URL', async () => { + const data = await harnessProcessor.search( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'Apache License', + ); + }); + + it('should return empty list of files for not found files.', async () => { + const data = await harnessProcessor.search( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts index 622d7f3379..6c53b25c0d 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts @@ -21,6 +21,7 @@ import { UrlReaderServiceReadUrlResponse, UrlReaderServiceSearchResponse, UrlReaderServiceReadTreeOptions, + UrlReaderServiceSearchOptions, } from '@backstage/backend-plugin-api'; import { getHarnessRequestOptions, @@ -35,6 +36,7 @@ import { ReadTreeResponseFactory, ReaderFactory } from './types'; import fetch, { Response } from 'node-fetch'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { + assertError, AuthenticationError, NotFoundError, NotModifiedError, @@ -154,8 +156,33 @@ export class HarnessUrlReader implements UrlReaderService { }); } - search(): Promise { - throw new Error('HarnessUrlReader search not implemented.'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts index 4a8f982027..447cab40ed 100644 --- a/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts @@ -180,12 +180,11 @@ describe('UrlReaderProcessor', () => { mockCache, ), )) as CatalogProcessorErrorResult; - expect(generated.type).toBe('error'); expect(generated.location).toBe(spec); expect(generated.error.name).toBe('NotFoundError'); expect(generated.error.message).toBe( - `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`, + `Unable to read url, no matching files found for ${mockApiOrigin}/component-notfound.yaml`, ); }); diff --git a/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts index e817e19616..8852f37de4 100644 --- a/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts @@ -18,7 +18,6 @@ import { Entity } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import limiterFactory, { Limit } from 'p-limit'; import { LocationSpec } from '@backstage/plugin-catalog-common'; -import parseGitUrl from 'git-url-parse'; import { CatalogProcessor, CatalogProcessorCache, @@ -80,6 +79,15 @@ export class UrlReaderProcessor implements CatalogProcessor { cacheItem?.etag, ); + if (response.length === 0 && !optional) { + emit( + processingResult.notFoundError( + location, + `Unable to read ${location.type}, no matching files found for ${location.target}`, + ), + ); + } + const parseResults: CatalogProcessorResult[] = []; for (const item of response) { for await (const parseResult of parser({ @@ -112,10 +120,6 @@ export class UrlReaderProcessor implements CatalogProcessor { } emit(processingResult.refresh(`${location.type}:${location.target}`)); await cache.set(CACHE_KEY, cacheItem); - } else if (error.name === 'NotFoundError') { - if (!optional) { - emit(processingResult.notFoundError(location, message)); - } } else { emit(processingResult.generalError(location, message)); } @@ -128,23 +132,13 @@ export class UrlReaderProcessor implements CatalogProcessor { location: string, etag?: string, ): Promise<{ response: { data: Buffer; url: string }[]; etag?: string }> { - // Does it contain globs? I.e. does it contain asterisks or question marks - // (no curly braces for now) + const response = await this.options.reader.search(location, { etag }); - const { filepath } = parseGitUrl(location); - if (filepath?.match(/[*?]/)) { - const response = await this.options.reader.search(location, { etag }); - const output = response.files.map(async file => ({ - url: file.url, - data: await this.#limiter(file.content), - })); - return { response: await Promise.all(output), etag: response.etag }; - } + const output = response.files.map(async file => ({ + url: file.url, + data: await this.#limiter(file.content), + })); - const data = await this.options.reader.readUrl(location, { etag }); - return { - response: [{ url: location, data: await data.buffer() }], - etag: data.etag, - }; + return { response: await Promise.all(output), etag: response.etag }; } }