diff --git a/.changeset/heavy-bobcats-search.md b/.changeset/heavy-bobcats-search.md new file mode 100644 index 0000000000..86f7ea09ef --- /dev/null +++ b/.changeset/heavy-bobcats-search.md @@ -0,0 +1,8 @@ +--- +'@backstage/create-app': patch +--- + +Enable `catalog.useUrlReadersSearch` configuration by default in newly created Backstage installations. + +This parameter makes `UrlReaderProcessor` always use the search method. +New adopters are encouraged to enable it as this behavior will be the default in a future release. diff --git a/.changeset/lucky-seas-live.md b/.changeset/lucky-seas-live.md new file mode 100644 index 0000000000..bd084f6ebf --- /dev/null +++ b/.changeset/lucky-seas-live.md @@ -0,0 +1,11 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +The `UrlReaderProccessor` accepts a new config flag `catalog.useUrlReadersSearch` to always call the `search` method of `UrlReaders`. + +This flag currently defaults to `false`, but adopters are encouraged to enable it as this behavior will be the default in a future release. + +Previous behavior was to call the `search` method only if the parsed Git URL's filename contained a wildcard and use `readUrl` otherwise. `UrlReaderService` must implement this logic in the `search` method instead. + +This allows each `UrlReaderService` implementation to check whether it's a search URL (that contains a wildcard pattern) or not using logic that is specific to each provider. diff --git a/.changeset/tiny-spoons-repeat.md b/.changeset/tiny-spoons-repeat.md new file mode 100644 index 0000000000..8b6c2669bd --- /dev/null +++ b/.changeset/tiny-spoons-repeat.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-defaults': patch +--- + +In the different `UrlReadersService`, the `search` method have been updated to use the `readUrl` if the given URL doesn't contain a pattern. +For `UrlReaders` that didn't implement the `search` method, `readUrl` is now called internally and throws if the given URL doesn't contain a pattern. diff --git a/app-config.yaml b/app-config.yaml index 64c2fa5c4c..89e417fad3 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -244,6 +244,7 @@ integrations: secretAccessKey: ${AWS_SECRET_ACCESS_KEY} catalog: + useUrlReadersSearch: true import: entityFilename: catalog-info.yaml pullRequestBranchName: backstage-integration diff --git a/packages/backend-defaults/report-urlReader.api.md b/packages/backend-defaults/report-urlReader.api.md index efe4a9ae47..72e8f0bfbe 100644 --- a/packages/backend-defaults/report-urlReader.api.md +++ b/packages/backend-defaults/report-urlReader.api.md @@ -57,7 +57,10 @@ export class AwsS3UrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -230,7 +233,10 @@ export class FetchUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -265,7 +271,10 @@ export class GerritUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -293,7 +302,10 @@ export class GiteaUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } @@ -384,7 +396,10 @@ export class HarnessUrlReader implements UrlReaderService { options?: UrlReaderServiceReadUrlOptions, ): Promise; // (undocumented) - search(): Promise; + search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise; // (undocumented) toString(): string; } diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts index a17f0c6c61..5c3cfe2ff2 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts @@ -608,4 +608,56 @@ describe('AwsCodeCommitUrlReader', () => { expect(bodySubfolderFile.toString().trim()).toBe('site_name: Test2'); }); }); + + describe('search', () => { + const [{ reader }] = createReader({ + integrations: { + awsCodeCommit: [ + { + host: AMAZON_AWS_CODECOMMIT_HOST, + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + region: 'fakeregion', + }, + ], + }, + }); + + beforeEach(() => { + codeCommitClient.reset(); + + codeCommitClient.on(GetFileCommand).resolves({ + fileContent: fs.readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml', + ), + ), + commitId: `123abc`, + }); + }); + + it('should return a file when given an exact valid url', async () => { + const data = await reader.search( + 'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml', + ); + + expect(data.etag).toBe('123abc'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'site_name: Test\n', + ); + }); + + it('throws if given URL with wildcard', async () => { + await expect( + reader.search( + 'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-*.yaml', + ), + ).rejects.toThrow('Unsupported search pattern URL'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts index 5a2655ad1c..7cffe6a28b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts @@ -21,6 +21,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { @@ -31,7 +32,11 @@ import { AwsCodeCommitIntegration, ScmIntegrations, } from '@backstage/integration'; -import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + ForwardedError, + NotModifiedError, +} from '@backstage/errors'; import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { CodeCommitClient, @@ -257,7 +262,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService { } return ReadUrlResponseFactory.fromReadable( - Readable.from([response?.fileContent] || []), + Readable.from([response?.fileContent]), { etag: response.commitId, }, @@ -357,7 +362,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService { commitSpecifier: commitSpecifier, }); const response = await codeCommitClient.send(getFileCommand); - const objectData = await Readable.from([response?.fileContent] || []); + const objectData = await Readable.from([response?.fileContent]); responses.push({ data: objectData, @@ -380,8 +385,39 @@ export class AwsCodeCommitUrlReader implements UrlReaderService { } } - async search(): Promise { - throw new Error('AwsCodeCommitReader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + const { path } = parseUrl(url, true); + + if (path.match(/[*?]/)) { + throw new Error('Unsupported search pattern URL'); + } + + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts index 15f46199da..b4b1803cc1 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts @@ -527,4 +527,58 @@ describe('AwsS3UrlReader', () => { expect(body.toString().trim()).toBe('site_name: Test'); }); }); + + describe('search', () => { + const [{ reader }] = createReader({ + integrations: { + awsS3: [ + { + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }, + ], + }, + }); + + beforeEach(() => { + s3Client.reset(); + + s3Client.on(GetObjectCommand).resolves({ + Body: sdkStreamMixin( + fs.createReadStream( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), + ), + ETag: '123abc', + LastModified: new Date('2020-01-01T00:00:00Z'), + }); + }); + + it('should return a file when given an exact valid url', async () => { + const data = await reader.search( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + + expect(data.etag).toBe('123abc'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'site_name: Test\n', + ); + }); + + it('throws if given URL with wildcard', async () => { + await expect( + reader.search( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-*.yaml', + ), + ).rejects.toThrow('Unsupported search pattern URL'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts index 324538add4..5060f6c632 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { ReaderFactory, ReadTreeResponseFactory } from './types'; @@ -32,7 +33,11 @@ import { ScmIntegrations, AwsS3IntegrationConfig, } from '@backstage/integration'; -import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + ForwardedError, + NotModifiedError, +} from '@backstage/errors'; import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; import { AwsCredentialIdentityProvider } from '@aws-sdk/types'; import { @@ -357,8 +362,39 @@ export class AwsS3UrlReader implements UrlReaderService { } } - async search(): Promise { - throw new Error('AwsS3Reader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + const { path } = parseUrl(url, this.integration.config); + + if (path.match(/[*?]/)) { + throw new Error('Unsupported search pattern URL'); + } + + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts index f53a7b6c11..12a7ae8093 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts @@ -351,6 +351,17 @@ describe('AzureUrlReader', () => { ); }); + it('works for exact urls', async () => { + const result = await processor.search( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + ); + expect(result.etag).toBe(''); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + ); + }); + it('works for the naive case', async () => { const result = await processor.search( 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster', diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts index b46e7c20cd..465ec2326c 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts @@ -32,8 +32,13 @@ import { ScmIntegrations, AzureIntegration, } from '@backstage/integration'; +import parseGitUrl from 'git-url-parse'; import { Minimatch } from 'minimatch'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; @@ -181,6 +186,35 @@ export class AzureUrlReader implements UrlReaderService { url: string, options?: UrlReaderServiceSearchOptions, ): Promise { + const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const treeUrl = new URL(url); const path = treeUrl.searchParams.get('path'); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts index f5caade3fa..d2d0d28d76 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts @@ -451,5 +451,31 @@ describe('BitbucketCloudUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('should work for exact URLs', async () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBeNull(); + return res( + ctx.status(200), + ctx.body('foo'), + ctx.set('ETag', 'etag-value'), + ); + }, + ), + ); + + const result = await reader.search( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + expect(result.etag).toBe('etag-value'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + expect((await result.files[0].content()).toString()).toEqual('foo'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts index f6742c7423..c968420a5e 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts @@ -23,7 +23,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { BitbucketCloudIntegration, getBitbucketCloudDefaultBranch, @@ -163,6 +167,34 @@ export class BitbucketCloudUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const matcher = new Minimatch(filepath); // TODO(freben): For now, read the entire repo and filter through that. In diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts index 6af2858cde..5e82246298 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts @@ -30,6 +30,7 @@ import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; createMockDirectory({ mockOsTmpDir: true }); @@ -277,5 +278,23 @@ describe('BitbucketServerUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('should work for exact URLs by using readUrl directly', async () => { + reader.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + + const result = await reader.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml', + ); + expect(reader.readUrl).toHaveBeenCalledTimes(1); + expect(result.etag).toBe('etag'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml', + ); + expect((await result.files[0].content()).toString()).toEqual('content'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts index d327f80704..0a838af68b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts @@ -23,7 +23,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { BitbucketServerIntegration, getBitbucketServerDownloadUrl, @@ -168,6 +172,34 @@ export class BitbucketServerUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const matcher = new Minimatch(filepath); // TODO(freben): For now, read the entire repo and filter through that. In diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts index 54880aaa75..29217ea1f5 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts @@ -32,6 +32,7 @@ import { NotModifiedError } from '@backstage/errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import getRawBody from 'raw-body'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const logger = mockServices.logger.mock(); @@ -613,5 +614,22 @@ describe('BitbucketUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('should work for exact URLs', async () => { + hostedBitbucketProcessor.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + + const result = await hostedBitbucketProcessor.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + expect(result.etag).toBe('etag'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + expect((await result.files[0].content()).toString()).toEqual('content'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts index c12998509e..4c6de31aa4 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts @@ -23,7 +23,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { BitbucketIntegration, getBitbucketDefaultBranch, @@ -174,6 +178,34 @@ export class BitbucketUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const matcher = new Minimatch(filepath); // TODO(freben): For now, read the entire repo and filter through that. In diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts index f84422c564..b1d7018308 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts @@ -266,4 +266,34 @@ describe('FetchUrlReader', () => { ).rejects.toThrow(Error); }); }); + + describe('search', () => { + it('should return a file', async () => { + const data = await fetchUrlReader.search( + `https://backstage.io/some-resource`, + { etag: 'etag' }, + ); + expect(data.etag).toBe('foo'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe(`https://backstage.io/some-resource`); + expect((await data.files[0].content()).toString()).toEqual('content foo'); + }); + + it('should return an empty list of file if not found', async () => { + const data = await fetchUrlReader.search( + `https://backstage.io/not-exists`, + { etag: 'etag' }, + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + + it('throws if given URL with wildcard', async () => { + await expect( + fetchUrlReader.search(`https://backstage.io/some-resource*`, { + etag: 'etag', + }), + ).rejects.toThrow('Unsupported search pattern URL'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts index 0e5a279c4d..a1a61e4cdc 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts @@ -19,9 +19,14 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { ReaderFactory } from './types'; import path from 'path'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; @@ -162,8 +167,39 @@ export class FetchUrlReader implements UrlReaderService { throw new Error('FetchUrlReader does not implement readTree'); } - async search(): Promise { - throw new Error('FetchUrlReader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + const { pathname } = new URL(url); + + if (pathname.match(/[*?]/)) { + throw new Error('Unsupported search pattern URL'); + } + + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts index 2fbb85b9b1..e292b31e08 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts @@ -420,4 +420,59 @@ describe.skip('GerritUrlReader', () => { expect(response.etag).toBe(sha); }); }); + + describe('search', () => { + const responseBuffer = Buffer.from('Apache License'); + + it('should return a single file when given an exact URL', async () => { + worker.use( + rest.get( + 'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content', + (_, res, ctx) => { + return res( + ctx.status(200), + ctx.body(responseBuffer.toString('base64')), + ); + }, + ), + ); + + const data = await gerritProcessor.search( + 'https://gerrit.com/web/project/+/refs/heads/master/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://gerrit.com/web/project/+/refs/heads/master/LICENSE', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'Apache License', + ); + }); + + it('should return empty list of files for not found files.', async () => { + worker.use( + rest.get( + 'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content', + (_, res, ctx) => { + return res(ctx.status(404, 'File not found.')); + }, + ), + ); + + const data = await gerritProcessor.search( + 'https://gerrit.com/web/project/+/refs/heads/master/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + + it('throws if given URL with wildcard', async () => { + await expect( + gerritProcessor.search( + 'https://gerrit.com/web/project/+/refs/heads/master/*.yaml', + ), + ).rejects.toThrow('Unsupported search pattern URL'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts index 835624627b..0688c8c42e 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { Base64Decode } from 'base64-stream'; @@ -39,6 +40,7 @@ import { NotFoundError, NotModifiedError, ResponseError, + assertError, } from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; @@ -144,8 +146,39 @@ export class GerritUrlReader implements UrlReaderService { return this.readTreeFromGitiles(url, urlRevision, options); } - async search(): Promise { - throw new Error('GerritReader does not implement search'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + const { pathname } = new URL(url); + + if (pathname.match(/[*?]/)) { + throw new Error('Unsupported search pattern URL'); + } + + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts index 77c995c552..0c545b898c 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts @@ -330,4 +330,70 @@ describe('GiteaUrlReader', () => { ).rejects.toThrow(NotFoundError); }); }); + + describe('search', () => { + const responseBuffer = Buffer.from('Apache License'); + const giteaApiResponse = (content: any) => { + return JSON.stringify({ + encoding: 'base64', + content: Buffer.from(content).toString('base64'), + }); + }; + + it('should return a single file when given an exact URL', async () => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE', + (req, res, ctx) => { + // Test utils prefers matching URL directly but it is part of Gitea's API + if (req.url.searchParams.get('ref') === 'branch2') { + return res( + ctx.status(200), + ctx.body(giteaApiResponse(responseBuffer.toString())), + ); + } + + return res(ctx.status(500)); + }, + ), + ); + + const data = await giteaProcessor.search( + 'https://gitea.com/owner/project/src/branch/branch2/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://gitea.com/owner/project/src/branch/branch2/LICENSE', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'Apache License', + ); + }); + + it('should return empty list of files for not found files.', async () => { + worker.use( + rest.get( + 'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE', + (_, res, ctx) => { + return res(ctx.status(404, 'File not found.')); + }, + ), + ); + + const data = await giteaProcessor.search( + 'https://gitea.com/owner/project/src/branch/branch2/LICENSE', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + + it('throws if given URL with wildcard', async () => { + await expect( + giteaProcessor.search( + 'https://gitea.com/owner/project/src/branch/branch2/*.yaml', + ), + ).rejects.toThrow('Unsupported search pattern URL'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts index 502c2d2ff9..bce033c2ae 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { @@ -34,12 +35,14 @@ import { import { ReaderFactory, ReadTreeResponseFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { + assertError, AuthenticationError, NotFoundError, NotModifiedError, } from '@backstage/errors'; import { Readable } from 'stream'; import { parseLastModified } from './util'; +import parseGitUrl from 'git-url-parse'; /** * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Gitea v1 api. @@ -155,8 +158,39 @@ export class GiteaUrlReader implements UrlReaderService { }); } - search(): Promise { - throw new Error('GiteaUrlReader search not implemented.'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + const { filepath } = parseGitUrl(url); + + if (filepath.match(/[*?]/)) { + throw new Error('Unsupported search pattern URL'); + } + + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts index e8d81046ab..70b18df091 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts @@ -37,6 +37,7 @@ import { GithubUrlReader, } from './GithubUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const mockDir = createMockDirectory({ mockOsTmpDir: true }); @@ -887,9 +888,6 @@ describe('GithubUrlReader', () => { expect(r2.etag).toBe('etag123abc'); expect(r2.files.length).toBe(2); - const r3 = await reader.search(`${baseUrl}/backstage/mock/tree/main/o`); - expect(r3.files.length).toBe(0); - const r4 = await reader.search( `${baseUrl}/backstage/mock/tree/main/*docs*`, ); @@ -1017,6 +1015,23 @@ describe('GithubUrlReader', () => { await runTests(gheProcessor, 'https://ghe.github.com'); }); + it('uses readUrl when searching for an exact file', async () => { + githubProcessor.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + const data = await githubProcessor.search( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect(githubProcessor.readUrl).toHaveBeenCalledTimes(1); + expect(data.etag).toBe('etag'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect((await data.files[0].content()).toString()).toEqual('content'); + }); + it('throws NotModifiedError when same etag', async () => { await expect( githubProcessor.search( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts index 913e382f92..d78475de0b 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts @@ -37,7 +37,11 @@ import fetch, { RequestInit, Response } from 'node-fetch'; import parseGitUrl from 'git-url-parse'; import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { ReadTreeResponseFactory, ReaderFactory } from './types'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { parseLastModified } from './util'; @@ -179,6 +183,35 @@ export class GithubUrlReader implements UrlReaderService { url: string, options?: UrlReaderServiceSearchOptions, ): Promise { + const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const repoDetails = await this.getRepoDetails(url); const commitSha = repoDetails.commitSha; @@ -186,7 +219,6 @@ export class GithubUrlReader implements UrlReaderService { throw new NotModifiedError(); } - const { filepath } = parseGitUrl(url); const { headers } = await this.getCredentials(url, options); const files = await this.doSearch( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts index d0607767c2..6274ba8dc3 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts @@ -31,6 +31,7 @@ import { GitLabIntegration, readGitLabIntegrationConfig, } from '@backstage/integration'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const logger = mockServices.logger.mock(); @@ -663,6 +664,23 @@ describe('GitlabUrlReader', () => { ), ).rejects.toThrow(NotModifiedError); }); + + it('returns a single file for exact urls', async () => { + gitlabProcessor.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + const data = await gitlabProcessor.search( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect(gitlabProcessor.readUrl).toHaveBeenCalledTimes(1); + expect(data.etag).toBe('etag'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://github.com/backstage/mock/tree/main/o', + ); + expect((await data.files[0].content()).toString()).toEqual('content'); + }); }); describe('getGitlabFetchUrl', () => { diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts index caf96b877c..4be6e65b96 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts @@ -26,7 +26,11 @@ import { UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + assertError, + NotFoundError, + NotModifiedError, +} from '@backstage/errors'; import { GitLabIntegration, ScmIntegrations, @@ -244,6 +248,34 @@ export class GitlabUrlReader implements UrlReaderService { options?: UrlReaderServiceSearchOptions, ): Promise { const { filepath } = parseGitUrl(url); + + // If it's a direct URL we use readUrl instead + if (!filepath?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + const staticPart = this.getStaticPart(filepath); const matcher = new Minimatch(filepath); const treeUrl = trimEnd(url.replace(filepath, staticPart), `/`); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts index 537b8c375e..7429bae94f 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts @@ -22,6 +22,7 @@ import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; import { UrlReaderPredicateTuple } from './types'; import packageinfo from '../../../../package.json'; import { mockServices } from '@backstage/backend-test-utils'; +import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; const bucketGetFilesMock = jest.fn(); class Bucket { @@ -130,7 +131,7 @@ describe('GcsUrlReader', () => { }); it('throws if search url does not end with *', async () => { - const glob = 'https://storage.cloud.google.com/bucket/no-asterisk'; + const glob = 'https://storage.cloud.google.com/bucket/*no-asterisk'; await expect(() => reader.search(glob)).rejects.toThrow( 'GcsUrlReader only supports prefix-based searches', ); @@ -167,5 +168,22 @@ describe('GcsUrlReader', () => { 'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt', ); }); + + it('returns single file if there is no wildcard', async () => { + reader.readUrl = jest.fn().mockResolvedValue({ + buffer: async () => Buffer.from('content'), + etag: 'etag', + } as UrlReaderServiceReadUrlResponse); + const data = await reader.search( + 'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt', + ); + expect(reader.readUrl).toHaveBeenCalledTimes(1); + expect(data.etag).toBe('etag'); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt', + ); + expect((await data.files[0].content()).toString()).toEqual('content'); + }); }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts index 10440dba20..36f2e183c7 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts @@ -20,6 +20,7 @@ import { UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; import { ReaderFactory } from './types'; @@ -31,6 +32,7 @@ import { import { Readable } from 'stream'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import packageinfo from '../../../../package.json'; +import { assertError } from '@backstage/errors'; const GOOGLE_GCS_HOST = 'storage.cloud.google.com'; @@ -117,9 +119,39 @@ export class GoogleGcsUrlReader implements UrlReaderService { throw new Error('GcsUrlReader does not implement readTree'); } - async search(url: string): Promise { + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { const { bucket, key: pattern } = parseURL(url); + // If it's a direct URL we use readUrl instead + if (!pattern?.match(/[*?]/)) { + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } + } + if (!pattern.endsWith('*') || pattern.indexOf('*') !== pattern.length - 1) { throw new Error('GcsUrlReader only supports prefix-based searches'); } diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts index 004e1931d0..c5cdaa9928 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts @@ -257,4 +257,36 @@ describe('HarnessUrlReader', () => { ).rejects.toThrow(NotFoundError); }); }); + + describe('search', () => { + it('should return a single file when given an exact URL', async () => { + const data = await harnessProcessor.search( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(1); + expect(data.files[0].url).toBe( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT', + ); + expect((await data.files[0].content()).toString()).toEqual( + 'Apache License', + ); + }); + + it('should return empty list of files for not found files.', async () => { + const data = await harnessProcessor.search( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml', + ); + expect(data.etag).toBe(''); + expect(data.files.length).toBe(0); + }); + + it('throws if given URL with wildcard', async () => { + await expect( + harnessProcessor.search( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/*.yaml', + ), + ).rejects.toThrow('Unsupported search pattern URL'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts index 622d7f3379..bc2e0dab11 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts @@ -21,6 +21,7 @@ import { UrlReaderServiceReadUrlResponse, UrlReaderServiceSearchResponse, UrlReaderServiceReadTreeOptions, + UrlReaderServiceSearchOptions, } from '@backstage/backend-plugin-api'; import { getHarnessRequestOptions, @@ -35,6 +36,7 @@ import { ReadTreeResponseFactory, ReaderFactory } from './types'; import fetch, { Response } from 'node-fetch'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import { + assertError, AuthenticationError, NotFoundError, NotModifiedError, @@ -154,8 +156,39 @@ export class HarnessUrlReader implements UrlReaderService { }); } - search(): Promise { - throw new Error('HarnessUrlReader search not implemented.'); + async search( + url: string, + options?: UrlReaderServiceSearchOptions, + ): Promise { + const { path } = parseHarnessUrl(this.integration.config, url); + + if (path.match(/[*?]/)) { + throw new Error('Unsupported search pattern URL'); + } + + try { + const data = await this.readUrl(url, options); + + return { + files: [ + { + url: url, + content: data.buffer, + lastModifiedAt: data.lastModifiedAt, + }, + ], + etag: data.etag ?? '', + }; + } catch (error) { + assertError(error); + if (error.name === 'NotFoundError') { + return { + files: [], + etag: '', + }; + } + throw error; + } } toString() { diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index ca52ec530c..fc6918acea 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -103,6 +103,9 @@ catalog: # target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme-corp.yaml # rules: # - allow: [User, Group] + # Experimental: Always use the search method in UrlReaderProcessor. + # New adopters are encouraged to enable it as this behavior will be the default in a future release. + useUrlReadersSearch: true kubernetes: # see https://backstage.io/docs/features/kubernetes/configuration for kubernetes configuration options diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 384765f92e..a005e7e81a 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -202,5 +202,19 @@ export interface Config { * housing catalog-info files. */ processingInterval?: HumanDuration | false; + + /** + * Defines if the UrlReaderProcessor should always call the search method of the + * different UrlReaders. + * + * If set to false, the UrlReaderProcessor will use the legacy behavior that tries to + * parse a Git URL and calls search if there's wildcard patterns and readUrl otherwise. + * + * If set to true, the UrlReaderProcessor always call the search method and lets each UrlReader + * determine if it's a search pattern or not. + * + * This flag is temporary and will be enabled by default in future releases. + */ + useUrlReadersSearch?: boolean; }; } diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 36d10ed268..0ffd2587ca 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -103,6 +103,7 @@ "@backstage/cli": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/repo-tools": "workspace:^", + "@backstage/test-utils": "workspace:^", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", "@types/glob": "^8.0.0", diff --git a/plugins/catalog-backend/report.api.md b/plugins/catalog-backend/report.api.md index a6389185a7..06f3ca6390 100644 --- a/plugins/catalog-backend/report.api.md +++ b/plugins/catalog-backend/report.api.md @@ -499,7 +499,11 @@ export function transformLegacyPolicyToProcessor( // @public (undocumented) export class UrlReaderProcessor implements CatalogProcessor_2 { - constructor(options: { reader: UrlReaderService; logger: LoggerService }); + constructor(options: { + reader: UrlReaderService; + logger: LoggerService; + config?: Config; + }); // (undocumented) getProcessorName(): string; // (undocumented) diff --git a/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts index 4a8f982027..2d7268eb33 100644 --- a/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts @@ -18,7 +18,6 @@ import { mockServices, registerMswTestHooks, } from '@backstage/backend-test-utils'; -import { ConfigReader } from '@backstage/config'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { @@ -31,6 +30,8 @@ import { import { defaultEntityDataParser } from '../util/parse'; import { UrlReaderProcessor } from './UrlReaderProcessor'; import { UrlReaders } from '@backstage/backend-defaults/urlReader'; +import { UrlReaderService } from '@backstage/backend-plugin-api'; +import { mockApis } from '@backstage/test-utils'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; @@ -49,8 +50,10 @@ describe('UrlReaderProcessor', () => { const logger = mockServices.logger.mock(); const reader = UrlReaders.default({ logger, - config: new ConfigReader({ - backend: { reading: { allow: [{ host: 'localhost' }] } }, + config: mockServices.rootConfig({ + data: { + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }, }), }); const processor = new UrlReaderProcessor({ reader, logger }); @@ -107,8 +110,10 @@ describe('UrlReaderProcessor', () => { const logger = mockServices.logger.mock(); const reader = UrlReaders.default({ logger, - config: new ConfigReader({ - backend: { reading: { allow: [{ host: 'localhost' }] } }, + config: mockServices.rootConfig({ + data: { + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }, }), }); server.use( @@ -155,8 +160,10 @@ describe('UrlReaderProcessor', () => { const logger = mockServices.logger.mock(); const reader = UrlReaders.default({ logger, - config: new ConfigReader({ - backend: { reading: { allow: [{ host: 'localhost' }] } }, + config: mockServices.rootConfig({ + data: { + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }, }), }); const processor = new UrlReaderProcessor({ reader, logger }); @@ -180,7 +187,6 @@ describe('UrlReaderProcessor', () => { mockCache, ), )) as CatalogProcessorErrorResult; - expect(generated.type).toBe('error'); expect(generated.location).toBe(spec); expect(generated.error.name).toBe('NotFoundError'); @@ -210,4 +216,36 @@ describe('UrlReaderProcessor', () => { expect(reader.search).toHaveBeenCalledTimes(1); }); + + it('uses search when catalog.useUrlReadersSearch flag is set to true', async () => { + const logger = mockServices.logger.mock(); + + const reader: jest.Mocked = { + readUrl: jest.fn(), + readTree: jest.fn(), + search: jest.fn().mockImplementation(async () => []), + }; + + const config = mockApis.config({ + data: { + catalog: { + useUrlReadersSearch: true, + }, + }, + }); + + const processor = new UrlReaderProcessor({ reader, logger, config }); + + const emit = jest.fn(); + + await processor.readLocation( + { type: 'url', target: 'https://github.com/a/b/blob/x/b.yaml' }, + false, + emit, + defaultEntityDataParser, + mockCache, + ); + + expect(reader.search).toHaveBeenCalledTimes(1); + }); }); diff --git a/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts index e817e19616..bc383512be 100644 --- a/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/processors/UrlReaderProcessor.ts @@ -29,6 +29,7 @@ import { processingResult, } from '@backstage/plugin-catalog-node'; import { LoggerService, UrlReaderService } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; const CACHE_KEY = 'v1'; @@ -47,14 +48,25 @@ export class UrlReaderProcessor implements CatalogProcessor { // This limiter is used for only consuming a limited number of read streams // concurrently. #limiter: Limit; + #useUrlReadersSearch: boolean; constructor( private readonly options: { reader: UrlReaderService; logger: LoggerService; + config?: Config; }, ) { this.#limiter = limiterFactory(5); + + this.#useUrlReadersSearch = + this.options.config?.getOptionalBoolean('catalog.useUrlReadersSearch') || + false; + if (!this.#useUrlReadersSearch) { + this.options.logger.warn( + 'UrlReaderProcessor uses the legacy readUrl/search behavior which will be removed in a future release. Set catalog.useUrlReadersSearch to true to adopt the new behavior.', + ); + } } getProcessorName() { @@ -80,6 +92,15 @@ export class UrlReaderProcessor implements CatalogProcessor { cacheItem?.etag, ); + if (response.length === 0 && !optional) { + emit( + processingResult.notFoundError( + location, + `Unable to read ${location.type}, no matching files found for ${location.target}`, + ), + ); + } + const parseResults: CatalogProcessorResult[] = []; for (const item of response) { for await (const parseResult of parser({ @@ -128,7 +149,19 @@ export class UrlReaderProcessor implements CatalogProcessor { location: string, etag?: string, ): Promise<{ response: { data: Buffer; url: string }[]; etag?: string }> { - // Does it contain globs? I.e. does it contain asterisks or question marks + // New behavior: always use the search method + if (this.#useUrlReadersSearch) { + const response = await this.options.reader.search(location, { etag }); + + const output = response.files.map(async file => ({ + url: file.url, + data: await this.#limiter(file.content), + })); + + return { response: await Promise.all(output), etag: response.etag }; + } + + // Old behavior: Does it contain globs? I.e. does it contain asterisks or question marks // (no curly braces for now) const { filepath } = parseGitUrl(location); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 7f70e2732d..ddf4b5d185 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -381,7 +381,7 @@ export class CatalogBuilder { return [ new FileReaderProcessor(), - new UrlReaderProcessor({ reader, logger }), + new UrlReaderProcessor({ reader, logger, config }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), ]; diff --git a/yarn.lock b/yarn.lock index e995d85226..94746fde49 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6136,6 +6136,7 @@ __metadata: "@backstage/plugin-search-backend-module-catalog": "workspace:^" "@backstage/plugin-search-common": "workspace:^" "@backstage/repo-tools": "workspace:^" + "@backstage/test-utils": "workspace:^" "@backstage/types": "workspace:^" "@opentelemetry/api": ^1.9.0 "@types/core-js": ^2.5.4