From 58b3bbe4e8210352cd7df6a2bc79e20f57a7859d Mon Sep 17 00:00:00 2001 From: "Kim S. Ly" Date: Wed, 9 Dec 2020 02:02:09 -0500 Subject: [PATCH] Implement readTree on BitBucketUrlReader - Implement getBitbucketDownloadUrl and remove hash for Bitbucket Server --- .../src/reading/BitbucketUrlReader.test.ts | 22 ++++--- .../src/reading/BitbucketUrlReader.ts | 55 +++++++----------- .../__fixtures__/bitbucket-server-repo.zip | Bin 0 -> 673 bytes .../integration/src/bitbucket/core.test.ts | 50 +++++++++++++++- packages/integration/src/bitbucket/core.ts | 32 ++++++++++ packages/integration/src/bitbucket/index.ts | 6 +- 6 files changed, 118 insertions(+), 47 deletions(-) create mode 100644 packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index d7e4632a69..03d85e813c 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -98,24 +98,22 @@ describe('BitbucketUrlReader', () => { }); it('uses private bitbucket host', async () => { + const privateBitbucketRepoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'bitbucket-server-repo.zip', + ), + ); worker.use( rest.get( - 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/archive?format=tgz', + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs', (_, res, ctx) => res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), - ctx.body(repoBuffer), - ), - ), - rest.get( - 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits/?until=master', - (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], - }), + ctx.body(privateBitbucketRepoBuffer), ), ), ); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 982422817a..c92174f1e4 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -16,6 +16,7 @@ import { BitbucketIntegrationConfig, + getBitbucketDownloadUrl, getBitbucketFileFetchUrl, getBitbucketRequestOptions, readBitbucketIntegrationConfigs, @@ -100,7 +101,7 @@ export class BitbucketUrlReader implements UrlReader { options?: ReadTreeOptions, ): Promise { const gitUrl: parseGitUri.GitUrl = parseGitUri(url); - const { name: repoName, owner, ref, protocol, resource, filepath } = gitUrl; + const { name: repoName, owner: project, ref, resource, filepath } = gitUrl; const isHosted = resource === 'bitbucket.org'; @@ -111,11 +112,10 @@ export class BitbucketUrlReader implements UrlReader { ); } - const archiveUrl = isHosted - ? `${protocol}://${resource}/${owner}/${repoName}/get/${ref}.zip` - : `${protocol}://${resource}/projects/${owner}/repos/${repoName}/archive?format=zip`; - - const response = await fetch(archiveUrl, getApiRequestOptions(this.config)); + const response = await fetch( + getBitbucketDownloadUrl(url, this.config), + getBitbucketRequestOptions(this.config), + ); if (!response.ok) { const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; if (response.status === 404) { @@ -124,11 +124,15 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } - const lastCommitShortHash = await this.getLastCommitShortHash(gitUrl); + let folderPath = `${project}-${repoName}`; + if (isHosted) { + const lastCommitShortHash = await this.getLastCommitShortHash(gitUrl); + folderPath = `${project}-${repoName}-${lastCommitShortHash}`; + } return this.treeResponseFactory.fromZipArchive({ stream: (response.body as unknown) as Readable, - path: `${owner}-${repoName}-${lastCommitShortHash}/${filepath}`, + path: `${folderPath}/${filepath}`, filter: options?.filter, }); } @@ -145,18 +149,14 @@ export class BitbucketUrlReader implements UrlReader { private async getLastCommitShortHash( gitUrl: parseGitUri.GitUrl, ): Promise { - const { name: repoName, owner, ref, protocol, resource } = gitUrl; - - const isHosted = resource === 'bitbucket.org'; + const { name: repoName, owner: project, ref } = gitUrl; const branch = ref ? ref : 'master'; - const commitsApiUrl = isHosted - ? `${protocol}://${this.config.apiBaseUrl}/repositories/${owner}/${repoName}/commits/${branch}` - : `${protocol}://${this.config.apiBaseUrl}/projects/${owner}/repos/${repoName}/commits/?until=${branch}`; + const commitsApiUrl = `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`; const commitsResponse = await fetch( commitsApiUrl, - getApiRequestOptions(this.config), + getBitbucketRequestOptions(this.config), ); if (!commitsResponse.ok) { const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`; @@ -167,24 +167,13 @@ export class BitbucketUrlReader implements UrlReader { } const commits = await commitsResponse.json(); - if (isHosted) { - if ( - commits && - commits.values && - commits.values.length > 0 && - commits.values[0].hash - ) { - return commits.values[0].hash.substring(0, 12); - } - } else { - if ( - commits && - commits.values && - commits.values.length > 0 && - commits.values[0].id - ) { - return commits.values[0].id.substring(0, 12); - } + if ( + commits && + commits.values && + commits.values.length > 0 && + commits.values[0].hash + ) { + return commits.values[0].hash.substring(0, 12); } throw new Error(`Failed to read response from ${commitsApiUrl}`); } diff --git a/packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip b/packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip new file mode 100644 index 0000000000000000000000000000000000000000..be6b20d12708da4e910b4540793ba1728a555904 GIT binary patch literal 673 zcmWIWW@h1H00E1=ErDPLl;CF&U`R?#&Mq!VOi$I#%}>tO4~^hq(8{_H-@|`8{wt#h zLjX?ABAA*}@{^0Pnjpf!0fgIHqfxAv!C^vXUP@|(UTzA=ErLL|{Aim2c1v&=ho`RI zLnff3*g2+&oR8oH`hy*aL5_r2u>s8rNlYtpvq8?(tIW-Tng0fAzV$Z`=lnFd!Rbi)j!vn9)r` z1Tu2i$RL@vq_G}{X^0@_fCo9mjfikX4hTsUQy*cPiX6%iBSE1Y;LXYgRLj5ugbs`h K4BkM@zyJVY-kw|l literal 0 HcmV?d00001 diff --git a/packages/integration/src/bitbucket/core.test.ts b/packages/integration/src/bitbucket/core.test.ts index 8d9f956c6b..1af6ff3d56 100644 --- a/packages/integration/src/bitbucket/core.test.ts +++ b/packages/integration/src/bitbucket/core.test.ts @@ -15,7 +15,11 @@ */ import { BitbucketIntegrationConfig } from './config'; -import { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core'; +import { + getBitbucketDownloadUrl, + getBitbucketFileFetchUrl, + getBitbucketRequestOptions, +} from './core'; describe('bitbucket core', () => { describe('getBitbucketRequestOptions', () => { @@ -97,4 +101,48 @@ describe('bitbucket core', () => { ); }); }); + + describe('getBitbucketDownloadUrl', () => { + it('add path param if a path is specified', () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }; + const result = getBitbucketDownloadUrl( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs', + config, + ); + expect(result).toEqual( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=backstage-mock&path=docs', + ); + }); + + it('do not add path param if no path is specified', () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }; + const result = getBitbucketDownloadUrl( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse', + config, + ); + expect(new URL(result).searchParams.get('format')).toEqual('zip'); + expect(new URL(result).searchParams.get('prefix')).not.toBeNull(); + expect(new URL(result).searchParams.get('path')).toBeNull(); + }); + + it('do not add path param if the repository is hosted on bitbucket.org', () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }; + const result = getBitbucketDownloadUrl( + 'https://bitbucket.org/backstage/mock/src/master', + config, + ); + expect(result).toEqual( + 'https://bitbucket.org/backstage/mock/get/master.zip', + ); + }); + }); }); diff --git a/packages/integration/src/bitbucket/core.ts b/packages/integration/src/bitbucket/core.ts index a522e7ce9f..0d6dd392c7 100644 --- a/packages/integration/src/bitbucket/core.ts +++ b/packages/integration/src/bitbucket/core.ts @@ -17,6 +17,38 @@ import parseGitUrl from 'git-url-parse'; import { BitbucketIntegrationConfig } from './config'; +/** + * Given a URL pointing to a path on a provider, returns a URL that is suitable + * for downloading the subtree. + * + * @param url A URL pointing to a path + */ +export function getBitbucketDownloadUrl( + url: string, + config: BitbucketIntegrationConfig, +): string { + const { + name: repoName, + owner: project, + ref, + protocol, + resource, + filepath, + } = parseGitUrl(url); + + const isHosted = resource === 'bitbucket.org'; + + // path will limit the downloaded content + // /docs will only download the docs folder and everything below it + // /docs/index.md will download the docs folder and everything below it + const path = filepath ? `&path=${encodeURIComponent(filepath)}` : ''; + const archiveUrl = isHosted + ? `${protocol}://${resource}/${project}/${repoName}/get/${ref}.zip` + : `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/archive?format=zip&prefix=${project}-${repoName}${path}`; + + return archiveUrl; +} + /** * Given a URL pointing to a file on a provider, returns a URL that is suitable * for fetching the contents of the data. diff --git a/packages/integration/src/bitbucket/index.ts b/packages/integration/src/bitbucket/index.ts index b8d37220db..d164182aee 100644 --- a/packages/integration/src/bitbucket/index.ts +++ b/packages/integration/src/bitbucket/index.ts @@ -19,4 +19,8 @@ export { readBitbucketIntegrationConfigs, } from './config'; export type { BitbucketIntegrationConfig } from './config'; -export { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core'; +export { + getBitbucketDownloadUrl, + getBitbucketFileFetchUrl, + getBitbucketRequestOptions, +} from './core';