diff --git a/.changeset/purple-olives-destroy.md b/.changeset/purple-olives-destroy.md new file mode 100644 index 0000000000..3578f5fdf5 --- /dev/null +++ b/.changeset/purple-olives-destroy.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +URL Reader: Use API response headers for archive filename in readTree. Fixes bug for users with hosted Bitbucket. diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 59e437607f..578db2ac92 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -76,6 +76,8 @@ export class AzureUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { + // TODO: Support filepath based reading tree feature like other providers + // Get latest commit SHA const commitsAzureResponse = await fetch( diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 3542e822e1..9661368b5e 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -95,6 +95,10 @@ describe('BitbucketUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-12ab34cd56ef.zip', + ), ctx.body(repoBuffer), ), ), @@ -114,6 +118,10 @@ describe('BitbucketUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.zip', + ), ctx.body(privateBitbucketRepoBuffer), ), ), diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 869870f248..e9727e04cf 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -101,17 +101,13 @@ export class BitbucketUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { name: repoName, owner: project, resource, filepath } = parseGitUrl( - url, - ); + const { filepath } = parseGitUrl(url); const lastCommitShortHash = await this.getLastCommitShortHash(url); if (options?.etag && options.etag === lastCommitShortHash) { throw new NotModifiedError(); } - const isHosted = resource === 'bitbucket.org'; - const downloadUrl = await getBitbucketDownloadUrl(url, this.config); const archiveBitbucketResponse = await fetch( downloadUrl, @@ -125,14 +121,31 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } - let folderPath = `${project}-${repoName}`; - if (isHosted) { - folderPath = `${project}-${repoName}-${lastCommitShortHash}`; + // Get the filename of archive from the header of the response + const contentDispositionHeader = archiveBitbucketResponse.headers.get( + 'content-disposition', + ) as string; + if (!contentDispositionHeader) { + throw new Error( + `Failed to read tree from ${url}. ` + + 'Bitbucket API response for downloading archive does not contain content-disposition header ', + ); + } + const fileNameRegEx = new RegExp( + /^attachment; filename=(?.*).zip$/, + ); + const archiveFileName = contentDispositionHeader.match(fileNameRegEx) + ?.groups?.fileName; + if (!archiveFileName) { + throw new Error( + `Failed to read tree from ${url}. Bitbucket API response for downloading archive has an unexpected ` + + `format of content-disposition header ${contentDispositionHeader} `, + ); } return await this.treeResponseFactory.fromZipArchive({ stream: (archiveBitbucketResponse.body as unknown) as Readable, - path: `${folderPath}/${filepath}`, + path: `${archiveFileName}/${filepath}`, etag: lastCommitShortHash, filter: options?.filter, }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 0b7d56480f..080e8b1d5b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -165,6 +165,10 @@ describe('GithubUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), ctx.body(repoBuffer), ), ), @@ -178,6 +182,10 @@ describe('GithubUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), ctx.body(repoBuffer), ), ), @@ -244,6 +252,10 @@ describe('GithubUrlReader', () => { return res( ctx.status(200), ctx.set('Content-Type', 'application/x-gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-etag123.tar.gz', + ), ctx.body(repoBuffer), ); }, diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index b5b4e9c1a7..6c7cefe2ef 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -166,18 +166,31 @@ export class GithubUrlReader implements UrlReader { throw new Error(message); } - // Note that repoResponseJson.full_name must be used over full_name because the path - // is case sensitive and full_name may not be inq the correct case. - // TODO(OrkoHunter): The directory name inside the tarball should be retrieved from the tar - // instead of being constructed here. Same goes for GitLab, Bitbucket and Azure. - const extractedDirName = `${repoResponseJson.full_name.replace( - '/', - '-', - )}-${commitSha.substr(0, 7)}`; + // Get the filename of archive from the header of the response + const contentDispositionHeader = archive.headers.get( + 'content-disposition', + ) as string; + if (!contentDispositionHeader) { + throw new Error( + `Failed to read tree from ${url}. ` + + 'GitHub API response for downloading archive does not contain content-disposition header ', + ); + } + const fileNameRegEx = new RegExp( + /^attachment; filename=(?.*).tar.gz$/, + ); + const archiveFileName = contentDispositionHeader.match(fileNameRegEx) + ?.groups?.fileName; + if (!archiveFileName) { + throw new Error( + `Failed to read tree from ${url}. GitHub API response for downloading archive has an unexpected ` + + `format of content-disposition header ${contentDispositionHeader} `, + ); + } // The path includes the name of the directory inside the tarball and a sub path // if requested in readTree. - const path = `${extractedDirName}/${filepath}`; + const path = `${archiveFileName}/${filepath}`; return await this.deps.treeResponseFactory.fromTarArchive({ // TODO(Rugvip): Underlying implementation of fetch will be node-fetch, we probably want diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 2e66794397..c0736f769d 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -176,6 +176,10 @@ describe('GitlabUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), ctx.body(archiveBuffer), ), ), @@ -225,6 +229,10 @@ describe('GitlabUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), ctx.body(archiveBuffer), ), ), @@ -254,6 +262,10 @@ describe('GitlabUrlReader', () => { res( ctx.status(200), ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename="mock-main-sha123abc.zip"', + ), ctx.body(archiveBuffer), ), ), diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 72db901ac1..654f4f9a85 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -78,7 +78,7 @@ export class GitlabUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { - const { name: repoName, ref, full_name, filepath } = parseGitUrl(url); + const { ref, full_name, filepath } = parseGitUrl(url); // Use GitLab API to get the default branch // encodeURIComponent is required for GitLab API @@ -140,9 +140,29 @@ export class GitlabUrlReader implements UrlReader { throw new Error(message); } - const path = filepath - ? `${repoName}-${branch}-${commitSha}/${filepath}/` - : ''; + // Get the filename of archive from the header of the response + const contentDispositionHeader = archiveGitLabResponse.headers.get( + 'content-disposition', + ) as string; + if (!contentDispositionHeader) { + throw new Error( + `Failed to read tree from ${url}. ` + + 'GitLab API response for downloading archive does not contain content-disposition header ', + ); + } + const fileNameRegEx = new RegExp( + /^attachment; filename="(?.*).zip"$/, + ); + const archiveFileName = contentDispositionHeader.match(fileNameRegEx) + ?.groups?.fileName; + if (!archiveFileName) { + throw new Error( + `Failed to read tree from ${url}. GitLab API response for downloading archive has an unexpected ` + + `format of content-disposition header ${contentDispositionHeader} `, + ); + } + + const path = filepath ? `${archiveFileName}/${filepath}/` : ''; return await this.treeResponseFactory.fromZipArchive({ stream: (archiveGitLabResponse.body as unknown) as Readable,