diff --git a/.changeset/grumpy-elephants-press.md b/.changeset/grumpy-elephants-press.md new file mode 100644 index 0000000000..d85d36e609 --- /dev/null +++ b/.changeset/grumpy-elephants-press.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Implement readTree on BitBucketUrlReader and getBitbucketDownloadUrl diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 2c8549f917..616cbaaadc 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -158,9 +158,7 @@ describe('AzureUrlReader', () => { it('returns the wanted files from an archive', async () => { const processor = new AzureUrlReader( - { - host: 'dev.azure.com', - }, + { host: 'dev.azure.com' }, { treeResponseFactory }, ); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 1c0bd39372..e327770114 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,15 +14,26 @@ * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; +import fs from 'fs'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import path from 'path'; import { BitbucketUrlReader } from './BitbucketUrlReader'; +import { ReadTreeResponseFactory } from './tree'; + +const treeResponseFactory = ReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); describe('BitbucketUrlReader', () => { describe('implementation', () => { it('rejects unknown targets', async () => { - const processor = new BitbucketUrlReader({ - host: 'bitbucket.org', - apiBaseUrl: 'https://api.bitbucket.org/2.0', - }); + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, + { treeResponseFactory }, + ); await expect( processor.read('https://not.bitbucket.com/apa'), ).rejects.toThrow( @@ -30,4 +41,141 @@ describe('BitbucketUrlReader', () => { ); }); }); + + describe('readTree', () => { + const worker = setupServer(); + msw.setupDefaultHandlers(worker); + + const repoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'bitbucket-repo-with-commit-hash.zip', + ), + ); + + it('returns the wanted files from an archive', async () => { + worker.use( + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://bitbucket.org/backstage/mock/src/master', + ); + + const files = await response.files(); + + expect(files.length).toBe(2); + const indexMarkdownFile = await files[0].content(); + const mkDocsFile = await files[1].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + }); + + it('uses private bitbucket host', async () => { + const privateBitbucketRepoBuffer = fs.readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'bitbucket-server-repo.zip', + ), + ); + worker.use( + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(privateBitbucketRepoBuffer), + ), + ), + ); + + const processor = new BitbucketUrlReader( + { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + worker.use( + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://bitbucket.org/backstage/mock/src/master/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + }); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 8c97bf2ee2..4367424423 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -16,13 +16,23 @@ import { BitbucketIntegrationConfig, + getBitbucketDefaultBranch, + getBitbucketDownloadUrl, getBitbucketFileFetchUrl, getBitbucketRequestOptions, readBitbucketIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; +import parseGitUri from 'git-url-parse'; +import { Readable } from 'stream'; import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import { ReadTreeResponseFactory } from './tree'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + UrlReader, +} from './types'; /** * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as @@ -30,19 +40,23 @@ import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; */ export class BitbucketUrlReader implements UrlReader { private readonly config: BitbucketIntegrationConfig; + private readonly treeResponseFactory: ReadTreeResponseFactory; - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readBitbucketIntegrationConfigs( config.getOptionalConfigArray('integrations.bitbucket') ?? [], ); return configs.map(provider => { - const reader = new BitbucketUrlReader(provider); + const reader = new BitbucketUrlReader(provider, { treeResponseFactory }); const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); }; - constructor(config: BitbucketIntegrationConfig) { + constructor( + config: BitbucketIntegrationConfig, + deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) { const { host, apiBaseUrl, token, username, appPassword } = config; if (!apiBaseUrl) { @@ -58,6 +72,7 @@ export class BitbucketUrlReader implements UrlReader { } this.config = config; + this.treeResponseFactory = deps.treeResponseFactory; } async read(url: string): Promise { @@ -82,8 +97,39 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('BitbucketUrlReader does not implement readTree'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const gitUrl: parseGitUri.GitUrl = parseGitUri(url); + const { name: repoName, owner: project, resource, filepath } = gitUrl; + + const isHosted = resource === 'bitbucket.org'; + + const downloadUrl = await getBitbucketDownloadUrl(url, this.config); + const response = await fetch( + downloadUrl, + getBitbucketRequestOptions(this.config), + ); + if (!response.ok) { + const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + let folderPath = `${project}-${repoName}`; + if (isHosted) { + const lastCommitShortHash = await this.getLastCommitShortHash(url); + folderPath = `${project}-${repoName}-${lastCommitShortHash}`; + } + + return this.treeResponseFactory.fromZipArchive({ + stream: (response.body as unknown) as Readable, + path: `${folderPath}/${filepath}`, + filter: options?.filter, + }); } toString() { @@ -94,4 +140,37 @@ export class BitbucketUrlReader implements UrlReader { } return `bitbucket{host=${host},authed=${authed}}`; } + + private async getLastCommitShortHash(url: string): Promise { + const { name: repoName, owner: project, ref } = parseGitUri(url); + + let branch = ref; + if (!branch) { + branch = await getBitbucketDefaultBranch(url, this.config); + } + const commitsApiUrl = `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`; + + const commitsResponse = await fetch( + commitsApiUrl, + getBitbucketRequestOptions(this.config), + ); + if (!commitsResponse.ok) { + const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`; + if (commitsResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + const commits = await commitsResponse.json(); + if ( + commits && + commits.values && + commits.values.length > 0 && + commits.values[0].hash + ) { + return commits.values[0].hash.substring(0, 12); + } + throw new Error(`Failed to read response from ${commitsApiUrl}`); + } } diff --git a/packages/backend-common/src/reading/__fixtures__/bitbucket-repo-with-commit-hash.zip b/packages/backend-common/src/reading/__fixtures__/bitbucket-repo-with-commit-hash.zip new file mode 100644 index 0000000000..135860afd3 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/bitbucket-repo-with-commit-hash.zip differ diff --git a/packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip b/packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip new file mode 100644 index 0000000000..be6b20d127 Binary files /dev/null and b/packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip differ diff --git a/packages/integration/package.json b/packages/integration/package.json index ccda8ea025..78112de311 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -35,6 +35,7 @@ }, "devDependencies": { "@backstage/cli": "^0.4.1", + "@backstage/test-utils": "^0.1.5", "@types/jest": "^26.0.7", "msw": "^0.21.2" }, diff --git a/packages/integration/src/bitbucket/core.test.ts b/packages/integration/src/bitbucket/core.test.ts index 8d9f956c6b..c6fe5ddf6c 100644 --- a/packages/integration/src/bitbucket/core.test.ts +++ b/packages/integration/src/bitbucket/core.test.ts @@ -14,10 +14,21 @@ * limitations under the License. */ +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { msw } from '@backstage/test-utils'; import { BitbucketIntegrationConfig } from './config'; -import { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core'; +import { + getBitbucketDefaultBranch, + getBitbucketDownloadUrl, + getBitbucketFileFetchUrl, + getBitbucketRequestOptions, +} from './core'; describe('bitbucket core', () => { + const worker = setupServer(); + msw.setupDefaultHandlers(worker); + describe('getBitbucketRequestOptions', () => { it('inserts a token when needed', () => { const withToken: BitbucketIntegrationConfig = { @@ -97,4 +108,148 @@ describe('bitbucket core', () => { ); }); }); + + describe('getBitbucketDownloadUrl', () => { + it('add path param if a path is specified for Bitbucket Server', async () => { + const defaultBranchResponse = { + displayId: 'main', + }; + worker.use( + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(defaultBranchResponse), + ), + ), + ); + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }; + const result = await getBitbucketDownloadUrl( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs', + config, + ); + expect(result).toEqual( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&at=main&prefix=backstage-mock&path=docs', + ); + }); + + it('do not add path param if no path is specified for Bitbucket Server', async () => { + const defaultBranchResponse = { + displayId: 'main', + }; + worker.use( + rest.get( + // TODO: Change URL to https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default when git-url-parse bug is fixed + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/projects/backstage/repos/mock/repos/browse/branches/default', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(defaultBranchResponse), + ), + ), + ); + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }; + const result = await getBitbucketDownloadUrl( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse?at=main', + config, + ); + expect(new URL(result).searchParams.get('format')).toEqual('zip'); + expect(new URL(result).searchParams.get('at')).toEqual('main'); + expect(new URL(result).searchParams.get('prefix')).not.toBeNull(); + expect(new URL(result).searchParams.get('path')).toBeNull(); + }); + + it('get by branch for Bitbucket Server', async () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }; + const result = await getBitbucketDownloadUrl( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch', + config, + ); + expect(result).toEqual( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&at=some-branch&prefix=backstage-mock&path=docs', + ); + }); + + it('do not add path param for Bitbucket Cloud', async () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }; + const result = await getBitbucketDownloadUrl( + 'https://bitbucket.org/backstage/mock/src/master', + config, + ); + expect(result).toEqual( + 'https://bitbucket.org/backstage/mock/get/master.zip', + ); + }); + }); + + describe('getBitbucketDefaultBranch', () => { + it('return default branch for Bitbucket Cloud', async () => { + const repoInfoResponse = { + mainbranch: { + name: 'main', + }, + }; + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(repoInfoResponse), + ), + ), + ); + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }; + const defaultBranch = await getBitbucketDefaultBranch( + 'https://bitbucket.org/backstage/mock/src/main', + config, + ); + expect(defaultBranch).toEqual('main'); + }); + + it('return default branch for Bitbucket Server', async () => { + const defaultBranchResponse = { + displayId: 'main', + }; + worker.use( + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json(defaultBranchResponse), + ), + ), + ); + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }; + const defaultBranch = await getBitbucketDefaultBranch( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/README.md', + config, + ); + expect(defaultBranch).toEqual('main'); + }); + }); }); diff --git a/packages/integration/src/bitbucket/core.ts b/packages/integration/src/bitbucket/core.ts index a522e7ce9f..ae61df497d 100644 --- a/packages/integration/src/bitbucket/core.ts +++ b/packages/integration/src/bitbucket/core.ts @@ -14,9 +14,84 @@ * limitations under the License. */ +import fetch from 'cross-fetch'; import parseGitUrl from 'git-url-parse'; import { BitbucketIntegrationConfig } from './config'; +/** + * Given a URL pointing to a path on a provider, returns the default branch. + * + * @param url A URL pointing to a path + * @param config The relevant provider config + */ +export async function getBitbucketDefaultBranch( + url: string, + config: BitbucketIntegrationConfig, +): Promise { + const { name: repoName, owner: project, resource } = parseGitUrl(url); + + const isHosted = resource === 'bitbucket.org'; + const branchUrl = isHosted + ? `${config.apiBaseUrl}/repositories/${project}/${repoName}` + : `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches/default`; + + const response = await fetch(branchUrl, getBitbucketRequestOptions(config)); + if (!response.ok) { + const message = `Failed to retrieve default branch from ${branchUrl}, ${response.status} ${response.statusText}`; + throw new Error(message); + } + + let defaultBranch; + if (isHosted) { + const repoInfo = await response.json(); + defaultBranch = repoInfo.mainbranch.name; + } else { + const { displayId } = await response.json(); + defaultBranch = displayId; + } + if (!defaultBranch) { + throw new Error(`Failed to read default branch from ${branchUrl}`); + } + return defaultBranch; +} + +/** + * Given a URL pointing to a path on a provider, returns a URL that is suitable + * for downloading the subtree. + * + * @param url A URL pointing to a path + * @param config The relevant provider config + */ +export async function getBitbucketDownloadUrl( + url: string, + config: BitbucketIntegrationConfig, +): Promise { + const { + name: repoName, + owner: project, + ref, + protocol, + resource, + filepath, + } = parseGitUrl(url); + + const isHosted = resource === 'bitbucket.org'; + + let branch = ref; + if (!branch) { + branch = await getBitbucketDefaultBranch(url, config); + } + // path will limit the downloaded content + // /docs will only download the docs folder and everything below it + // /docs/index.md will download the docs folder and everything below it + const path = filepath ? `&path=${encodeURIComponent(filepath)}` : ''; + const archiveUrl = isHosted + ? `${protocol}://${resource}/${project}/${repoName}/get/${branch}.zip` + : `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/archive?format=zip&at=${branch}&prefix=${project}-${repoName}${path}`; + + return archiveUrl; +} + /** * Given a URL pointing to a file on a provider, returns a URL that is suitable * for fetching the contents of the data. diff --git a/packages/integration/src/bitbucket/index.ts b/packages/integration/src/bitbucket/index.ts index b8d37220db..9df4d3d3fe 100644 --- a/packages/integration/src/bitbucket/index.ts +++ b/packages/integration/src/bitbucket/index.ts @@ -19,4 +19,9 @@ export { readBitbucketIntegrationConfigs, } from './config'; export type { BitbucketIntegrationConfig } from './config'; -export { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core'; +export { + getBitbucketDefaultBranch, + getBitbucketDownloadUrl, + getBitbucketFileFetchUrl, + getBitbucketRequestOptions, +} from './core';