From 43e1277796abb064d9670caee7d06c6998f8cc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 4 Dec 2020 08:46:43 +0100 Subject: [PATCH] WIP: implement readTree for bitbucket --- .../src/reading/AzureUrlReader.test.ts | 4 +- .../src/reading/BitbucketUrlReader.test.ts | 133 +++++++++++++++++- .../src/reading/BitbucketUrlReader.ts | 66 ++++++++- 3 files changed, 189 insertions(+), 14 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 2c8549f917..616cbaaadc 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -158,9 +158,7 @@ describe('AzureUrlReader', () => { it('returns the wanted files from an archive', async () => { const processor = new AzureUrlReader( - { - host: 'dev.azure.com', - }, + { host: 'dev.azure.com' }, { treeResponseFactory }, ); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 1c0bd39372..76db191909 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,15 +14,26 @@ * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; +import { msw } from '@backstage/test-utils'; +import fs from 'fs'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import path from 'path'; import { BitbucketUrlReader } from './BitbucketUrlReader'; +import { ReadTreeResponseFactory } from './tree'; + +const treeResponseFactory = ReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); describe('BitbucketUrlReader', () => { describe('implementation', () => { it('rejects unknown targets', async () => { - const processor = new BitbucketUrlReader({ - host: 'bitbucket.org', - apiBaseUrl: 'https://api.bitbucket.org/2.0', - }); + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' }, + { treeResponseFactory }, + ); await expect( processor.read('https://not.bitbucket.com/apa'), ).rejects.toThrow( @@ -30,4 +41,118 @@ describe('BitbucketUrlReader', () => { ); }); }); + + describe('readTree', () => { + const worker = setupServer(); + msw.setupDefaultHandlers(worker); + + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.zip'), + ); + + it('returns the wanted files from an archive', async () => { + worker.use( + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'x' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://bitbucket.org/backstage/mock/src/master', + ); + + const files = await response.files(); + + expect(files.length).toBe(2); + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('uses private bitbucket host', async () => { + worker.use( + rest.get( + 'https://bitbucket.mycompany.net/projects/a/repos/b/archive?format=tgz', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + + const processor = new BitbucketUrlReader( + { host: 'bitbucket.mycompany.net', apiBaseUrl: 'x' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://bitbucket.mycompany.net/projects/a/repos/b/browse/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('must specify a branch', async () => { + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'x' }, + { treeResponseFactory }, + ); + + await expect( + processor.readTree('https://bitbucket.org/backstage/mock'), + ).rejects.toThrow( + 'Bitbucket URL must contain branch to be able to fetch tree', + ); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + worker.use( + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.zip', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.body(repoBuffer), + ), + ), + ); + + const processor = new BitbucketUrlReader( + { host: 'bitbucket.org', apiBaseUrl: 'x' }, + { treeResponseFactory }, + ); + + const response = await processor.readTree( + 'https://bitbucket.org/backstage/mock/src/master/docs', + ); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + }); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 8c97bf2ee2..00a386d4cc 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -21,8 +21,16 @@ import { readBitbucketIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import { NotFoundError } from '../errors'; -import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; +import parseGitUri from 'git-url-parse'; +import { Readable } from 'stream'; +import { InputError, NotFoundError } from '../errors'; +import { ReadTreeResponseFactory } from './tree'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + UrlReader, +} from './types'; /** * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as @@ -30,19 +38,23 @@ import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; */ export class BitbucketUrlReader implements UrlReader { private readonly config: BitbucketIntegrationConfig; + private readonly treeResponseFactory: ReadTreeResponseFactory; - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readBitbucketIntegrationConfigs( config.getOptionalConfigArray('integrations.bitbucket') ?? [], ); return configs.map(provider => { - const reader = new BitbucketUrlReader(provider); + const reader = new BitbucketUrlReader(provider, { treeResponseFactory }); const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); }; - constructor(config: BitbucketIntegrationConfig) { + constructor( + config: BitbucketIntegrationConfig, + deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) { const { host, apiBaseUrl, token, username, appPassword } = config; if (!apiBaseUrl) { @@ -58,6 +70,7 @@ export class BitbucketUrlReader implements UrlReader { } this.config = config; + this.treeResponseFactory = deps.treeResponseFactory; } async read(url: string): Promise { @@ -82,8 +95,47 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('BitbucketUrlReader does not implement readTree'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const { + name: repoName, + owner, + ref, + protocol, + resource, + // filepath, + } = parseGitUri(url); + + const isHosted = resource === 'bitbucket.org'; + + if (isHosted && !ref) { + // TODO(freben): We should add support for defaulting to the default branch + throw new InputError( + 'Bitbucket URL must contain branch to be able to fetch tree', + ); + } + + const archiveUrl = isHosted + ? `${protocol}://${resource}/${owner}/${repoName}/get/${ref}.zip` + : `${protocol}://${resource}/projects/${owner}/repos/${repoName}/archive?format=zip`; + + const response = await fetch(archiveUrl, getApiRequestOptions(this.config)); + if (!response.ok) { + const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return this.treeResponseFactory.fromZipArchive({ + stream: (response.body as unknown) as Readable, + // TODO: The zip contains the commit hash, not branch name - may need additional api call to get it + // path: `${owner}-${repoName}-4f9778cd49a4/${filepath}`, + filter: options?.filter, + }); } toString() {