From 75bf9e1da920b1e59dd21f14b17d4293e26148ba Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Sun, 10 Apr 2022 02:12:25 +0200 Subject: [PATCH] feat: split BitbucketUrlReader Split `BitbucketUrlReader` into `BitbucketCloudUrlReader` and `BitbucketServerUrlReader` while staying backwards compatible and conflict free (== always only one of the readers will apply). Relates-to: #9923 Signed-off-by: Patrick Jungermann --- .changeset/dirty-eagles-hunt.md | 5 + packages/backend-common/api-report.md | 47 +++ .../reading/BitbucketCloudUrlReader.test.ts | 362 ++++++++++++++++++ .../src/reading/BitbucketCloudUrlReader.ts | 234 +++++++++++ .../reading/BitbucketServerUrlReader.test.ts | 184 +++++++++ .../src/reading/BitbucketServerUrlReader.ts | 218 +++++++++++ .../src/reading/BitbucketUrlReader.test.ts | 94 +++-- .../src/reading/BitbucketUrlReader.ts | 29 +- .../backend-common/src/reading/UrlReaders.ts | 4 + packages/backend-common/src/reading/index.ts | 2 + 10 files changed, 1144 insertions(+), 35 deletions(-) create mode 100644 .changeset/dirty-eagles-hunt.md create mode 100644 packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts create mode 100644 packages/backend-common/src/reading/BitbucketCloudUrlReader.ts create mode 100644 packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts create mode 100644 packages/backend-common/src/reading/BitbucketServerUrlReader.ts diff --git a/.changeset/dirty-eagles-hunt.md b/.changeset/dirty-eagles-hunt.md new file mode 100644 index 0000000000..d3ad27dfc3 --- /dev/null +++ b/.changeset/dirty-eagles-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Split BitbucketUrlReader into BitbucketCloudUrlReader and BitbucketServerUrlReader. Backwards compatible. diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index cd4d08df87..1e77e5e3de 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -10,7 +10,9 @@ import { AbortController as AbortController_2 } from 'node-abort-controller'; import { AbortSignal as AbortSignal_2 } from 'node-abort-controller'; import { AwsS3Integration } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; +import { BitbucketCloudIntegration } from '@backstage/integration'; import { BitbucketIntegration } from '@backstage/integration'; +import { BitbucketServerIntegration } from '@backstage/integration'; import { Config } from '@backstage/config'; import cors from 'cors'; import Docker from 'dockerode'; @@ -83,9 +85,54 @@ export class AzureUrlReader implements UrlReader { } // @public +export class BitbucketCloudUrlReader implements UrlReader { + constructor( + integration: BitbucketCloudIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; +} + +// @public +export class BitbucketServerUrlReader implements UrlReader { + constructor( + integration: BitbucketServerIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string, options?: ReadTreeOptions): Promise; + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(url: string, options?: SearchOptions): Promise; + // (undocumented) + toString(): string; +} + +// @public @deprecated export class BitbucketUrlReader implements UrlReader { constructor( integration: BitbucketIntegration, + logger: Logger, deps: { treeResponseFactory: ReadTreeResponseFactory; }, diff --git a/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts new file mode 100644 index 0000000000..38841d75ad --- /dev/null +++ b/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts @@ -0,0 +1,362 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { + BitbucketCloudIntegration, + readBitbucketCloudIntegrationConfig, +} from '@backstage/integration'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import os from 'os'; +import path from 'path'; +import { NotModifiedError } from '@backstage/errors'; +import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader'; +import { DefaultReadTreeResponseFactory } from './tree'; + +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + +const reader = new BitbucketCloudUrlReader( + new BitbucketCloudIntegration( + readBitbucketCloudIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + username: 'username', + appPassword: 'password', + }), + ), + ), + { treeResponseFactory }, +); + +const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; + +describe('BitbucketCloudUrlReader', () => { + beforeEach(() => { + mockFs({ + [tmpDir]: mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + const worker = setupServer(); + setupRequestMockHandlers(worker); + + describe('readUrl', () => { + it('should be able to readUrl without ETag', async () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBeNull(); + return res( + ctx.status(200), + ctx.body('foo'), + ctx.set('ETag', 'etag-value'), + ); + }, + ), + ); + + const result = await reader.readUrl( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + const buffer = await result.buffer(); + expect(buffer.toString()).toBe('foo'); + }); + + it('should be able to readUrl with matching ETag', async () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBe( + 'matching-etag-value', + ); + return res(ctx.status(304)); + }, + ), + ); + + await expect( + reader.readUrl( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + { etag: 'matching-etag-value' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should be able to readUrl without matching ETag', async () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBe( + 'previous-etag-value', + ); + return res( + ctx.status(200), + ctx.body('foo'), + ctx.set('ETag', 'new-etag-value'), + ); + }, + ), + ); + + const result = await reader.readUrl( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + { etag: 'previous-etag-value' }, + ); + const buffer = await result.buffer(); + expect(buffer.toString()).toBe('foo'); + expect(result.etag).toBe('new-etag-value'); + }); + }); + + describe('read', () => { + it('rejects unknown targets', async () => { + await expect( + reader.read('https://not.bitbucket.com/apa'), + ).rejects.toThrow( + 'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket Cloud URL or file path', + ); + }); + }); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve( + __dirname, + '__fixtures__/bitbucket-repo-with-commit-hash.tar.gz', + ), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + mainbranch: { + type: 'branch', + name: 'master', + }, + }), + ), + ), + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.tar.gz', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const response = await reader.readTree( + 'https://bitbucket.org/backstage/mock/src/master', + ); + + expect(response.etag).toBe('12ab34cd56ef'); + + const files = await response.files(); + + expect(files.length).toBe(2); + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + }); + + it('creates a directory with the wanted files', async () => { + const response = await reader.readTree( + 'https://bitbucket.org/backstage/mock', + ); + + const dir = await response.dir({ targetDir: tmpDir }); + + await expect( + fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); + + it('returns the wanted files from an archive with a subpath', async () => { + const response = await reader.readTree( + 'https://bitbucket.org/backstage/mock/src/master/docs', + ); + + expect(response.etag).toBe('12ab34cd56ef'); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('creates a directory with the wanted files with a subpath', async () => { + const response = await reader.readTree( + 'https://bitbucket.org/backstage/mock/src/master/docs', + ); + + const dir = await response.dir({ targetDir: tmpDir }); + + await expect( + fs.readFile(path.join(dir, 'index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); + + it('throws a NotModifiedError when given a etag in options', async () => { + const fnBitbucket = async () => { + await reader.readTree('https://bitbucket.org/backstage/mock', { + etag: '12ab34cd56ef', + }); + }; + + await expect(fnBitbucket).rejects.toThrow(NotModifiedError); + }); + + it('should not throw a NotModifiedError when given an outdated etag in options', async () => { + const response = await reader.readTree( + 'https://bitbucket.org/backstage/mock', + { etag: 'outdatedetag123abc' }, + ); + + expect(response.etag).toBe('12ab34cd56ef'); + }); + }); + + describe('search hosted', () => { + const repoBuffer = fs.readFileSync( + path.resolve( + __dirname, + '__fixtures__/bitbucket-repo-with-commit-hash.tar.gz', + ), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + mainbranch: { + type: 'branch', + name: 'master', + }, + }), + ), + ), + rest.get( + 'https://bitbucket.org/backstage/mock/get/master.tar.gz', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + }); + + it('works for the naive case', async () => { + const result = await reader.search( + 'https://bitbucket.org/backstage/mock/src/master/**/index.*', + ); + expect(result.etag).toBe('12ab34cd56ef'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.org/backstage/mock/src/master/docs/index.md', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('works in nested folders', async () => { + const result = await reader.search( + 'https://bitbucket.org/backstage/mock/src/master/docs/index.*', + ); + expect(result.etag).toBe('12ab34cd56ef'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.org/backstage/mock/src/master/docs/index.md', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + reader.search( + 'https://bitbucket.org/backstage/mock/src/master/**/index.*', + { etag: '12ab34cd56ef' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); +}); diff --git a/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts b/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts new file mode 100644 index 0000000000..6a9406e920 --- /dev/null +++ b/packages/backend-common/src/reading/BitbucketCloudUrlReader.ts @@ -0,0 +1,234 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + BitbucketCloudIntegration, + getBitbucketCloudDefaultBranch, + getBitbucketCloudDownloadUrl, + getBitbucketCloudFileFetchUrl, + getBitbucketCloudRequestOptions, + ScmIntegrations, +} from '@backstage/integration'; +import fetch, { Response } from 'node-fetch'; +import parseGitUrl from 'git-url-parse'; +import { trimEnd } from 'lodash'; +import { Minimatch } from 'minimatch'; +import { Readable } from 'stream'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + ReadTreeResponseFactory, + ReadUrlOptions, + ReadUrlResponse, + SearchOptions, + SearchResponse, + UrlReader, +} from './types'; + +/** + * Implements a {@link UrlReader} for files from Bitbucket Cloud. + * + * @public + */ +export class BitbucketCloudUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const integrations = ScmIntegrations.fromConfig(config); + return integrations.bitbucketCloud.list().map(integration => { + const reader = new BitbucketCloudUrlReader(integration, { + treeResponseFactory, + }); + const predicate = (url: URL) => url.host === integration.config.host; + return { reader, predicate }; + }); + }; + + constructor( + private readonly integration: BitbucketCloudIntegration, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) { + const { host, username, appPassword } = integration.config; + + if (username && !appPassword) { + throw new Error( + `Bitbucket Cloud integration for '${host}' has configured a username but is missing a required appPassword.`, + ); + } + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + const { etag, signal } = options ?? {}; + const bitbucketUrl = getBitbucketCloudFileFetchUrl( + url, + this.integration.config, + ); + const requestOptions = getBitbucketCloudRequestOptions( + this.integration.config, + ); + + let response: Response; + try { + response = await fetch(bitbucketUrl.toString(), { + headers: { + ...requestOptions.headers, + ...(etag && { 'If-None-Match': etag }), + }, + // TODO(freben): The signal cast is there because pre-3.x versions of + // node-fetch have a very slightly deviating AbortSignal type signature. + // The difference does not affect us in practice however. The cast can be + // removed after we support ESM for CLI dependencies and migrate to + // version 3 of node-fetch. + // https://github.com/backstage/backstage/issues/8242 + ...(signal && { signal: signal as any }), + }); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.status === 304) { + throw new NotModifiedError(); + } + + if (response.ok) { + return { + buffer: async () => Buffer.from(await response.arrayBuffer()), + etag: response.headers.get('ETag') ?? undefined, + }; + } + + const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const { filepath } = parseGitUrl(url); + + const lastCommitShortHash = await this.getLastCommitShortHash(url); + if (options?.etag && options.etag === lastCommitShortHash) { + throw new NotModifiedError(); + } + + const downloadUrl = await getBitbucketCloudDownloadUrl( + url, + this.integration.config, + ); + const archiveResponse = await fetch( + downloadUrl, + getBitbucketCloudRequestOptions(this.integration.config), + ); + if (!archiveResponse.ok) { + const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`; + if (archiveResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return await this.deps.treeResponseFactory.fromTarArchive({ + stream: archiveResponse.body as unknown as Readable, + subpath: filepath, + etag: lastCommitShortHash, + filter: options?.filter, + }); + } + + async search(url: string, options?: SearchOptions): Promise { + const { filepath } = parseGitUrl(url); + const matcher = new Minimatch(filepath); + + // TODO(freben): For now, read the entire repo and filter through that. In + // a future improvement, we could be smart and try to deduce that non-glob + // prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used + // to get just that part of the repo. + const treeUrl = trimEnd(url.replace(filepath, ''), '/'); + + const tree = await this.readTree(treeUrl, { + etag: options?.etag, + filter: path => matcher.match(path), + }); + const files = await tree.files(); + + return { + etag: tree.etag, + files: files.map(file => ({ + url: this.integration.resolveUrl({ + url: `/${file.path}`, + base: url, + }), + content: file.content, + })), + }; + } + + toString() { + const { host, username, appPassword } = this.integration.config; + const authed = Boolean(username && appPassword); + return `bitbucketCloud{host=${host},authed=${authed}}`; + } + + private async getLastCommitShortHash(url: string): Promise { + const { name: repoName, owner: project, ref } = parseGitUrl(url); + + let branch = ref; + if (!branch) { + branch = await getBitbucketCloudDefaultBranch( + url, + this.integration.config, + ); + } + + const commitsApiUrl = `${this.integration.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`; + + const commitsResponse = await fetch( + commitsApiUrl, + getBitbucketCloudRequestOptions(this.integration.config), + ); + if (!commitsResponse.ok) { + const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`; + if (commitsResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + const commits = await commitsResponse.json(); + if ( + commits && + commits.values && + commits.values.length > 0 && + commits.values[0].hash + ) { + return commits.values[0].hash.substring(0, 12); + } + + throw new Error(`Failed to read response from ${commitsApiUrl}`); + } +} diff --git a/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts new file mode 100644 index 0000000000..1417752494 --- /dev/null +++ b/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts @@ -0,0 +1,184 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { + BitbucketServerIntegration, + readBitbucketServerIntegrationConfig, +} from '@backstage/integration'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import os from 'os'; +import path from 'path'; +import { NotModifiedError } from '@backstage/errors'; +import { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; +import { DefaultReadTreeResponseFactory } from './tree'; + +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + +const reader = new BitbucketServerUrlReader( + new BitbucketServerIntegration( + readBitbucketServerIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }), + ), + ), + { treeResponseFactory }, +); + +const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; + +describe('BitbucketServerUrlReader', () => { + beforeEach(() => { + mockFs({ + [tmpDir]: mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + const worker = setupServer(); + setupRequestMockHandlers(worker); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.tgz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + }); + + it('uses private bitbucket host', async () => { + const response = await reader.readTree( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch', + ); + + expect(response.etag).toBe('12ab34cd56ef'); + + const files = await response.files(); + + expect(files.length).toBe(1); + const indexMarkdownFile = await files[0].content(); + + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + }); + + describe('search private', () => { + const repoBuffer = fs.readFileSync( + path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/zip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.tgz', + ), + ctx.body(repoBuffer), + ), + ), + rest.get( + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits', + (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + }), + ), + ), + ); + }); + + it('works for the naive case', async () => { + const result = await reader.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master', + ); + expect(result.etag).toBe('12ab34cd56ef'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('works in nested folders', async () => { + const result = await reader.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.*?at=master', + ); + expect(result.etag).toBe('12ab34cd56ef'); + expect(result.files.length).toBe(1); + expect(result.files[0].url).toBe( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master', + ); + await expect(result.files[0].content()).resolves.toEqual( + Buffer.from('# Test\n'), + ); + }); + + it('throws NotModifiedError when same etag', async () => { + await expect( + reader.search( + 'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master', + { etag: '12ab34cd56ef' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); +}); diff --git a/packages/backend-common/src/reading/BitbucketServerUrlReader.ts b/packages/backend-common/src/reading/BitbucketServerUrlReader.ts new file mode 100644 index 0000000000..320b8f2719 --- /dev/null +++ b/packages/backend-common/src/reading/BitbucketServerUrlReader.ts @@ -0,0 +1,218 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import { + BitbucketServerIntegration, + getBitbucketServerDownloadUrl, + getBitbucketServerFileFetchUrl, + getBitbucketServerRequestOptions, + ScmIntegrations, +} from '@backstage/integration'; +import fetch, { Response } from 'node-fetch'; +import parseGitUrl from 'git-url-parse'; +import { trimEnd } from 'lodash'; +import { Minimatch } from 'minimatch'; +import { Readable } from 'stream'; +import { + ReaderFactory, + ReadTreeOptions, + ReadTreeResponse, + ReadTreeResponseFactory, + ReadUrlOptions, + ReadUrlResponse, + SearchOptions, + SearchResponse, + UrlReader, +} from './types'; + +/** + * Implements a {@link UrlReader} for files from Bitbucket Server APIs. + * + * @public + */ +export class BitbucketServerUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const integrations = ScmIntegrations.fromConfig(config); + return integrations.bitbucketServer.list().map(integration => { + const reader = new BitbucketServerUrlReader(integration, { + treeResponseFactory, + }); + const predicate = (url: URL) => url.host === integration.config.host; + return { reader, predicate }; + }); + }; + + constructor( + private readonly integration: BitbucketServerIntegration, + private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, + ) {} + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + const { etag, signal } = options ?? {}; + const bitbucketUrl = getBitbucketServerFileFetchUrl( + url, + this.integration.config, + ); + const requestOptions = getBitbucketServerRequestOptions( + this.integration.config, + ); + + let response: Response; + try { + response = await fetch(bitbucketUrl.toString(), { + headers: { + ...requestOptions.headers, + ...(etag && { 'If-None-Match': etag }), + }, + // TODO(freben): The signal cast is there because pre-3.x versions of + // node-fetch have a very slightly deviating AbortSignal type signature. + // The difference does not affect us in practice however. The cast can be + // removed after we support ESM for CLI dependencies and migrate to + // version 3 of node-fetch. + // https://github.com/backstage/backstage/issues/8242 + ...(signal && { signal: signal as any }), + }); + } catch (e) { + throw new Error(`Unable to read ${url}, ${e}`); + } + + if (response.status === 304) { + throw new NotModifiedError(); + } + + if (response.ok) { + return { + buffer: async () => Buffer.from(await response.arrayBuffer()), + etag: response.headers.get('ETag') ?? undefined, + }; + } + + const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const { filepath } = parseGitUrl(url); + + const lastCommitShortHash = await this.getLastCommitShortHash(url); + if (options?.etag && options.etag === lastCommitShortHash) { + throw new NotModifiedError(); + } + + const downloadUrl = await getBitbucketServerDownloadUrl( + url, + this.integration.config, + ); + const archiveResponse = await fetch( + downloadUrl, + getBitbucketServerRequestOptions(this.integration.config), + ); + if (!archiveResponse.ok) { + const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`; + if (archiveResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return await this.deps.treeResponseFactory.fromTarArchive({ + stream: archiveResponse.body as unknown as Readable, + subpath: filepath, + etag: lastCommitShortHash, + filter: options?.filter, + }); + } + + async search(url: string, options?: SearchOptions): Promise { + const { filepath } = parseGitUrl(url); + const matcher = new Minimatch(filepath); + + // TODO(freben): For now, read the entire repo and filter through that. In + // a future improvement, we could be smart and try to deduce that non-glob + // prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used + // to get just that part of the repo. + const treeUrl = trimEnd(url.replace(filepath, ''), '/'); + + const tree = await this.readTree(treeUrl, { + etag: options?.etag, + filter: path => matcher.match(path), + }); + const files = await tree.files(); + + return { + etag: tree.etag, + files: files.map(file => ({ + url: this.integration.resolveUrl({ + url: `/${file.path}`, + base: url, + }), + content: file.content, + })), + }; + } + + toString() { + const { host, token } = this.integration.config; + const authed = Boolean(token); + return `bitbucketServer{host=${host},authed=${authed}}`; + } + + private async getLastCommitShortHash(url: string): Promise { + const { name: repoName, owner: project } = parseGitUrl(url); + + // Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222 + const commitsApiUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`; + + const commitsResponse = await fetch( + commitsApiUrl, + getBitbucketServerRequestOptions(this.integration.config), + ); + if (!commitsResponse.ok) { + const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`; + if (commitsResponse.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + const commits = await commitsResponse.json(); + if ( + commits && + commits.values && + commits.values.length > 0 && + commits.values[0].id + ) { + return commits.values[0].id.substring(0, 12); + } + + throw new Error(`Failed to read response from ${commitsApiUrl}`); + } +} diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index eff982581b..3a2360debb 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -29,38 +29,76 @@ import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +import { getVoidLogger } from '../logging'; -const treeResponseFactory = DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), +const logger = getVoidLogger(); + +describe('BitbucketUrlReader.factory', () => { + it('only apply integration configs not inherited from bitbucketCloud or bitbucketServer', () => { + const config = new ConfigReader({ + integrations: { + bitbucket: [], + bitbucketCloud: [ + { + username: 'username', + appPassword: 'password', + }, + ], + bitbucketServer: [ + { + host: 'bitbucket-server.local', + token: 'test-token', + }, + ], + }, + }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: config, + }); + + const tuples = BitbucketUrlReader.factory({ + config, + logger, + treeResponseFactory, + }); + + expect(tuples).toHaveLength(0); + }); }); -const bitbucketProcessor = new BitbucketUrlReader( - new BitbucketIntegration( - readBitbucketIntegrationConfig( - new ConfigReader({ - host: 'bitbucket.org', - apiBaseUrl: 'https://api.bitbucket.org/2.0', - }), - ), - ), - { treeResponseFactory }, -); - -const hostedBitbucketProcessor = new BitbucketUrlReader( - new BitbucketIntegration( - readBitbucketIntegrationConfig( - new ConfigReader({ - host: 'bitbucket.mycompany.net', - apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', - }), - ), - ), - { treeResponseFactory }, -); - -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('BitbucketUrlReader', () => { + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), + }); + + const bitbucketProcessor = new BitbucketUrlReader( + new BitbucketIntegration( + readBitbucketIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }), + ), + ), + logger, + { treeResponseFactory }, + ); + + const hostedBitbucketProcessor = new BitbucketUrlReader( + new BitbucketIntegration( + readBitbucketIntegrationConfig( + new ConfigReader({ + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0', + }), + ), + ), + logger, + { treeResponseFactory }, + ); + + const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; + beforeEach(() => { mockFs({ [tmpDir]: mockFs.directory(), diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index df22f1633f..1687a66262 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -28,6 +28,7 @@ import parseGitUrl from 'git-url-parse'; import { trimEnd } from 'lodash'; import { Minimatch } from 'minimatch'; import { Readable } from 'stream'; +import { Logger } from 'winston'; import { ReaderFactory, ReadTreeOptions, @@ -45,24 +46,38 @@ import { * as the one exposed by Bitbucket Cloud itself. * * @public + * @deprecated in favor of BitbucketCloudUrlReader and BitbucketServerUrlReader */ export class BitbucketUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + static factory: ReaderFactory = ({ config, logger, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); - return integrations.bitbucket.list().map(integration => { - const reader = new BitbucketUrlReader(integration, { - treeResponseFactory, + return integrations.bitbucket + .list() + .filter( + item => + !integrations.bitbucketCloud.byHost(item.config.host) && + !integrations.bitbucketServer.byHost(item.config.host), + ) + .map(integration => { + const reader = new BitbucketUrlReader(integration, logger, { + treeResponseFactory, + }); + const predicate = (url: URL) => url.host === integration.config.host; + return { reader, predicate }; }); - const predicate = (url: URL) => url.host === integration.config.host; - return { reader, predicate }; - }); }; constructor( private readonly integration: BitbucketIntegration, + logger: Logger, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { const { host, token, username, appPassword } = integration.config; + const replacement = + host === 'bitbucket.org' ? 'bitbucketCloud' : 'bitbucketServer'; + logger.warn( + `[Deprecated] Please migrate from "integrations.bitbucket" to "integrations.${replacement}".`, + ); if (!token && username && !appPassword) { throw new Error( diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index e676d0e14c..72833454c3 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -19,6 +19,8 @@ import { Config } from '@backstage/config'; import { ReaderFactory, UrlReader } from './types'; import { UrlReaderPredicateMux } from './UrlReaderPredicateMux'; import { AzureUrlReader } from './AzureUrlReader'; +import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader'; +import { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; import { BitbucketUrlReader } from './BitbucketUrlReader'; import { GerritUrlReader } from './GerritUrlReader'; import { GithubUrlReader } from './GithubUrlReader'; @@ -82,6 +84,8 @@ export class UrlReaders { config, factories: factories.concat([ AzureUrlReader.factory, + BitbucketCloudUrlReader.factory, + BitbucketServerUrlReader.factory, BitbucketUrlReader.factory, GerritUrlReader.factory, GithubUrlReader.factory, diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index c2c82e2bd5..f1609179b5 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -15,7 +15,9 @@ */ export { AzureUrlReader } from './AzureUrlReader'; +export { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; +export { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; export { GerritUrlReader } from './GerritUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader';