feat: split BitbucketUrlReader

Split `BitbucketUrlReader` into `BitbucketCloudUrlReader`
and `BitbucketServerUrlReader`
while staying backwards compatible and conflict free (== always
only one of the readers will apply).

Relates-to: #9923
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-04-10 02:12:25 +02:00
parent 1b4e1e2306
commit 75bf9e1da9
10 changed files with 1144 additions and 35 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-common': patch
---
Split BitbucketUrlReader into BitbucketCloudUrlReader and BitbucketServerUrlReader. Backwards compatible.
+47
View File
@@ -10,7 +10,9 @@ import { AbortController as AbortController_2 } from 'node-abort-controller';
import { AbortSignal as AbortSignal_2 } from 'node-abort-controller';
import { AwsS3Integration } from '@backstage/integration';
import { AzureIntegration } from '@backstage/integration';
import { BitbucketCloudIntegration } from '@backstage/integration';
import { BitbucketIntegration } from '@backstage/integration';
import { BitbucketServerIntegration } from '@backstage/integration';
import { Config } from '@backstage/config';
import cors from 'cors';
import Docker from 'dockerode';
@@ -83,9 +85,54 @@ export class AzureUrlReader implements UrlReader {
}
// @public
export class BitbucketCloudUrlReader implements UrlReader {
constructor(
integration: BitbucketCloudIntegration,
deps: {
treeResponseFactory: ReadTreeResponseFactory;
},
);
// (undocumented)
static factory: ReaderFactory;
// (undocumented)
read(url: string): Promise<Buffer>;
// (undocumented)
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
// (undocumented)
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
// (undocumented)
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
// (undocumented)
toString(): string;
}
// @public
export class BitbucketServerUrlReader implements UrlReader {
constructor(
integration: BitbucketServerIntegration,
deps: {
treeResponseFactory: ReadTreeResponseFactory;
},
);
// (undocumented)
static factory: ReaderFactory;
// (undocumented)
read(url: string): Promise<Buffer>;
// (undocumented)
readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
// (undocumented)
readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
// (undocumented)
search(url: string, options?: SearchOptions): Promise<SearchResponse>;
// (undocumented)
toString(): string;
}
// @public @deprecated
export class BitbucketUrlReader implements UrlReader {
constructor(
integration: BitbucketIntegration,
logger: Logger,
deps: {
treeResponseFactory: ReadTreeResponseFactory;
},
@@ -0,0 +1,362 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import {
BitbucketCloudIntegration,
readBitbucketCloudIntegrationConfig,
} from '@backstage/integration';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import os from 'os';
import path from 'path';
import { NotModifiedError } from '@backstage/errors';
import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader';
import { DefaultReadTreeResponseFactory } from './tree';
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const reader = new BitbucketCloudUrlReader(
new BitbucketCloudIntegration(
readBitbucketCloudIntegrationConfig(
new ConfigReader({
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
username: 'username',
appPassword: 'password',
}),
),
),
{ treeResponseFactory },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('BitbucketCloudUrlReader', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
setupRequestMockHandlers(worker);
describe('readUrl', () => {
it('should be able to readUrl without ETag', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-None-Match')).toBeNull();
return res(
ctx.status(200),
ctx.body('foo'),
ctx.set('ETag', 'etag-value'),
);
},
),
);
const result = await reader.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
);
const buffer = await result.buffer();
expect(buffer.toString()).toBe('foo');
});
it('should be able to readUrl with matching ETag', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-None-Match')).toBe(
'matching-etag-value',
);
return res(ctx.status(304));
},
),
);
await expect(
reader.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
{ etag: 'matching-etag-value' },
),
).rejects.toThrow(NotModifiedError);
});
it('should be able to readUrl without matching ETag', async () => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
(req, res, ctx) => {
expect(req.headers.get('If-None-Match')).toBe(
'previous-etag-value',
);
return res(
ctx.status(200),
ctx.body('foo'),
ctx.set('ETag', 'new-etag-value'),
);
},
),
);
const result = await reader.readUrl(
'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
{ etag: 'previous-etag-value' },
);
const buffer = await result.buffer();
expect(buffer.toString()).toBe('foo');
expect(result.etag).toBe('new-etag-value');
});
});
describe('read', () => {
it('rejects unknown targets', async () => {
await expect(
reader.read('https://not.bitbucket.com/apa'),
).rejects.toThrow(
'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket Cloud URL or file path',
);
});
});
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve(
__dirname,
'__fixtures__/bitbucket-repo-with-commit-hash.tar.gz',
),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
mainbranch: {
type: 'branch',
name: 'master',
},
}),
),
),
rest.get(
'https://bitbucket.org/backstage/mock/get/master.tar.gz',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
it('returns the wanted files from an archive', async () => {
const response = await reader.readTree(
'https://bitbucket.org/backstage/mock/src/master',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(2);
const mkDocsFile = await files[0].content();
const indexMarkdownFile = await files[1].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
});
it('creates a directory with the wanted files', async () => {
const response = await reader.readTree(
'https://bitbucket.org/backstage/mock',
);
const dir = await response.dir({ targetDir: tmpDir });
await expect(
fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(path.join(dir, 'docs', 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('returns the wanted files from an archive with a subpath', async () => {
const response = await reader.readTree(
'https://bitbucket.org/backstage/mock/src/master/docs',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('creates a directory with the wanted files with a subpath', async () => {
const response = await reader.readTree(
'https://bitbucket.org/backstage/mock/src/master/docs',
);
const dir = await response.dir({ targetDir: tmpDir });
await expect(
fs.readFile(path.join(dir, 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('throws a NotModifiedError when given a etag in options', async () => {
const fnBitbucket = async () => {
await reader.readTree('https://bitbucket.org/backstage/mock', {
etag: '12ab34cd56ef',
});
};
await expect(fnBitbucket).rejects.toThrow(NotModifiedError);
});
it('should not throw a NotModifiedError when given an outdated etag in options', async () => {
const response = await reader.readTree(
'https://bitbucket.org/backstage/mock',
{ etag: 'outdatedetag123abc' },
);
expect(response.etag).toBe('12ab34cd56ef');
});
});
describe('search hosted', () => {
const repoBuffer = fs.readFileSync(
path.resolve(
__dirname,
'__fixtures__/bitbucket-repo-with-commit-hash.tar.gz',
),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
mainbranch: {
type: 'branch',
name: 'master',
},
}),
),
),
rest.get(
'https://bitbucket.org/backstage/mock/get/master.tar.gz',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename=backstage-mock-12ab34cd56ef.tar.gz',
),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
it('works for the naive case', async () => {
const result = await reader.search(
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
);
expect(result.etag).toBe('12ab34cd56ef');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('works in nested folders', async () => {
const result = await reader.search(
'https://bitbucket.org/backstage/mock/src/master/docs/index.*',
);
expect(result.etag).toBe('12ab34cd56ef');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://bitbucket.org/backstage/mock/src/master/docs/index.md',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('throws NotModifiedError when same etag', async () => {
await expect(
reader.search(
'https://bitbucket.org/backstage/mock/src/master/**/index.*',
{ etag: '12ab34cd56ef' },
),
).rejects.toThrow(NotModifiedError);
});
});
});
@@ -0,0 +1,234 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { NotFoundError, NotModifiedError } from '@backstage/errors';
import {
BitbucketCloudIntegration,
getBitbucketCloudDefaultBranch,
getBitbucketCloudDownloadUrl,
getBitbucketCloudFileFetchUrl,
getBitbucketCloudRequestOptions,
ScmIntegrations,
} from '@backstage/integration';
import fetch, { Response } from 'node-fetch';
import parseGitUrl from 'git-url-parse';
import { trimEnd } from 'lodash';
import { Minimatch } from 'minimatch';
import { Readable } from 'stream';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
ReadTreeResponseFactory,
ReadUrlOptions,
ReadUrlResponse,
SearchOptions,
SearchResponse,
UrlReader,
} from './types';
/**
* Implements a {@link UrlReader} for files from Bitbucket Cloud.
*
* @public
*/
export class BitbucketCloudUrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const integrations = ScmIntegrations.fromConfig(config);
return integrations.bitbucketCloud.list().map(integration => {
const reader = new BitbucketCloudUrlReader(integration, {
treeResponseFactory,
});
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
private readonly integration: BitbucketCloudIntegration,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
const { host, username, appPassword } = integration.config;
if (username && !appPassword) {
throw new Error(
`Bitbucket Cloud integration for '${host}' has configured a username but is missing a required appPassword.`,
);
}
}
async read(url: string): Promise<Buffer> {
const response = await this.readUrl(url);
return response.buffer();
}
async readUrl(
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const bitbucketUrl = getBitbucketCloudFileFetchUrl(
url,
this.integration.config,
);
const requestOptions = getBitbucketCloudRequestOptions(
this.integration.config,
);
let response: Response;
try {
response = await fetch(bitbucketUrl.toString(), {
headers: {
...requestOptions.headers,
...(etag && { 'If-None-Match': etag }),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
// The difference does not affect us in practice however. The cast can be
// removed after we support ESM for CLI dependencies and migrate to
// version 3 of node-fetch.
// https://github.com/backstage/backstage/issues/8242
...(signal && { signal: signal as any }),
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
if (response.status === 304) {
throw new NotModifiedError();
}
if (response.ok) {
return {
buffer: async () => Buffer.from(await response.arrayBuffer()),
etag: response.headers.get('ETag') ?? undefined,
};
}
const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const { filepath } = parseGitUrl(url);
const lastCommitShortHash = await this.getLastCommitShortHash(url);
if (options?.etag && options.etag === lastCommitShortHash) {
throw new NotModifiedError();
}
const downloadUrl = await getBitbucketCloudDownloadUrl(
url,
this.integration.config,
);
const archiveResponse = await fetch(
downloadUrl,
getBitbucketCloudRequestOptions(this.integration.config),
);
if (!archiveResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`;
if (archiveResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
return await this.deps.treeResponseFactory.fromTarArchive({
stream: archiveResponse.body as unknown as Readable,
subpath: filepath,
etag: lastCommitShortHash,
filter: options?.filter,
});
}
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
const { filepath } = parseGitUrl(url);
const matcher = new Minimatch(filepath);
// TODO(freben): For now, read the entire repo and filter through that. In
// a future improvement, we could be smart and try to deduce that non-glob
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
// to get just that part of the repo.
const treeUrl = trimEnd(url.replace(filepath, ''), '/');
const tree = await this.readTree(treeUrl, {
etag: options?.etag,
filter: path => matcher.match(path),
});
const files = await tree.files();
return {
etag: tree.etag,
files: files.map(file => ({
url: this.integration.resolveUrl({
url: `/${file.path}`,
base: url,
}),
content: file.content,
})),
};
}
toString() {
const { host, username, appPassword } = this.integration.config;
const authed = Boolean(username && appPassword);
return `bitbucketCloud{host=${host},authed=${authed}}`;
}
private async getLastCommitShortHash(url: string): Promise<string> {
const { name: repoName, owner: project, ref } = parseGitUrl(url);
let branch = ref;
if (!branch) {
branch = await getBitbucketCloudDefaultBranch(
url,
this.integration.config,
);
}
const commitsApiUrl = `${this.integration.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`;
const commitsResponse = await fetch(
commitsApiUrl,
getBitbucketCloudRequestOptions(this.integration.config),
);
if (!commitsResponse.ok) {
const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
if (commitsResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commits = await commitsResponse.json();
if (
commits &&
commits.values &&
commits.values.length > 0 &&
commits.values[0].hash
) {
return commits.values[0].hash.substring(0, 12);
}
throw new Error(`Failed to read response from ${commitsApiUrl}`);
}
}
@@ -0,0 +1,184 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import {
BitbucketServerIntegration,
readBitbucketServerIntegrationConfig,
} from '@backstage/integration';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import os from 'os';
import path from 'path';
import { NotModifiedError } from '@backstage/errors';
import { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
import { DefaultReadTreeResponseFactory } from './tree';
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const reader = new BitbucketServerUrlReader(
new BitbucketServerIntegration(
readBitbucketServerIntegrationConfig(
new ConfigReader({
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
}),
),
),
{ treeResponseFactory },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('BitbucketServerUrlReader', () => {
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
const worker = setupServer();
setupRequestMockHandlers(worker);
describe('readTree', () => {
const repoBuffer = fs.readFileSync(
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename=backstage-mock.tgz',
),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
it('uses private bitbucket host', async () => {
const response = await reader.readTree(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
);
expect(response.etag).toBe('12ab34cd56ef');
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
});
describe('search private', () => {
const repoBuffer = fs.readFileSync(
path.resolve(__dirname, '__fixtures__/bitbucket-server-repo.tar.gz'),
);
beforeEach(() => {
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.set(
'content-disposition',
'attachment; filename=backstage-mock.tgz',
),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
});
it('works for the naive case', async () => {
const result = await reader.search(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
);
expect(result.etag).toBe('12ab34cd56ef');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('works in nested folders', async () => {
const result = await reader.search(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.*?at=master',
);
expect(result.etag).toBe('12ab34cd56ef');
expect(result.files.length).toBe(1);
expect(result.files[0].url).toBe(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
);
await expect(result.files[0].content()).resolves.toEqual(
Buffer.from('# Test\n'),
);
});
it('throws NotModifiedError when same etag', async () => {
await expect(
reader.search(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/**/index.*?at=master',
{ etag: '12ab34cd56ef' },
),
).rejects.toThrow(NotModifiedError);
});
});
});
@@ -0,0 +1,218 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { NotFoundError, NotModifiedError } from '@backstage/errors';
import {
BitbucketServerIntegration,
getBitbucketServerDownloadUrl,
getBitbucketServerFileFetchUrl,
getBitbucketServerRequestOptions,
ScmIntegrations,
} from '@backstage/integration';
import fetch, { Response } from 'node-fetch';
import parseGitUrl from 'git-url-parse';
import { trimEnd } from 'lodash';
import { Minimatch } from 'minimatch';
import { Readable } from 'stream';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
ReadTreeResponseFactory,
ReadUrlOptions,
ReadUrlResponse,
SearchOptions,
SearchResponse,
UrlReader,
} from './types';
/**
* Implements a {@link UrlReader} for files from Bitbucket Server APIs.
*
* @public
*/
export class BitbucketServerUrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const integrations = ScmIntegrations.fromConfig(config);
return integrations.bitbucketServer.list().map(integration => {
const reader = new BitbucketServerUrlReader(integration, {
treeResponseFactory,
});
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
private readonly integration: BitbucketServerIntegration,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {}
async read(url: string): Promise<Buffer> {
const response = await this.readUrl(url);
return response.buffer();
}
async readUrl(
url: string,
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
const { etag, signal } = options ?? {};
const bitbucketUrl = getBitbucketServerFileFetchUrl(
url,
this.integration.config,
);
const requestOptions = getBitbucketServerRequestOptions(
this.integration.config,
);
let response: Response;
try {
response = await fetch(bitbucketUrl.toString(), {
headers: {
...requestOptions.headers,
...(etag && { 'If-None-Match': etag }),
},
// TODO(freben): The signal cast is there because pre-3.x versions of
// node-fetch have a very slightly deviating AbortSignal type signature.
// The difference does not affect us in practice however. The cast can be
// removed after we support ESM for CLI dependencies and migrate to
// version 3 of node-fetch.
// https://github.com/backstage/backstage/issues/8242
...(signal && { signal: signal as any }),
});
} catch (e) {
throw new Error(`Unable to read ${url}, ${e}`);
}
if (response.status === 304) {
throw new NotModifiedError();
}
if (response.ok) {
return {
buffer: async () => Buffer.from(await response.arrayBuffer()),
etag: response.headers.get('ETag') ?? undefined,
};
}
const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const { filepath } = parseGitUrl(url);
const lastCommitShortHash = await this.getLastCommitShortHash(url);
if (options?.etag && options.etag === lastCommitShortHash) {
throw new NotModifiedError();
}
const downloadUrl = await getBitbucketServerDownloadUrl(
url,
this.integration.config,
);
const archiveResponse = await fetch(
downloadUrl,
getBitbucketServerRequestOptions(this.integration.config),
);
if (!archiveResponse.ok) {
const message = `Failed to read tree from ${url}, ${archiveResponse.status} ${archiveResponse.statusText}`;
if (archiveResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
return await this.deps.treeResponseFactory.fromTarArchive({
stream: archiveResponse.body as unknown as Readable,
subpath: filepath,
etag: lastCommitShortHash,
filter: options?.filter,
});
}
async search(url: string, options?: SearchOptions): Promise<SearchResponse> {
const { filepath } = parseGitUrl(url);
const matcher = new Minimatch(filepath);
// TODO(freben): For now, read the entire repo and filter through that. In
// a future improvement, we could be smart and try to deduce that non-glob
// prefixes (like for filepaths such as some-prefix/**/a.yaml) can be used
// to get just that part of the repo.
const treeUrl = trimEnd(url.replace(filepath, ''), '/');
const tree = await this.readTree(treeUrl, {
etag: options?.etag,
filter: path => matcher.match(path),
});
const files = await tree.files();
return {
etag: tree.etag,
files: files.map(file => ({
url: this.integration.resolveUrl({
url: `/${file.path}`,
base: url,
}),
content: file.content,
})),
};
}
toString() {
const { host, token } = this.integration.config;
const authed = Boolean(token);
return `bitbucketServer{host=${host},authed=${authed}}`;
}
private async getLastCommitShortHash(url: string): Promise<string> {
const { name: repoName, owner: project } = parseGitUrl(url);
// Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222
const commitsApiUrl = `${this.integration.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`;
const commitsResponse = await fetch(
commitsApiUrl,
getBitbucketServerRequestOptions(this.integration.config),
);
if (!commitsResponse.ok) {
const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
if (commitsResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commits = await commitsResponse.json();
if (
commits &&
commits.values &&
commits.values.length > 0 &&
commits.values[0].id
) {
return commits.values[0].id.substring(0, 12);
}
throw new Error(`Failed to read response from ${commitsApiUrl}`);
}
}
@@ -29,38 +29,76 @@ import path from 'path';
import { NotModifiedError } from '@backstage/errors';
import { BitbucketUrlReader } from './BitbucketUrlReader';
import { DefaultReadTreeResponseFactory } from './tree';
import { getVoidLogger } from '../logging';
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
config: new ConfigReader({}),
const logger = getVoidLogger();
describe('BitbucketUrlReader.factory', () => {
it('only apply integration configs not inherited from bitbucketCloud or bitbucketServer', () => {
const config = new ConfigReader({
integrations: {
bitbucket: [],
bitbucketCloud: [
{
username: 'username',
appPassword: 'password',
},
],
bitbucketServer: [
{
host: 'bitbucket-server.local',
token: 'test-token',
},
],
},
});
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
config: config,
});
const tuples = BitbucketUrlReader.factory({
config,
logger,
treeResponseFactory,
});
expect(tuples).toHaveLength(0);
});
});
const bitbucketProcessor = new BitbucketUrlReader(
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
}),
),
),
{ treeResponseFactory },
);
const hostedBitbucketProcessor = new BitbucketUrlReader(
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
}),
),
),
{ treeResponseFactory },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
describe('BitbucketUrlReader', () => {
const treeResponseFactory = DefaultReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
const bitbucketProcessor = new BitbucketUrlReader(
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
}),
),
),
logger,
{ treeResponseFactory },
);
const hostedBitbucketProcessor = new BitbucketUrlReader(
new BitbucketIntegration(
readBitbucketIntegrationConfig(
new ConfigReader({
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
}),
),
),
logger,
{ treeResponseFactory },
);
const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
beforeEach(() => {
mockFs({
[tmpDir]: mockFs.directory(),
@@ -28,6 +28,7 @@ import parseGitUrl from 'git-url-parse';
import { trimEnd } from 'lodash';
import { Minimatch } from 'minimatch';
import { Readable } from 'stream';
import { Logger } from 'winston';
import {
ReaderFactory,
ReadTreeOptions,
@@ -45,24 +46,38 @@ import {
* as the one exposed by Bitbucket Cloud itself.
*
* @public
* @deprecated in favor of BitbucketCloudUrlReader and BitbucketServerUrlReader
*/
export class BitbucketUrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
static factory: ReaderFactory = ({ config, logger, treeResponseFactory }) => {
const integrations = ScmIntegrations.fromConfig(config);
return integrations.bitbucket.list().map(integration => {
const reader = new BitbucketUrlReader(integration, {
treeResponseFactory,
return integrations.bitbucket
.list()
.filter(
item =>
!integrations.bitbucketCloud.byHost(item.config.host) &&
!integrations.bitbucketServer.byHost(item.config.host),
)
.map(integration => {
const reader = new BitbucketUrlReader(integration, logger, {
treeResponseFactory,
});
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
const predicate = (url: URL) => url.host === integration.config.host;
return { reader, predicate };
});
};
constructor(
private readonly integration: BitbucketIntegration,
logger: Logger,
private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
const { host, token, username, appPassword } = integration.config;
const replacement =
host === 'bitbucket.org' ? 'bitbucketCloud' : 'bitbucketServer';
logger.warn(
`[Deprecated] Please migrate from "integrations.bitbucket" to "integrations.${replacement}".`,
);
if (!token && username && !appPassword) {
throw new Error(
@@ -19,6 +19,8 @@ import { Config } from '@backstage/config';
import { ReaderFactory, UrlReader } from './types';
import { UrlReaderPredicateMux } from './UrlReaderPredicateMux';
import { AzureUrlReader } from './AzureUrlReader';
import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader';
import { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
import { BitbucketUrlReader } from './BitbucketUrlReader';
import { GerritUrlReader } from './GerritUrlReader';
import { GithubUrlReader } from './GithubUrlReader';
@@ -82,6 +84,8 @@ export class UrlReaders {
config,
factories: factories.concat([
AzureUrlReader.factory,
BitbucketCloudUrlReader.factory,
BitbucketServerUrlReader.factory,
BitbucketUrlReader.factory,
GerritUrlReader.factory,
GithubUrlReader.factory,
@@ -15,7 +15,9 @@
*/
export { AzureUrlReader } from './AzureUrlReader';
export { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader';
export { BitbucketUrlReader } from './BitbucketUrlReader';
export { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
export { GerritUrlReader } from './GerritUrlReader';
export { GithubUrlReader } from './GithubUrlReader';
export { GitlabUrlReader } from './GitlabUrlReader';