paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

Merge pull request #3609 from lykims/feature/bitbucket-read-tree

Browse Source 
Implement readTree on BitBucketUrlReader
This commit is contained in:
Ben Lambert
2020-12-15 13:17:54 +01:00
committed by GitHub
parent 186f9327e7 bbd0e354ff
commit 9be24ddfae
10 changed files with 482 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/grumpy-elephants-press.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/backend-common': patch
'@backstage/integration': patch
---
Implement readTree on BitBucketUrlReader and getBitbucketDownloadUrl
packages/backend-common/src/reading/AzureUrlReader.test.ts
+1 -3
View File
@@ -158,9 +158,7 @@ describe('AzureUrlReader', () => {
it('returns the wanted files from an archive', async () => {
const processor = new AzureUrlReader(
{
host: 'dev.azure.com',
},
{ host: 'dev.azure.com' },
{ treeResponseFactory },
);
packages/backend-common/src/reading/BitbucketUrlReader.test.ts
+152 -4
View File
@@ -14,15 +14,26 @@
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { msw } from '@backstage/test-utils';
import fs from 'fs';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import path from 'path';
import { BitbucketUrlReader } from './BitbucketUrlReader';
import { ReadTreeResponseFactory } from './tree';
const treeResponseFactory = ReadTreeResponseFactory.create({
config: new ConfigReader({}),
});
describe('BitbucketUrlReader', () => {
describe('implementation', () => {
it('rejects unknown targets', async () => {
const processor = new BitbucketUrlReader({
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
});
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
await expect(
processor.read('https://not.bitbucket.com/apa'),
).rejects.toThrow(
@@ -30,4 +41,141 @@ describe('BitbucketUrlReader', () => {
);
});
});
describe('readTree', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
const repoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-repo-with-commit-hash.zip',
),
);
it('returns the wanted files from an archive', async () => {
worker.use(
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://bitbucket.org/backstage/mock/src/master',
);
const files = await response.files();
expect(files.length).toBe(2);
const indexMarkdownFile = await files[0].content();
const mkDocsFile = await files[1].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
expect(mkDocsFile.toString()).toBe('site_name: Test\n');
});
it('uses private bitbucket host', async () => {
const privateBitbucketRepoBuffer = fs.readFileSync(
path.resolve(
'src',
'reading',
'__fixtures__',
'bitbucket-server-repo.zip',
),
);
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(privateBitbucketRepoBuffer),
),
),
);
const processor = new BitbucketUrlReader(
{
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
},
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
);
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
it('returns the wanted files from an archive with a subpath', async () => {
worker.use(
rest.get(
'https://bitbucket.org/backstage/mock/get/master.zip',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/zip'),
ctx.body(repoBuffer),
),
),
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
}),
),
),
);
const processor = new BitbucketUrlReader(
{ host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
{ treeResponseFactory },
);
const response = await processor.readTree(
'https://bitbucket.org/backstage/mock/src/master/docs',
);
const files = await response.files();
expect(files.length).toBe(1);
const indexMarkdownFile = await files[0].content();
expect(indexMarkdownFile.toString()).toBe('# Test\n');
});
});
});
packages/backend-common/src/reading/BitbucketUrlReader.ts
+85 -6
View File
@@ -16,13 +16,23 @@
import {
BitbucketIntegrationConfig,
getBitbucketDefaultBranch,
getBitbucketDownloadUrl,
getBitbucketFileFetchUrl,
getBitbucketRequestOptions,
readBitbucketIntegrationConfigs,
} from '@backstage/integration';
import fetch from 'cross-fetch';
import parseGitUri from 'git-url-parse';
import { Readable } from 'stream';
import { NotFoundError } from '../errors';
import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
import { ReadTreeResponseFactory } from './tree';
import {
ReaderFactory,
ReadTreeOptions,
ReadTreeResponse,
UrlReader,
} from './types';
/**
* A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as
@@ -30,19 +40,23 @@ import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
*/
export class BitbucketUrlReader implements UrlReader {
private readonly config: BitbucketIntegrationConfig;
private readonly treeResponseFactory: ReadTreeResponseFactory;
static factory: ReaderFactory = ({ config }) => {
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
const configs = readBitbucketIntegrationConfigs(
config.getOptionalConfigArray('integrations.bitbucket') ?? [],
);
return configs.map(provider => {
const reader = new BitbucketUrlReader(provider);
const reader = new BitbucketUrlReader(provider, { treeResponseFactory });
const predicate = (url: URL) => url.host === provider.host;
return { reader, predicate };
});
};
constructor(config: BitbucketIntegrationConfig) {
constructor(
config: BitbucketIntegrationConfig,
deps: { treeResponseFactory: ReadTreeResponseFactory },
) {
const { host, apiBaseUrl, token, username, appPassword } = config;
if (!apiBaseUrl) {
@@ -58,6 +72,7 @@ export class BitbucketUrlReader implements UrlReader {
}
this.config = config;
this.treeResponseFactory = deps.treeResponseFactory;
}
async read(url: string): Promise<Buffer> {
@@ -82,8 +97,39 @@ export class BitbucketUrlReader implements UrlReader {
throw new Error(message);
}
readTree(): Promise<ReadTreeResponse> {
throw new Error('BitbucketUrlReader does not implement readTree');
async readTree(
url: string,
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
const gitUrl: parseGitUri.GitUrl = parseGitUri(url);
const { name: repoName, owner: project, resource, filepath } = gitUrl;
const isHosted = resource === 'bitbucket.org';
const downloadUrl = await getBitbucketDownloadUrl(url, this.config);
const response = await fetch(
downloadUrl,
getBitbucketRequestOptions(this.config),
);
if (!response.ok) {
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
let folderPath = `${project}-${repoName}`;
if (isHosted) {
const lastCommitShortHash = await this.getLastCommitShortHash(url);
folderPath = `${project}-${repoName}-${lastCommitShortHash}`;
}
return this.treeResponseFactory.fromZipArchive({
stream: (response.body as unknown) as Readable,
path: `${folderPath}/${filepath}`,
filter: options?.filter,
});
}
toString() {
@@ -94,4 +140,37 @@ export class BitbucketUrlReader implements UrlReader {
}
return `bitbucket{host=${host},authed=${authed}}`;
}
private async getLastCommitShortHash(url: string): Promise<String> {
const { name: repoName, owner: project, ref } = parseGitUri(url);
let branch = ref;
if (!branch) {
branch = await getBitbucketDefaultBranch(url, this.config);
}
const commitsApiUrl = `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`;
const commitsResponse = await fetch(
commitsApiUrl,
getBitbucketRequestOptions(this.config),
);
if (!commitsResponse.ok) {
const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
if (commitsResponse.status === 404) {
throw new NotFoundError(message);
}
throw new Error(message);
}
const commits = await commitsResponse.json();
if (
commits &&
commits.values &&
commits.values.length > 0 &&
commits.values[0].hash
) {
return commits.values[0].hash.substring(0, 12);
}
throw new Error(`Failed to read response from ${commitsApiUrl}`);
}
}
packages/backend-common/src/reading/__fixtures__/bitbucket-repo-with-commit-hash.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/__fixtures__/bitbucket-server-repo.zip
BIN
View File
Binary file not shown.
packages/integration/package.json
+1
View File
@@ -35,6 +35,7 @@
},
"devDependencies": {
"@backstage/cli": "^0.4.1",
"@backstage/test-utils": "^0.1.5",
"@types/jest": "^26.0.7",
"msw": "^0.21.2"
},
packages/integration/src/bitbucket/core.test.ts
+156 -1
View File
@@ -14,10 +14,21 @@
* limitations under the License.
*/
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { msw } from '@backstage/test-utils';
import { BitbucketIntegrationConfig } from './config';
import { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core';
import {
getBitbucketDefaultBranch,
getBitbucketDownloadUrl,
getBitbucketFileFetchUrl,
getBitbucketRequestOptions,
} from './core';
describe('bitbucket core', () => {
const worker = setupServer();
msw.setupDefaultHandlers(worker);
describe('getBitbucketRequestOptions', () => {
it('inserts a token when needed', () => {
const withToken: BitbucketIntegrationConfig = {
@@ -97,4 +108,148 @@ describe('bitbucket core', () => {
);
});
});
describe('getBitbucketDownloadUrl', () => {
it('add path param if a path is specified for Bitbucket Server', async () => {
const defaultBranchResponse = {
displayId: 'main',
};
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(defaultBranchResponse),
),
),
);
const config: BitbucketIntegrationConfig = {
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
};
const result = await getBitbucketDownloadUrl(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs',
config,
);
expect(result).toEqual(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&at=main&prefix=backstage-mock&path=docs',
);
});
it('do not add path param if no path is specified for Bitbucket Server', async () => {
const defaultBranchResponse = {
displayId: 'main',
};
worker.use(
rest.get(
// TODO: Change URL to https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default when git-url-parse bug is fixed
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/projects/backstage/repos/mock/repos/browse/branches/default',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(defaultBranchResponse),
),
),
);
const config: BitbucketIntegrationConfig = {
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
};
const result = await getBitbucketDownloadUrl(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse?at=main',
config,
);
expect(new URL(result).searchParams.get('format')).toEqual('zip');
expect(new URL(result).searchParams.get('at')).toEqual('main');
expect(new URL(result).searchParams.get('prefix')).not.toBeNull();
expect(new URL(result).searchParams.get('path')).toBeNull();
});
it('get by branch for Bitbucket Server', async () => {
const config: BitbucketIntegrationConfig = {
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
};
const result = await getBitbucketDownloadUrl(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
config,
);
expect(result).toEqual(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&at=some-branch&prefix=backstage-mock&path=docs',
);
});
it('do not add path param for Bitbucket Cloud', async () => {
const config: BitbucketIntegrationConfig = {
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
};
const result = await getBitbucketDownloadUrl(
'https://bitbucket.org/backstage/mock/src/master',
config,
);
expect(result).toEqual(
'https://bitbucket.org/backstage/mock/get/master.zip',
);
});
});
describe('getBitbucketDefaultBranch', () => {
it('return default branch for Bitbucket Cloud', async () => {
const repoInfoResponse = {
mainbranch: {
name: 'main',
},
};
worker.use(
rest.get(
'https://api.bitbucket.org/2.0/repositories/backstage/mock',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(repoInfoResponse),
),
),
);
const config: BitbucketIntegrationConfig = {
host: 'bitbucket.org',
apiBaseUrl: 'https://api.bitbucket.org/2.0',
};
const defaultBranch = await getBitbucketDefaultBranch(
'https://bitbucket.org/backstage/mock/src/main',
config,
);
expect(defaultBranch).toEqual('main');
});
it('return default branch for Bitbucket Server', async () => {
const defaultBranchResponse = {
displayId: 'main',
};
worker.use(
rest.get(
'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default',
(_, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/json'),
ctx.json(defaultBranchResponse),
),
),
);
const config: BitbucketIntegrationConfig = {
host: 'bitbucket.mycompany.net',
apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
};
const defaultBranch = await getBitbucketDefaultBranch(
'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/README.md',
config,
);
expect(defaultBranch).toEqual('main');
});
});
});
packages/integration/src/bitbucket/core.ts
+75
View File
@@ -14,9 +14,84 @@
* limitations under the License.
*/
import fetch from 'cross-fetch';
import parseGitUrl from 'git-url-parse';
import { BitbucketIntegrationConfig } from './config';
/**
* Given a URL pointing to a path on a provider, returns the default branch.
*
* @param url A URL pointing to a path
* @param config The relevant provider config
*/
export async function getBitbucketDefaultBranch(
url: string,
config: BitbucketIntegrationConfig,
): Promise<string> {
const { name: repoName, owner: project, resource } = parseGitUrl(url);
const isHosted = resource === 'bitbucket.org';
const branchUrl = isHosted
? `${config.apiBaseUrl}/repositories/${project}/${repoName}`
: `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches/default`;
const response = await fetch(branchUrl, getBitbucketRequestOptions(config));
if (!response.ok) {
const message = `Failed to retrieve default branch from ${branchUrl}, ${response.status} ${response.statusText}`;
throw new Error(message);
}
let defaultBranch;
if (isHosted) {
const repoInfo = await response.json();
defaultBranch = repoInfo.mainbranch.name;
} else {
const { displayId } = await response.json();
defaultBranch = displayId;
}
if (!defaultBranch) {
throw new Error(`Failed to read default branch from ${branchUrl}`);
}
return defaultBranch;
}
/**
* Given a URL pointing to a path on a provider, returns a URL that is suitable
* for downloading the subtree.
*
* @param url A URL pointing to a path
* @param config The relevant provider config
*/
export async function getBitbucketDownloadUrl(
url: string,
config: BitbucketIntegrationConfig,
): Promise<string> {
const {
name: repoName,
owner: project,
ref,
protocol,
resource,
filepath,
} = parseGitUrl(url);
const isHosted = resource === 'bitbucket.org';
let branch = ref;
if (!branch) {
branch = await getBitbucketDefaultBranch(url, config);
}
// path will limit the downloaded content
// /docs will only download the docs folder and everything below it
// /docs/index.md will download the docs folder and everything below it
const path = filepath ? `&path=${encodeURIComponent(filepath)}` : '';
const archiveUrl = isHosted
? `${protocol}://${resource}/${project}/${repoName}/get/${branch}.zip`
: `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/archive?format=zip&at=${branch}&prefix=${project}-${repoName}${path}`;
return archiveUrl;
}
/**
* Given a URL pointing to a file on a provider, returns a URL that is suitable
* for fetching the contents of the data.
packages/integration/src/bitbucket/index.ts
+6 -1
View File
@@ -19,4 +19,9 @@ export {
readBitbucketIntegrationConfigs,
} from './config';
export type { BitbucketIntegrationConfig } from './config';
export { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core';
export {
getBitbucketDefaultBranch,
getBitbucketDownloadUrl,
getBitbucketFileFetchUrl,
getBitbucketRequestOptions,
} from './core';