integration: move the core url and auth logic to integration for the four major providers
This commit is contained in:
Fredrik Adelöw
@@ -0,0 +1,6 @@
|
||||
---
|
||||
'@backstage/backend-common': patch
|
||||
'@backstage/integration': patch
|
||||
---
|
||||
|
||||
Move the core url and auth logic to integration for the four major providers
|
||||
@@ -20,7 +20,7 @@ import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { getVoidLogger } from '../logging';
|
||||
import { AzureUrlReader, getDownloadUrl } from './AzureUrlReader';
|
||||
import { AzureUrlReader } from './AzureUrlReader';
|
||||
import { msw } from '@backstage/test-utils';
|
||||
import { ReadTreeResponseFactory } from './tree';
|
||||
|
||||
@@ -111,13 +111,13 @@ describe('AzureUrlReader', () => {
|
||||
url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
|
||||
config: createConfig(),
|
||||
error:
|
||||
'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
|
||||
'Incorrect URL: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
|
||||
},
|
||||
{
|
||||
url: 'com/a/b/blob/master/path/to/c.yaml',
|
||||
config: createConfig(),
|
||||
error:
|
||||
'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
|
||||
'Incorrect URL: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
|
||||
},
|
||||
{
|
||||
url: '',
|
||||
@@ -178,21 +178,4 @@ describe('AzureUrlReader', () => {
|
||||
expect(indexMarkdownFile.toString()).toBe('# Test\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDownloadUrl', () => {
|
||||
it('do not add scopePath if no path is specified', async () => {
|
||||
const result = getDownloadUrl(
|
||||
'https://dev.azure.com/organization/project/_git/repository',
|
||||
);
|
||||
|
||||
expect(result.searchParams.get('scopePath')).toBeNull();
|
||||
});
|
||||
|
||||
it('add scopePath if a path is specified', async () => {
|
||||
const result = getDownloadUrl(
|
||||
'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs',
|
||||
);
|
||||
expect(result.searchParams.get('scopePath')).toEqual('docs');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,10 +17,12 @@
|
||||
import {
|
||||
AzureIntegrationConfig,
|
||||
readAzureIntegrationConfigs,
|
||||
getAzureFileFetchUrl,
|
||||
getAzureDownloadUrl,
|
||||
getAzureRequestOptions,
|
||||
} from '@backstage/integration';
|
||||
import fetch from 'cross-fetch';
|
||||
import { Readable } from 'stream';
|
||||
import parseGitUri from 'git-url-parse';
|
||||
import { NotFoundError } from '../errors';
|
||||
import {
|
||||
ReaderFactory,
|
||||
@@ -30,28 +32,6 @@ import {
|
||||
} from './types';
|
||||
import { ReadTreeResponseFactory } from './tree';
|
||||
|
||||
export function getDownloadUrl(url: string): URL {
|
||||
const {
|
||||
name: repoName,
|
||||
owner: project,
|
||||
organization,
|
||||
protocol,
|
||||
resource,
|
||||
filepath,
|
||||
} = parseGitUri(url);
|
||||
|
||||
// scopePath will limit the downloaded content
|
||||
// /docs will only download the docs folder and everything below it
|
||||
// /docs/index.md will only download index.md but put it in the root of the archive
|
||||
const scopePath = filepath
|
||||
? `&scopePath=${encodeURIComponent(filepath)}`
|
||||
: '';
|
||||
|
||||
return new URL(
|
||||
`${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`,
|
||||
);
|
||||
}
|
||||
|
||||
export class AzureUrlReader implements UrlReader {
|
||||
static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
|
||||
const configs = readAzureIntegrationConfigs(
|
||||
@@ -76,11 +56,11 @@ export class AzureUrlReader implements UrlReader {
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const builtUrl = this.buildRawUrl(url);
|
||||
const builtUrl = getAzureFileFetchUrl(url);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(builtUrl.toString(), this.getRequestOptions());
|
||||
response = await fetch(builtUrl, getAzureRequestOptions(this.options));
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
@@ -102,8 +82,8 @@ export class AzureUrlReader implements UrlReader {
|
||||
options?: ReadTreeOptions,
|
||||
): Promise<ReadTreeResponse> {
|
||||
const response = await fetch(
|
||||
getDownloadUrl(url).toString(),
|
||||
this.getRequestOptions({ Accept: 'application/zip' }),
|
||||
getAzureDownloadUrl(url),
|
||||
getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
|
||||
@@ -119,80 +99,6 @@ export class AzureUrlReader implements UrlReader {
|
||||
});
|
||||
}
|
||||
|
||||
// Converts
|
||||
// from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
|
||||
// to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch}
|
||||
private buildRawUrl(target: string): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const [
|
||||
empty,
|
||||
userOrOrg,
|
||||
project,
|
||||
srcKeyword,
|
||||
repoName,
|
||||
] = url.pathname.split('/');
|
||||
|
||||
const path = url.searchParams.get('path') || '';
|
||||
const ref = url.searchParams.get('version')?.substr(2);
|
||||
|
||||
if (
|
||||
url.hostname !== 'dev.azure.com' ||
|
||||
empty !== '' ||
|
||||
userOrOrg === '' ||
|
||||
project === '' ||
|
||||
srcKeyword !== '_git' ||
|
||||
repoName === '' ||
|
||||
path === '' ||
|
||||
ref === ''
|
||||
) {
|
||||
throw new Error('Wrong Azure Devops URL or Invalid file path');
|
||||
}
|
||||
|
||||
// transform to api
|
||||
url.pathname = [
|
||||
empty,
|
||||
userOrOrg,
|
||||
project,
|
||||
'_apis',
|
||||
'git',
|
||||
'repositories',
|
||||
repoName,
|
||||
'items',
|
||||
].join('/');
|
||||
|
||||
const queryParams = [`path=${path}`];
|
||||
|
||||
if (ref) {
|
||||
queryParams.push(`version=${ref}`);
|
||||
}
|
||||
|
||||
url.search = queryParams.join('&');
|
||||
|
||||
url.protocol = 'https';
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
private getRequestOptions(additionalHeaders?: {
|
||||
[key: string]: string;
|
||||
}): RequestInit {
|
||||
const headers: HeadersInit = additionalHeaders ?? {};
|
||||
|
||||
if (this.options.token) {
|
||||
headers.Authorization = `Basic ${Buffer.from(
|
||||
`:${this.options.token}`,
|
||||
'utf8',
|
||||
).toString('base64')}`;
|
||||
}
|
||||
|
||||
return { headers };
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, token } = this.options;
|
||||
return `azure{host=${host},authed=${Boolean(token)}}`;
|
||||
|
||||
@@ -14,94 +14,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { BitbucketIntegrationConfig } from '@backstage/integration';
|
||||
import {
|
||||
BitbucketUrlReader,
|
||||
getApiRequestOptions,
|
||||
getApiUrl,
|
||||
} from './BitbucketUrlReader';
|
||||
import { BitbucketUrlReader } from './BitbucketUrlReader';
|
||||
|
||||
describe('BitbucketUrlReader', () => {
|
||||
describe('getApiRequestOptions', () => {
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getApiRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('Bearer A');
|
||||
expect(
|
||||
(getApiRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('insert basic auth when needed', () => {
|
||||
const withUsernameAndPassword: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
username: 'some-user',
|
||||
appPassword: 'my-secret',
|
||||
};
|
||||
const withoutUsernameAndPassword: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getApiRequestOptions(withUsernameAndPassword).headers as any)
|
||||
.Authorization,
|
||||
).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA==');
|
||||
expect(
|
||||
(getApiRequestOptions(withoutUsernameAndPassword).headers as any)
|
||||
.Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getApiUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '' };
|
||||
expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
|
||||
});
|
||||
it('happy path for Bitbucket Cloud', () => {
|
||||
const config: BitbucketIntegrationConfig = {
|
||||
host: 'bitbucket.org',
|
||||
apiBaseUrl: 'https://api.bitbucket.org/2.0',
|
||||
};
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml',
|
||||
),
|
||||
);
|
||||
});
|
||||
it('happy path for Bitbucket Server', () => {
|
||||
const config: BitbucketIntegrationConfig = {
|
||||
host: 'bitbucket.mycompany.net',
|
||||
apiBaseUrl: 'https://bitbucket.mycompany.net/rest/api/1.0',
|
||||
};
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml',
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation', () => {
|
||||
it('rejects unknown targets', async () => {
|
||||
const processor = new BitbucketUrlReader({
|
||||
|
||||
@@ -16,69 +16,14 @@
|
||||
|
||||
import {
|
||||
BitbucketIntegrationConfig,
|
||||
getBitbucketFileFetchUrl,
|
||||
getBitbucketRequestOptions,
|
||||
readBitbucketIntegrationConfigs,
|
||||
} from '@backstage/integration';
|
||||
import fetch from 'cross-fetch';
|
||||
import parseGitUri from 'git-url-parse';
|
||||
import { NotFoundError } from '../errors';
|
||||
import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
|
||||
|
||||
export function getApiRequestOptions(
|
||||
provider: BitbucketIntegrationConfig,
|
||||
): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `Bearer ${provider.token}`;
|
||||
} else if (provider.username && provider.appPassword) {
|
||||
headers.Authorization = `Basic ${Buffer.from(
|
||||
`${provider.username}:${provider.appPassword}`,
|
||||
'utf8',
|
||||
).toString('base64')}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
// Converts for example
|
||||
// from: https://bitbucket.org/orgname/reponame/src/master/file.yaml
|
||||
// to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml
|
||||
export function getApiUrl(
|
||||
target: string,
|
||||
provider: BitbucketIntegrationConfig,
|
||||
): URL {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
(filepathtype !== 'browse' &&
|
||||
filepathtype !== 'raw' &&
|
||||
filepathtype !== 'src')
|
||||
) {
|
||||
throw new Error('Invalid Bitbucket URL or file path');
|
||||
}
|
||||
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
|
||||
if (provider.host === 'bitbucket.org') {
|
||||
if (!ref) {
|
||||
throw new Error('Invalid Bitbucket URL or file path');
|
||||
}
|
||||
return new URL(
|
||||
`${provider.apiBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`,
|
||||
);
|
||||
}
|
||||
return new URL(
|
||||
`${provider.apiBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`,
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as
|
||||
* the one exposed by Bitbucket Cloud itself.
|
||||
@@ -116,9 +61,8 @@ export class BitbucketUrlReader implements UrlReader {
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const bitbucketUrl = getApiUrl(url, this.config);
|
||||
|
||||
const options = getApiRequestOptions(this.config);
|
||||
const bitbucketUrl = getBitbucketFileFetchUrl(url, this.config);
|
||||
const options = getBitbucketRequestOptions(this.config);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
|
||||
@@ -15,19 +15,12 @@
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { GitHubIntegrationConfig } from '@backstage/integration';
|
||||
import { msw } from '@backstage/test-utils';
|
||||
import fs from 'fs';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import path from 'path';
|
||||
import {
|
||||
getApiRequestOptions,
|
||||
getApiUrl,
|
||||
getRawRequestOptions,
|
||||
getRawUrl,
|
||||
GithubUrlReader,
|
||||
} from './GithubUrlReader';
|
||||
import { GithubUrlReader } from './GithubUrlReader';
|
||||
import { ReadTreeResponseFactory } from './tree';
|
||||
|
||||
const treeResponseFactory = ReadTreeResponseFactory.create({
|
||||
@@ -35,143 +28,6 @@ const treeResponseFactory = ReadTreeResponseFactory.create({
|
||||
});
|
||||
|
||||
describe('GithubUrlReader', () => {
|
||||
describe('getApiRequestOptions', () => {
|
||||
it('sets the correct API version', () => {
|
||||
const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' };
|
||||
expect((getApiRequestOptions(config).headers as any).Accept).toEqual(
|
||||
'application/vnd.github.v3.raw',
|
||||
);
|
||||
});
|
||||
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: GitHubIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: GitHubIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getApiRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('token A');
|
||||
expect(
|
||||
(getApiRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRawRequestOptions', () => {
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: GitHubIntegrationConfig = {
|
||||
host: '',
|
||||
rawBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: GitHubIntegrationConfig = {
|
||||
host: '',
|
||||
rawBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getRawRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('token A');
|
||||
expect(
|
||||
(getRawRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getApiUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' };
|
||||
expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
|
||||
});
|
||||
|
||||
it('happy path for github', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
};
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://github.com/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for ghe', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'ghe.mycompany.net',
|
||||
apiBaseUrl: 'https://ghe.mycompany.net/api/v3',
|
||||
};
|
||||
expect(
|
||||
getApiUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRawUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' };
|
||||
expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
|
||||
});
|
||||
|
||||
it('happy path for github', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'github.com',
|
||||
rawBaseUrl: 'https://raw.githubusercontent.com',
|
||||
};
|
||||
expect(
|
||||
getRawUrl(
|
||||
'https://github.com/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL(
|
||||
'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for ghe', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'ghe.mycompany.net',
|
||||
rawBaseUrl: 'https://ghe.mycompany.net/raw',
|
||||
};
|
||||
expect(
|
||||
getRawUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation', () => {
|
||||
it('rejects unknown targets', async () => {
|
||||
const processor = new GithubUrlReader(
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
import {
|
||||
GitHubIntegrationConfig,
|
||||
readGitHubIntegrationConfigs,
|
||||
getGitHubFileFetchUrl,
|
||||
getGitHubRequestOptions,
|
||||
} from '@backstage/integration';
|
||||
import fetch from 'cross-fetch';
|
||||
import parseGitUri from 'git-url-parse';
|
||||
@@ -30,92 +32,6 @@ import {
|
||||
UrlReader,
|
||||
} from './types';
|
||||
|
||||
export function getApiRequestOptions(
|
||||
provider: GitHubIntegrationConfig,
|
||||
): RequestInit {
|
||||
const headers: HeadersInit = {
|
||||
Accept: 'application/vnd.github.v3.raw',
|
||||
};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `token ${provider.token}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
export function getRawRequestOptions(
|
||||
provider: GitHubIntegrationConfig,
|
||||
): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
if (provider.token) {
|
||||
headers.Authorization = `token ${provider.token}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
|
||||
// Converts for example
|
||||
// from: https://github.com/a/b/blob/branchname/path/to/c.yaml
|
||||
// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname
|
||||
export function getApiUrl(
|
||||
target: string,
|
||||
provider: GitHubIntegrationConfig,
|
||||
): URL {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
|
||||
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
!ref ||
|
||||
(filepathtype !== 'blob' && filepathtype !== 'raw')
|
||||
) {
|
||||
throw new Error('Invalid GitHub URL or file path');
|
||||
}
|
||||
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
return new URL(
|
||||
`${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`,
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Converts for example
|
||||
// from: https://github.com/a/b/blob/branchname/c.yaml
|
||||
// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml
|
||||
export function getRawUrl(
|
||||
target: string,
|
||||
provider: GitHubIntegrationConfig,
|
||||
): URL {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
|
||||
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
!ref ||
|
||||
(filepathtype !== 'blob' && filepathtype !== 'raw')
|
||||
) {
|
||||
throw new Error('Invalid GitHub URL or file path');
|
||||
}
|
||||
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
return new URL(
|
||||
`${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`,
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A processor that adds the ability to read files from GitHub v3 APIs, such as
|
||||
* the one exposed by GitHub itself.
|
||||
@@ -144,14 +60,8 @@ export class GithubUrlReader implements UrlReader {
|
||||
}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
const useApi =
|
||||
this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl);
|
||||
const ghUrl = useApi
|
||||
? getApiUrl(url, this.config)
|
||||
: getRawUrl(url, this.config);
|
||||
const options = useApi
|
||||
? getApiRequestOptions(this.config)
|
||||
: getRawRequestOptions(this.config);
|
||||
const ghUrl = getGitHubFileFetchUrl(url, this.config);
|
||||
const options = getGitHubRequestOptions(this.config);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
@@ -196,7 +106,7 @@ export class GithubUrlReader implements UrlReader {
|
||||
new URL(
|
||||
`${protocol}://${resource}/${full_name}/archive/${ref}.tar.gz`,
|
||||
).toString(),
|
||||
getRawRequestOptions(this.config),
|
||||
getGitHubRequestOptions(this.config),
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
*/
|
||||
|
||||
import {
|
||||
getGitLabFileFetchUrl,
|
||||
getGitLabRequestOptions,
|
||||
GitLabIntegrationConfig,
|
||||
readGitLabIntegrationConfigs,
|
||||
} from '@backstage/integration';
|
||||
@@ -37,20 +39,11 @@ export class GitlabUrlReader implements UrlReader {
|
||||
constructor(private readonly options: GitLabIntegrationConfig) {}
|
||||
|
||||
async read(url: string): Promise<Buffer> {
|
||||
// TODO(Rugvip): merged the old GitlabReaderProcessor in here and used
|
||||
// the existence of /~/blob/ to switch the logic. Don't know if this
|
||||
// makes sense and it might require some more work.
|
||||
let builtUrl: URL;
|
||||
if (url.includes('/-/blob/')) {
|
||||
const projectID = await this.getProjectID(url);
|
||||
builtUrl = this.buildProjectUrl(url, projectID);
|
||||
} else {
|
||||
builtUrl = this.buildRawUrl(url);
|
||||
}
|
||||
const builtUrl = await getGitLabFileFetchUrl(url, this.options);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(builtUrl.toString(), this.getRequestOptions());
|
||||
response = await fetch(builtUrl, getGitLabRequestOptions(this.options));
|
||||
} catch (e) {
|
||||
throw new Error(`Unable to read ${url}, ${e}`);
|
||||
}
|
||||
@@ -70,109 +63,6 @@ export class GitlabUrlReader implements UrlReader {
|
||||
throw new Error('GitlabUrlReader does not implement readTree');
|
||||
}
|
||||
|
||||
// Converts
|
||||
// from: https://gitlab.example.com/a/b/blob/master/c.yaml
|
||||
// to: https://gitlab.example.com/a/b/raw/master/c.yaml
|
||||
private buildRawUrl(target: string): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const [
|
||||
empty,
|
||||
userOrOrg,
|
||||
repoName,
|
||||
blobKeyword,
|
||||
...restOfPath
|
||||
] = url.pathname.split('/');
|
||||
|
||||
if (
|
||||
empty !== '' ||
|
||||
userOrOrg === '' ||
|
||||
repoName === '' ||
|
||||
blobKeyword !== 'blob' ||
|
||||
!restOfPath.join('/').match(/\.yaml$/)
|
||||
) {
|
||||
throw new Error('Wrong GitLab URL');
|
||||
}
|
||||
|
||||
// Replace 'blob' with 'raw'
|
||||
url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join(
|
||||
'/',
|
||||
);
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
// convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
// to https://gitlab.com/api/v4/projects/<PROJECTID>/repository/files/filepath?ref=branch
|
||||
private buildProjectUrl(target: string, projectID: Number): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const branchAndFilePath = url.pathname.split('/-/blob/')[1];
|
||||
|
||||
const [branch, ...filePath] = branchAndFilePath.split('/');
|
||||
|
||||
url.pathname = [
|
||||
'/api/v4/projects',
|
||||
projectID,
|
||||
'repository/files',
|
||||
encodeURIComponent(filePath.join('/')),
|
||||
'raw',
|
||||
].join('/');
|
||||
url.search = `?ref=${branch}`;
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async getProjectID(target: string): Promise<Number> {
|
||||
const url = new URL(target);
|
||||
|
||||
if (
|
||||
// absPaths to gitlab files should contain /-/blob
|
||||
// ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
!url.pathname.match(/\/\-\/blob\//)
|
||||
) {
|
||||
throw new Error('Please provide full path to yaml file from Gitlab');
|
||||
}
|
||||
try {
|
||||
const repo = url.pathname.split('/-/blob/')[0];
|
||||
|
||||
// Find ProjectID from url
|
||||
// convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath'
|
||||
// to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo'
|
||||
const repoIDLookup = new URL(
|
||||
`${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent(
|
||||
repo.replace(/^\//, ''),
|
||||
)}`,
|
||||
);
|
||||
const response = await fetch(
|
||||
repoIDLookup.toString(),
|
||||
this.getRequestOptions(),
|
||||
);
|
||||
const projectIDJson = await response.json();
|
||||
const projectID: Number = projectIDJson.id;
|
||||
|
||||
return projectID;
|
||||
} catch (e) {
|
||||
throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
private getRequestOptions(): RequestInit {
|
||||
return {
|
||||
headers: {
|
||||
['PRIVATE-TOKEN']: this.options.token ?? '',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
toString() {
|
||||
const { host, token } = this.options;
|
||||
return `gitlab{host=${host},authed=${Boolean(token)}}`;
|
||||
|
||||
@@ -21,11 +21,13 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@backstage/config": "^0.1.1",
|
||||
"cross-fetch": "^3.0.6",
|
||||
"git-url-parse": "^11.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/cli": "^0.4.0",
|
||||
"@types/jest": "^26.0.7"
|
||||
"@types/jest": "^26.0.7",
|
||||
"msw": "^0.21.2"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
getAzureFileFetchUrl,
|
||||
getAzureDownloadUrl,
|
||||
getAzureRequestOptions,
|
||||
} from './core';
|
||||
|
||||
describe('azure core', () => {
|
||||
describe('getAzureRequestOptions', () => {
|
||||
it('fills in the token if necessary', () => {
|
||||
expect(getAzureRequestOptions({ host: '', token: '0123456789' })).toEqual(
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Authorization: 'Basic OjAxMjM0NTY3ODk=',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(getAzureRequestOptions({ host: '' })).toEqual(
|
||||
expect.objectContaining({
|
||||
headers: expect.not.objectContaining({
|
||||
Authorization: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAzureFileFetchUrl', () => {
|
||||
it.each([
|
||||
{
|
||||
url:
|
||||
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
|
||||
result:
|
||||
'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master',
|
||||
},
|
||||
{
|
||||
url:
|
||||
'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
|
||||
result:
|
||||
'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml',
|
||||
},
|
||||
])('should handle happy path %#', async ({ url, result }) => {
|
||||
expect(getAzureFileFetchUrl(url)).toBe(result);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
|
||||
error:
|
||||
'Incorrect URL: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
|
||||
},
|
||||
{
|
||||
url: 'com/a/b/blob/master/path/to/c.yaml',
|
||||
error:
|
||||
'Incorrect URL: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
|
||||
},
|
||||
])('should handle error path %#', ({ url, error }) => {
|
||||
expect(() => getAzureFileFetchUrl(url)).toThrow(error);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAzureDownloadUrl', () => {
|
||||
it('do not add scopePath if no path is specified', async () => {
|
||||
const result = getAzureDownloadUrl(
|
||||
'https://dev.azure.com/organization/project/_git/repository',
|
||||
);
|
||||
|
||||
expect(new URL(result).searchParams.get('scopePath')).toBeNull();
|
||||
});
|
||||
|
||||
it('add scopePath if a path is specified', async () => {
|
||||
const result = getAzureDownloadUrl(
|
||||
'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs',
|
||||
);
|
||||
expect(new URL(result).searchParams.get('scopePath')).toEqual('docs');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { AzureIntegrationConfig } from './config';
|
||||
|
||||
/**
|
||||
* Given a URL pointing to a file on a provider, returns a URL that is suitable
|
||||
* for fetching the contents of the data.
|
||||
*
|
||||
* Converts
|
||||
* from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
|
||||
* to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch}
|
||||
*
|
||||
* @param url A URL pointing to a file
|
||||
*/
|
||||
export function getAzureFileFetchUrl(url: string): string {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
|
||||
const [
|
||||
empty,
|
||||
userOrOrg,
|
||||
project,
|
||||
srcKeyword,
|
||||
repoName,
|
||||
] = parsedUrl.pathname.split('/');
|
||||
|
||||
const path = parsedUrl.searchParams.get('path') || '';
|
||||
const ref = parsedUrl.searchParams.get('version')?.substr(2);
|
||||
|
||||
if (
|
||||
parsedUrl.hostname !== 'dev.azure.com' ||
|
||||
empty !== '' ||
|
||||
userOrOrg === '' ||
|
||||
project === '' ||
|
||||
srcKeyword !== '_git' ||
|
||||
repoName === '' ||
|
||||
path === '' ||
|
||||
ref === ''
|
||||
) {
|
||||
throw new Error('Wrong Azure Devops URL or Invalid file path');
|
||||
}
|
||||
|
||||
// transform to api
|
||||
parsedUrl.pathname = [
|
||||
empty,
|
||||
userOrOrg,
|
||||
project,
|
||||
'_apis',
|
||||
'git',
|
||||
'repositories',
|
||||
repoName,
|
||||
'items',
|
||||
].join('/');
|
||||
|
||||
const queryParams = [`path=${path}`];
|
||||
|
||||
if (ref) {
|
||||
queryParams.push(`version=${ref}`);
|
||||
}
|
||||
|
||||
parsedUrl.search = queryParams.join('&');
|
||||
|
||||
parsedUrl.protocol = 'https';
|
||||
|
||||
return parsedUrl.toString();
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${url}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a URL pointing to a path on a provider, returns a URL that is suitable
|
||||
* for downloading the subtree.
|
||||
*
|
||||
* @param url A URL pointing to a path
|
||||
*/
|
||||
export function getAzureDownloadUrl(url: string): string {
|
||||
const {
|
||||
name: repoName,
|
||||
owner: project,
|
||||
organization,
|
||||
protocol,
|
||||
resource,
|
||||
filepath,
|
||||
} = parseGitUrl(url);
|
||||
|
||||
// scopePath will limit the downloaded content
|
||||
// /docs will only download the docs folder and everything below it
|
||||
// /docs/index.md will only download index.md but put it in the root of the archive
|
||||
const scopePath = filepath
|
||||
? `&scopePath=${encodeURIComponent(filepath)}`
|
||||
: '';
|
||||
|
||||
return `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the request options necessary to make requests to a given provider.
|
||||
*
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export function getAzureRequestOptions(
|
||||
config: AzureIntegrationConfig,
|
||||
additionalHeaders?: Record<string, string>,
|
||||
): RequestInit {
|
||||
const headers: HeadersInit = additionalHeaders
|
||||
? { ...additionalHeaders }
|
||||
: {};
|
||||
|
||||
if (config.token) {
|
||||
const buffer = Buffer.from(`:${config.token}`, 'utf8');
|
||||
headers.Authorization = `Basic ${buffer.toString('base64')}`;
|
||||
}
|
||||
|
||||
return { headers };
|
||||
}
|
||||
@@ -19,3 +19,8 @@ export {
|
||||
readAzureIntegrationConfigs,
|
||||
} from './config';
|
||||
export type { AzureIntegrationConfig } from './config';
|
||||
export {
|
||||
getAzureDownloadUrl,
|
||||
getAzureFileFetchUrl,
|
||||
getAzureRequestOptions,
|
||||
} from './core';
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { BitbucketIntegrationConfig } from './config';
|
||||
import { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core';
|
||||
|
||||
describe('bitbucket core', () => {
|
||||
describe('getBitbucketRequestOptions', () => {
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getBitbucketRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('Bearer A');
|
||||
expect(
|
||||
(getBitbucketRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('insert basic auth when needed', () => {
|
||||
const withUsernameAndPassword: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
username: 'some-user',
|
||||
appPassword: 'my-secret',
|
||||
};
|
||||
const withoutUsernameAndPassword: BitbucketIntegrationConfig = {
|
||||
host: '',
|
||||
apiBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getBitbucketRequestOptions(withUsernameAndPassword).headers as any)
|
||||
.Authorization,
|
||||
).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA==');
|
||||
expect(
|
||||
(getBitbucketRequestOptions(withoutUsernameAndPassword).headers as any)
|
||||
.Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getBitbucketFileFetchUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '' };
|
||||
expect(() => getBitbucketFileFetchUrl('a/b', config)).toThrow(
|
||||
/Incorrect URL: a\/b/,
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for Bitbucket Cloud', () => {
|
||||
const config: BitbucketIntegrationConfig = {
|
||||
host: 'bitbucket.org',
|
||||
apiBaseUrl: 'https://api.bitbucket.org/2.0',
|
||||
};
|
||||
expect(
|
||||
getBitbucketFileFetchUrl(
|
||||
'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml',
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for Bitbucket Server', () => {
|
||||
const config: BitbucketIntegrationConfig = {
|
||||
host: 'bitbucket.mycompany.net',
|
||||
apiBaseUrl: 'https://bitbucket.mycompany.net/rest/api/1.0',
|
||||
};
|
||||
expect(
|
||||
getBitbucketFileFetchUrl(
|
||||
'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml?at=',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { BitbucketIntegrationConfig } from './config';
|
||||
|
||||
/**
|
||||
* Given a URL pointing to a file on a provider, returns a URL that is suitable
|
||||
* for fetching the contents of the data.
|
||||
*
|
||||
* Converts
|
||||
* from: https://bitbucket.org/orgname/reponame/src/master/file.yaml
|
||||
* to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml
|
||||
*
|
||||
* @param url A URL pointing to a file
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export function getBitbucketFileFetchUrl(
|
||||
url: string,
|
||||
config: BitbucketIntegrationConfig,
|
||||
): string {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUrl(url);
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
(filepathtype !== 'browse' &&
|
||||
filepathtype !== 'raw' &&
|
||||
filepathtype !== 'src')
|
||||
) {
|
||||
throw new Error('Invalid Bitbucket URL or file path');
|
||||
}
|
||||
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
|
||||
if (config.host === 'bitbucket.org') {
|
||||
if (!ref) {
|
||||
throw new Error('Invalid Bitbucket URL or file path');
|
||||
}
|
||||
return `${config.apiBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`;
|
||||
}
|
||||
return `${config.apiBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${url}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the request options necessary to make requests to a given provider.
|
||||
*
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export function getBitbucketRequestOptions(
|
||||
config: BitbucketIntegrationConfig,
|
||||
): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
if (config.token) {
|
||||
headers.Authorization = `Bearer ${config.token}`;
|
||||
} else if (config.username && config.appPassword) {
|
||||
const buffer = Buffer.from(
|
||||
`${config.username}:${config.appPassword}`,
|
||||
'utf8',
|
||||
);
|
||||
headers.Authorization = `Basic ${buffer.toString('base64')}`;
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
};
|
||||
}
|
||||
@@ -19,3 +19,4 @@ export {
|
||||
readBitbucketIntegrationConfigs,
|
||||
} from './config';
|
||||
export type { BitbucketIntegrationConfig } from './config';
|
||||
export { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core';
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { GitHubIntegrationConfig } from './config';
|
||||
import { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core';
|
||||
|
||||
describe('github core', () => {
|
||||
describe('getGitHubRequestOptions', () => {
|
||||
it('inserts a token when needed', () => {
|
||||
const withToken: GitHubIntegrationConfig = {
|
||||
host: '',
|
||||
rawBaseUrl: '',
|
||||
token: 'A',
|
||||
};
|
||||
const withoutToken: GitHubIntegrationConfig = {
|
||||
host: '',
|
||||
rawBaseUrl: '',
|
||||
};
|
||||
expect(
|
||||
(getGitHubRequestOptions(withToken).headers as any).Authorization,
|
||||
).toEqual('token A');
|
||||
expect(
|
||||
(getGitHubRequestOptions(withoutToken).headers as any).Authorization,
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGitHubFileFetchUrl', () => {
|
||||
it('rejects targets that do not look like URLs', () => {
|
||||
const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' };
|
||||
expect(() => getGitHubFileFetchUrl('a/b', config)).toThrow(
|
||||
/Incorrect URL: a\/b/,
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for github api', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'github.com',
|
||||
apiBaseUrl: 'https://api.github.com',
|
||||
};
|
||||
expect(
|
||||
getGitHubFileFetchUrl(
|
||||
'https://github.com/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
);
|
||||
expect(
|
||||
getGitHubFileFetchUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for ghe api', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'ghe.mycompany.net',
|
||||
apiBaseUrl: 'https://ghe.mycompany.net/api/v3',
|
||||
};
|
||||
expect(
|
||||
getGitHubFileFetchUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname',
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for github raw', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'github.com',
|
||||
rawBaseUrl: 'https://raw.githubusercontent.com',
|
||||
};
|
||||
expect(
|
||||
getGitHubFileFetchUrl(
|
||||
'https://github.com/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual(
|
||||
'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml',
|
||||
);
|
||||
});
|
||||
|
||||
it('happy path for ghe raw', () => {
|
||||
const config: GitHubIntegrationConfig = {
|
||||
host: 'ghe.mycompany.net',
|
||||
rawBaseUrl: 'https://ghe.mycompany.net/raw',
|
||||
};
|
||||
expect(
|
||||
getGitHubFileFetchUrl(
|
||||
'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
|
||||
config,
|
||||
),
|
||||
).toEqual('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import parseGitUrl from 'git-url-parse';
|
||||
import { GitHubIntegrationConfig } from './config';
|
||||
|
||||
/**
|
||||
* Given a URL pointing to a file on a provider, returns a URL that is suitable
|
||||
* for fetching the contents of the data.
|
||||
*
|
||||
* Converts
|
||||
* from: https://github.com/a/b/blob/branchname/path/to/c.yaml
|
||||
* to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname
|
||||
* or: https://raw.githubusercontent.com/a/b/branchname/c.yaml
|
||||
*
|
||||
* @param url A URL pointing to a file
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export function getGitHubFileFetchUrl(
|
||||
url: string,
|
||||
config: GitHubIntegrationConfig,
|
||||
): string {
|
||||
try {
|
||||
const { owner, name, ref, filepathtype, filepath } = parseGitUrl(url);
|
||||
if (
|
||||
!owner ||
|
||||
!name ||
|
||||
!ref ||
|
||||
(filepathtype !== 'blob' && filepathtype !== 'raw')
|
||||
) {
|
||||
throw new Error('Invalid GitHub URL or file path');
|
||||
}
|
||||
|
||||
const pathWithoutSlash = filepath.replace(/^\//, '');
|
||||
if (chooseEndpoint(config) === 'api') {
|
||||
return `${config.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`;
|
||||
}
|
||||
return `${config.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect URL: ${url}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the request options necessary to make requests to a given provider.
|
||||
*
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export function getGitHubRequestOptions(
|
||||
config: GitHubIntegrationConfig,
|
||||
): RequestInit {
|
||||
const headers: HeadersInit = {};
|
||||
|
||||
if (chooseEndpoint(config) === 'api') {
|
||||
headers.Accept = 'application/vnd.github.v3.raw';
|
||||
}
|
||||
if (config.token) {
|
||||
headers.Authorization = `token ${config.token}`;
|
||||
}
|
||||
|
||||
return { headers };
|
||||
}
|
||||
|
||||
export function chooseEndpoint(config: GitHubIntegrationConfig): 'api' | 'raw' {
|
||||
if (config.apiBaseUrl && (config.token || !config.rawBaseUrl)) {
|
||||
return 'api';
|
||||
}
|
||||
return 'raw';
|
||||
}
|
||||
@@ -19,3 +19,4 @@ export {
|
||||
readGitHubIntegrationConfigs,
|
||||
} from './config';
|
||||
export type { GitHubIntegrationConfig } from './config';
|
||||
export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core';
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { GitLabIntegrationConfig } from './config';
|
||||
import { getGitLabFileFetchUrl } from './core';
|
||||
|
||||
const worker = setupServer();
|
||||
|
||||
describe('gitlab core', () => {
|
||||
beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
|
||||
afterAll(() => worker.close());
|
||||
afterEach(() => worker.resetHandlers());
|
||||
|
||||
beforeEach(() => {
|
||||
worker.use(
|
||||
rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json({ id: 12345 })),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
const configWithToken: GitLabIntegrationConfig = {
|
||||
host: 'g.com',
|
||||
token: '0123456789',
|
||||
};
|
||||
|
||||
const configWithNoToken: GitLabIntegrationConfig = {
|
||||
host: 'g.com',
|
||||
};
|
||||
|
||||
describe('getGitLabFileFetchUrl', () => {
|
||||
it.each([
|
||||
// Project URLs
|
||||
{
|
||||
config: configWithNoToken,
|
||||
url:
|
||||
'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
result:
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
},
|
||||
{
|
||||
config: configWithToken,
|
||||
url:
|
||||
'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
|
||||
result:
|
||||
'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
},
|
||||
{
|
||||
config: configWithNoToken,
|
||||
url:
|
||||
'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
|
||||
result:
|
||||
'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
|
||||
},
|
||||
// Raw URLs
|
||||
{
|
||||
config: configWithNoToken,
|
||||
url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
|
||||
result: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
|
||||
},
|
||||
])('should handle happy path %#', async ({ config, url, result }) => {
|
||||
await expect(getGitLabFileFetchUrl(url, config)).resolves.toBe(result);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { GitLabIntegrationConfig } from './config';
|
||||
import fetch from 'cross-fetch';
|
||||
|
||||
/**
|
||||
* Given a URL pointing to a file on a provider, returns a URL that is suitable
|
||||
* for fetching the contents of the data.
|
||||
*
|
||||
* Converts
|
||||
* from: https://gitlab.example.com/a/b/blob/master/c.yaml
|
||||
* to: https://gitlab.example.com/a/b/raw/master/c.yaml
|
||||
* -or-
|
||||
* from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
* to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch
|
||||
*
|
||||
* @param url A URL pointing to a file
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export async function getGitLabFileFetchUrl(
|
||||
url: string,
|
||||
config: GitLabIntegrationConfig,
|
||||
): Promise<string> {
|
||||
// TODO(Rugvip): From the old GitlabReaderProcessor; used
|
||||
// the existence of /-/blob/ to switch the logic. Don't know if this
|
||||
// makes sense and it might require some more work.
|
||||
if (url.includes('/-/blob/')) {
|
||||
const projectID = await getProjectId(url, config);
|
||||
return buildProjectUrl(url, projectID).toString();
|
||||
}
|
||||
return buildRawUrl(url).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the request options necessary to make requests to a given provider.
|
||||
*
|
||||
* @param config The relevant provider config
|
||||
*/
|
||||
export function getGitLabRequestOptions(
|
||||
config: GitLabIntegrationConfig,
|
||||
): RequestInit {
|
||||
const { token = '' } = config;
|
||||
return {
|
||||
headers: {
|
||||
'PRIVATE-TOKEN': token,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Converts
|
||||
// from: https://gitlab.example.com/a/b/blob/master/c.yaml
|
||||
// to: https://gitlab.example.com/a/b/raw/master/c.yaml
|
||||
export function buildRawUrl(target: string): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const [
|
||||
empty,
|
||||
userOrOrg,
|
||||
repoName,
|
||||
blobKeyword,
|
||||
...restOfPath
|
||||
] = url.pathname.split('/');
|
||||
|
||||
if (
|
||||
empty !== '' ||
|
||||
userOrOrg === '' ||
|
||||
repoName === '' ||
|
||||
blobKeyword !== 'blob' ||
|
||||
!restOfPath.join('/').match(/\.yaml$/)
|
||||
) {
|
||||
throw new Error('Wrong GitLab URL');
|
||||
}
|
||||
|
||||
// Replace 'blob' with 'raw'
|
||||
url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join('/');
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Converts
|
||||
// from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
// to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch
|
||||
export function buildProjectUrl(target: string, projectID: Number): URL {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
const branchAndFilePath = url.pathname.split('/-/blob/')[1];
|
||||
const [branch, ...filePath] = branchAndFilePath.split('/');
|
||||
|
||||
url.pathname = [
|
||||
'/api/v4/projects',
|
||||
projectID,
|
||||
'repository/files',
|
||||
encodeURIComponent(filePath.join('/')),
|
||||
'raw',
|
||||
].join('/');
|
||||
url.search = `?ref=${branch}`;
|
||||
|
||||
return url;
|
||||
} catch (e) {
|
||||
throw new Error(`Incorrect url: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Convert
|
||||
// from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
|
||||
// to: The project ID that corresponds to the URL
|
||||
export async function getProjectId(
|
||||
target: string,
|
||||
config: GitLabIntegrationConfig,
|
||||
): Promise<number> {
|
||||
const url = new URL(target);
|
||||
|
||||
if (!url.pathname.includes('/-/blob/')) {
|
||||
throw new Error('Please provide full path to yaml file from Gitlab');
|
||||
}
|
||||
|
||||
try {
|
||||
const repo = url.pathname.split('/-/blob/')[0];
|
||||
|
||||
// Convert
|
||||
// to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo
|
||||
const repoIDLookup = new URL(
|
||||
`${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent(
|
||||
repo.replace(/^\//, ''),
|
||||
)}`,
|
||||
);
|
||||
const response = await fetch(
|
||||
repoIDLookup.toString(),
|
||||
getGitLabRequestOptions(config),
|
||||
);
|
||||
const projectIDJson = await response.json();
|
||||
const projectID = Number(projectIDJson.id);
|
||||
|
||||
return projectID;
|
||||
} catch (e) {
|
||||
throw new Error(`Could not get GitLab project ID for: ${target}, ${e}`);
|
||||
}
|
||||
}
|
||||
@@ -19,3 +19,4 @@ export {
|
||||
readGitLabIntegrationConfigs,
|
||||
} from './config';
|
||||
export type { GitLabIntegrationConfig } from './config';
|
||||
export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core';
|
||||
|
||||
Reference in New Issue
Block a user