diff --git a/.changeset/lazy-beans-search.md b/.changeset/lazy-beans-search.md new file mode 100644 index 0000000000..1ea3c4bb80 --- /dev/null +++ b/.changeset/lazy-beans-search.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Move the core url and auth logic to integration for the four major providers diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index ab97d1b073..2c8549f917 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -20,7 +20,7 @@ import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '../logging'; -import { AzureUrlReader, getDownloadUrl } from './AzureUrlReader'; +import { AzureUrlReader } from './AzureUrlReader'; import { msw } from '@backstage/test-utils'; import { ReadTreeResponseFactory } from './tree'; @@ -111,13 +111,13 @@ describe('AzureUrlReader', () => { url: 'https://api.com/a/b/blob/master/path/to/c.yaml', config: createConfig(), error: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + 'Incorrect URL: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', }, { url: 'com/a/b/blob/master/path/to/c.yaml', config: createConfig(), error: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + 'Incorrect URL: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', }, { url: '', @@ -178,21 +178,4 @@ describe('AzureUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); }); - - describe('getDownloadUrl', () => { - it('do not add scopePath if no path is specified', async () => { - const result = getDownloadUrl( - 'https://dev.azure.com/organization/project/_git/repository', - ); - - expect(result.searchParams.get('scopePath')).toBeNull(); - }); - - it('add scopePath if a path is specified', async () => { - const result = getDownloadUrl( - 'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs', - ); - expect(result.searchParams.get('scopePath')).toEqual('docs'); - }); - }); }); diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index ad990d1d5d..ca934ee42c 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -17,10 +17,12 @@ import { AzureIntegrationConfig, readAzureIntegrationConfigs, + getAzureFileFetchUrl, + getAzureDownloadUrl, + getAzureRequestOptions, } from '@backstage/integration'; import fetch from 'cross-fetch'; import { Readable } from 'stream'; -import parseGitUri from 'git-url-parse'; import { NotFoundError } from '../errors'; import { ReaderFactory, @@ -30,28 +32,6 @@ import { } from './types'; import { ReadTreeResponseFactory } from './tree'; -export function getDownloadUrl(url: string): URL { - const { - name: repoName, - owner: project, - organization, - protocol, - resource, - filepath, - } = parseGitUri(url); - - // scopePath will limit the downloaded content - // /docs will only download the docs folder and everything below it - // /docs/index.md will only download index.md but put it in the root of the archive - const scopePath = filepath - ? `&scopePath=${encodeURIComponent(filepath)}` - : ''; - - return new URL( - `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`, - ); -} - export class AzureUrlReader implements UrlReader { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const configs = readAzureIntegrationConfigs( @@ -76,11 +56,11 @@ export class AzureUrlReader implements UrlReader { } async read(url: string): Promise { - const builtUrl = this.buildRawUrl(url); + const builtUrl = getAzureFileFetchUrl(url); let response: Response; try { - response = await fetch(builtUrl.toString(), this.getRequestOptions()); + response = await fetch(builtUrl, getAzureRequestOptions(this.options)); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -102,8 +82,8 @@ export class AzureUrlReader implements UrlReader { options?: ReadTreeOptions, ): Promise { const response = await fetch( - getDownloadUrl(url).toString(), - this.getRequestOptions({ Accept: 'application/zip' }), + getAzureDownloadUrl(url), + getAzureRequestOptions(this.options, { Accept: 'application/zip' }), ); if (!response.ok) { const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; @@ -119,80 +99,6 @@ export class AzureUrlReader implements UrlReader { }); } - // Converts - // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents - // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - project, - srcKeyword, - repoName, - ] = url.pathname.split('/'); - - const path = url.searchParams.get('path') || ''; - const ref = url.searchParams.get('version')?.substr(2); - - if ( - url.hostname !== 'dev.azure.com' || - empty !== '' || - userOrOrg === '' || - project === '' || - srcKeyword !== '_git' || - repoName === '' || - path === '' || - ref === '' - ) { - throw new Error('Wrong Azure Devops URL or Invalid file path'); - } - - // transform to api - url.pathname = [ - empty, - userOrOrg, - project, - '_apis', - 'git', - 'repositories', - repoName, - 'items', - ].join('/'); - - const queryParams = [`path=${path}`]; - - if (ref) { - queryParams.push(`version=${ref}`); - } - - url.search = queryParams.join('&'); - - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } - - private getRequestOptions(additionalHeaders?: { - [key: string]: string; - }): RequestInit { - const headers: HeadersInit = additionalHeaders ?? {}; - - if (this.options.token) { - headers.Authorization = `Basic ${Buffer.from( - `:${this.options.token}`, - 'utf8', - ).toString('base64')}`; - } - - return { headers }; - } - toString() { const { host, token } = this.options; return `azure{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 01744db28a..1c0bd39372 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,94 +14,9 @@ * limitations under the License. */ -import { BitbucketIntegrationConfig } from '@backstage/integration'; -import { - BitbucketUrlReader, - getApiRequestOptions, - getApiUrl, -} from './BitbucketUrlReader'; +import { BitbucketUrlReader } from './BitbucketUrlReader'; describe('BitbucketUrlReader', () => { - describe('getApiRequestOptions', () => { - it('inserts a token when needed', () => { - const withToken: BitbucketIntegrationConfig = { - host: '', - apiBaseUrl: '', - token: 'A', - }; - const withoutToken: BitbucketIntegrationConfig = { - host: '', - apiBaseUrl: '', - }; - expect( - (getApiRequestOptions(withToken).headers as any).Authorization, - ).toEqual('Bearer A'); - expect( - (getApiRequestOptions(withoutToken).headers as any).Authorization, - ).toBeUndefined(); - }); - - it('insert basic auth when needed', () => { - const withUsernameAndPassword: BitbucketIntegrationConfig = { - host: '', - apiBaseUrl: '', - username: 'some-user', - appPassword: 'my-secret', - }; - const withoutUsernameAndPassword: BitbucketIntegrationConfig = { - host: '', - apiBaseUrl: '', - }; - expect( - (getApiRequestOptions(withUsernameAndPassword).headers as any) - .Authorization, - ).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA=='); - expect( - (getApiRequestOptions(withoutUsernameAndPassword).headers as any) - .Authorization, - ).toBeUndefined(); - }); - }); - - describe('getApiUrl', () => { - it('rejects targets that do not look like URLs', () => { - const config: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '' }; - expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); - }); - it('happy path for Bitbucket Cloud', () => { - const config: BitbucketIntegrationConfig = { - host: 'bitbucket.org', - apiBaseUrl: 'https://api.bitbucket.org/2.0', - }; - expect( - getApiUrl( - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - config, - ), - ).toEqual( - new URL( - 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', - ), - ); - }); - it('happy path for Bitbucket Server', () => { - const config: BitbucketIntegrationConfig = { - host: 'bitbucket.mycompany.net', - apiBaseUrl: 'https://bitbucket.mycompany.net/rest/api/1.0', - }; - expect( - getApiUrl( - 'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml', - ), - ); - }); - }); - describe('implementation', () => { it('rejects unknown targets', async () => { const processor = new BitbucketUrlReader({ diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 9694c1d987..8c97bf2ee2 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -16,69 +16,14 @@ import { BitbucketIntegrationConfig, + getBitbucketFileFetchUrl, + getBitbucketRequestOptions, readBitbucketIntegrationConfigs, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import parseGitUri from 'git-url-parse'; import { NotFoundError } from '../errors'; import { ReaderFactory, ReadTreeResponse, UrlReader } from './types'; -export function getApiRequestOptions( - provider: BitbucketIntegrationConfig, -): RequestInit { - const headers: HeadersInit = {}; - - if (provider.token) { - headers.Authorization = `Bearer ${provider.token}`; - } else if (provider.username && provider.appPassword) { - headers.Authorization = `Basic ${Buffer.from( - `${provider.username}:${provider.appPassword}`, - 'utf8', - ).toString('base64')}`; - } - - return { - headers, - }; -} - -// Converts for example -// from: https://bitbucket.org/orgname/reponame/src/master/file.yaml -// to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml -export function getApiUrl( - target: string, - provider: BitbucketIntegrationConfig, -): URL { - try { - const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); - if ( - !owner || - !name || - (filepathtype !== 'browse' && - filepathtype !== 'raw' && - filepathtype !== 'src') - ) { - throw new Error('Invalid Bitbucket URL or file path'); - } - - const pathWithoutSlash = filepath.replace(/^\//, ''); - - if (provider.host === 'bitbucket.org') { - if (!ref) { - throw new Error('Invalid Bitbucket URL or file path'); - } - return new URL( - `${provider.apiBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`, - ); - } - return new URL( - `${provider.apiBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`, - ); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - /** * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as * the one exposed by Bitbucket Cloud itself. @@ -116,9 +61,8 @@ export class BitbucketUrlReader implements UrlReader { } async read(url: string): Promise { - const bitbucketUrl = getApiUrl(url, this.config); - - const options = getApiRequestOptions(this.config); + const bitbucketUrl = getBitbucketFileFetchUrl(url, this.config); + const options = getBitbucketRequestOptions(this.config); let response: Response; try { diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index abe8b4f640..f842adcf90 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -15,19 +15,12 @@ */ import { ConfigReader } from '@backstage/config'; -import { GitHubIntegrationConfig } from '@backstage/integration'; import { msw } from '@backstage/test-utils'; import fs from 'fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import path from 'path'; -import { - getApiRequestOptions, - getApiUrl, - getRawRequestOptions, - getRawUrl, - GithubUrlReader, -} from './GithubUrlReader'; +import { GithubUrlReader } from './GithubUrlReader'; import { ReadTreeResponseFactory } from './tree'; const treeResponseFactory = ReadTreeResponseFactory.create({ @@ -35,143 +28,6 @@ const treeResponseFactory = ReadTreeResponseFactory.create({ }); describe('GithubUrlReader', () => { - describe('getApiRequestOptions', () => { - it('sets the correct API version', () => { - const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; - expect((getApiRequestOptions(config).headers as any).Accept).toEqual( - 'application/vnd.github.v3.raw', - ); - }); - - it('inserts a token when needed', () => { - const withToken: GitHubIntegrationConfig = { - host: '', - apiBaseUrl: '', - token: 'A', - }; - const withoutToken: GitHubIntegrationConfig = { - host: '', - apiBaseUrl: '', - }; - expect( - (getApiRequestOptions(withToken).headers as any).Authorization, - ).toEqual('token A'); - expect( - (getApiRequestOptions(withoutToken).headers as any).Authorization, - ).toBeUndefined(); - }); - }); - - describe('getRawRequestOptions', () => { - it('inserts a token when needed', () => { - const withToken: GitHubIntegrationConfig = { - host: '', - rawBaseUrl: '', - token: 'A', - }; - const withoutToken: GitHubIntegrationConfig = { - host: '', - rawBaseUrl: '', - }; - expect( - (getRawRequestOptions(withToken).headers as any).Authorization, - ).toEqual('token A'); - expect( - (getRawRequestOptions(withoutToken).headers as any).Authorization, - ).toBeUndefined(); - }); - }); - - describe('getApiUrl', () => { - it('rejects targets that do not look like URLs', () => { - const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; - expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); - }); - - it('happy path for github', () => { - const config: GitHubIntegrationConfig = { - host: 'github.com', - apiBaseUrl: 'https://api.github.com', - }; - expect( - getApiUrl( - 'https://github.com/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', - ), - ); - expect( - getApiUrl( - 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', - ), - ); - }); - - it('happy path for ghe', () => { - const config: GitHubIntegrationConfig = { - host: 'ghe.mycompany.net', - apiBaseUrl: 'https://ghe.mycompany.net/api/v3', - }; - expect( - getApiUrl( - 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname', - ), - ); - }); - }); - - describe('getRawUrl', () => { - it('rejects targets that do not look like URLs', () => { - const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; - expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); - }); - - it('happy path for github', () => { - const config: GitHubIntegrationConfig = { - host: 'github.com', - rawBaseUrl: 'https://raw.githubusercontent.com', - }; - expect( - getRawUrl( - 'https://github.com/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml', - ), - ); - }); - - it('happy path for ghe', () => { - const config: GitHubIntegrationConfig = { - host: 'ghe.mycompany.net', - rawBaseUrl: 'https://ghe.mycompany.net/raw', - }; - expect( - getRawUrl( - 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', - config, - ), - ).toEqual( - new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'), - ); - }); - }); - describe('implementation', () => { it('rejects unknown targets', async () => { const processor = new GithubUrlReader( diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index de798a7067..5ca2a99692 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -17,6 +17,8 @@ import { GitHubIntegrationConfig, readGitHubIntegrationConfigs, + getGitHubFileFetchUrl, + getGitHubRequestOptions, } from '@backstage/integration'; import fetch from 'cross-fetch'; import parseGitUri from 'git-url-parse'; @@ -30,92 +32,6 @@ import { UrlReader, } from './types'; -export function getApiRequestOptions( - provider: GitHubIntegrationConfig, -): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -export function getRawRequestOptions( - provider: GitHubIntegrationConfig, -): RequestInit { - const headers: HeadersInit = {}; - - if (provider.token) { - headers.Authorization = `token ${provider.token}`; - } - - return { - headers, - }; -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/path/to/c.yaml -// to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname -export function getApiUrl( - target: string, - provider: GitHubIntegrationConfig, -): URL { - try { - const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); - - if ( - !owner || - !name || - !ref || - (filepathtype !== 'blob' && filepathtype !== 'raw') - ) { - throw new Error('Invalid GitHub URL or file path'); - } - - const pathWithoutSlash = filepath.replace(/^\//, ''); - return new URL( - `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`, - ); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - -// Converts for example -// from: https://github.com/a/b/blob/branchname/c.yaml -// to: https://raw.githubusercontent.com/a/b/branchname/c.yaml -export function getRawUrl( - target: string, - provider: GitHubIntegrationConfig, -): URL { - try { - const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); - - if ( - !owner || - !name || - !ref || - (filepathtype !== 'blob' && filepathtype !== 'raw') - ) { - throw new Error('Invalid GitHub URL or file path'); - } - - const pathWithoutSlash = filepath.replace(/^\//, ''); - return new URL( - `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`, - ); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - /** * A processor that adds the ability to read files from GitHub v3 APIs, such as * the one exposed by GitHub itself. @@ -144,14 +60,8 @@ export class GithubUrlReader implements UrlReader { } async read(url: string): Promise { - const useApi = - this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl); - const ghUrl = useApi - ? getApiUrl(url, this.config) - : getRawUrl(url, this.config); - const options = useApi - ? getApiRequestOptions(this.config) - : getRawRequestOptions(this.config); + const ghUrl = getGitHubFileFetchUrl(url, this.config); + const options = getGitHubRequestOptions(this.config); let response: Response; try { @@ -196,7 +106,7 @@ export class GithubUrlReader implements UrlReader { new URL( `${protocol}://${resource}/${full_name}/archive/${ref}.tar.gz`, ).toString(), - getRawRequestOptions(this.config), + getGitHubRequestOptions(this.config), ); if (!response.ok) { const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index e2d3edfea2..d6d1da5cbb 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -15,6 +15,8 @@ */ import { + getGitLabFileFetchUrl, + getGitLabRequestOptions, GitLabIntegrationConfig, readGitLabIntegrationConfigs, } from '@backstage/integration'; @@ -37,20 +39,11 @@ export class GitlabUrlReader implements UrlReader { constructor(private readonly options: GitLabIntegrationConfig) {} async read(url: string): Promise { - // TODO(Rugvip): merged the old GitlabReaderProcessor in here and used - // the existence of /~/blob/ to switch the logic. Don't know if this - // makes sense and it might require some more work. - let builtUrl: URL; - if (url.includes('/-/blob/')) { - const projectID = await this.getProjectID(url); - builtUrl = this.buildProjectUrl(url, projectID); - } else { - builtUrl = this.buildRawUrl(url); - } + const builtUrl = await getGitLabFileFetchUrl(url, this.options); let response: Response; try { - response = await fetch(builtUrl.toString(), this.getRequestOptions()); + response = await fetch(builtUrl, getGitLabRequestOptions(this.options)); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -70,109 +63,6 @@ export class GitlabUrlReader implements UrlReader { throw new Error('GitlabUrlReader does not implement readTree'); } - // Converts - // from: https://gitlab.example.com/a/b/blob/master/c.yaml - // to: https://gitlab.example.com/a/b/raw/master/c.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitLab URL'); - } - - // Replace 'blob' with 'raw' - url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( - '/', - ); - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } - - // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch - private buildProjectUrl(target: string, projectID: Number): URL { - try { - const url = new URL(target); - - const branchAndFilePath = url.pathname.split('/-/blob/')[1]; - - const [branch, ...filePath] = branchAndFilePath.split('/'); - - url.pathname = [ - '/api/v4/projects', - projectID, - 'repository/files', - encodeURIComponent(filePath.join('/')), - 'raw', - ].join('/'); - url.search = `?ref=${branch}`; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } - - private async getProjectID(target: string): Promise { - const url = new URL(target); - - if ( - // absPaths to gitlab files should contain /-/blob - // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - !url.pathname.match(/\/\-\/blob\//) - ) { - throw new Error('Please provide full path to yaml file from Gitlab'); - } - try { - const repo = url.pathname.split('/-/blob/')[0]; - - // Find ProjectID from url - // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' - // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' - const repoIDLookup = new URL( - `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( - repo.replace(/^\//, ''), - )}`, - ); - const response = await fetch( - repoIDLookup.toString(), - this.getRequestOptions(), - ); - const projectIDJson = await response.json(); - const projectID: Number = projectIDJson.id; - - return projectID; - } catch (e) { - throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`); - } - } - - private getRequestOptions(): RequestInit { - return { - headers: { - ['PRIVATE-TOKEN']: this.options.token ?? '', - }, - }; - } - toString() { const { host, token } = this.options; return `gitlab{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/integration/package.json b/packages/integration/package.json index 342476c38f..8af3cb60b8 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -21,11 +21,13 @@ }, "dependencies": { "@backstage/config": "^0.1.1", + "cross-fetch": "^3.0.6", "git-url-parse": "^11.4.0" }, "devDependencies": { "@backstage/cli": "^0.4.0", - "@types/jest": "^26.0.7" + "@types/jest": "^26.0.7", + "msw": "^0.21.2" }, "files": [ "dist", diff --git a/packages/integration/src/azure/core.test.ts b/packages/integration/src/azure/core.test.ts new file mode 100644 index 0000000000..17041a9eb2 --- /dev/null +++ b/packages/integration/src/azure/core.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + getAzureFileFetchUrl, + getAzureDownloadUrl, + getAzureRequestOptions, +} from './core'; + +describe('azure core', () => { + describe('getAzureRequestOptions', () => { + it('fills in the token if necessary', () => { + expect(getAzureRequestOptions({ host: '', token: '0123456789' })).toEqual( + expect.objectContaining({ + headers: expect.objectContaining({ + Authorization: 'Basic OjAxMjM0NTY3ODk=', + }), + }), + ); + expect(getAzureRequestOptions({ host: '' })).toEqual( + expect.objectContaining({ + headers: expect.not.objectContaining({ + Authorization: expect.anything(), + }), + }), + ); + }); + }); + + describe('getAzureFileFetchUrl', () => { + it.each([ + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + result: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + }, + { + url: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + result: + 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + }, + ])('should handle happy path %#', async ({ url, result }) => { + expect(getAzureFileFetchUrl(url)).toBe(result); + }); + + it.each([ + { + url: 'https://api.com/a/b/blob/master/path/to/c.yaml', + error: + 'Incorrect URL: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + url: 'com/a/b/blob/master/path/to/c.yaml', + error: + 'Incorrect URL: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ])('should handle error path %#', ({ url, error }) => { + expect(() => getAzureFileFetchUrl(url)).toThrow(error); + }); + }); + + describe('getAzureDownloadUrl', () => { + it('do not add scopePath if no path is specified', async () => { + const result = getAzureDownloadUrl( + 'https://dev.azure.com/organization/project/_git/repository', + ); + + expect(new URL(result).searchParams.get('scopePath')).toBeNull(); + }); + + it('add scopePath if a path is specified', async () => { + const result = getAzureDownloadUrl( + 'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs', + ); + expect(new URL(result).searchParams.get('scopePath')).toEqual('docs'); + }); + }); +}); diff --git a/packages/integration/src/azure/core.ts b/packages/integration/src/azure/core.ts new file mode 100644 index 0000000000..44591f2509 --- /dev/null +++ b/packages/integration/src/azure/core.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import parseGitUrl from 'git-url-parse'; +import { AzureIntegrationConfig } from './config'; + +/** + * Given a URL pointing to a file on a provider, returns a URL that is suitable + * for fetching the contents of the data. + * + * Converts + * from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + * to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch} + * + * @param url A URL pointing to a file + */ +export function getAzureFileFetchUrl(url: string): string { + try { + const parsedUrl = new URL(url); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = parsedUrl.pathname.split('/'); + + const path = parsedUrl.searchParams.get('path') || ''; + const ref = parsedUrl.searchParams.get('version')?.substr(2); + + if ( + parsedUrl.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + parsedUrl.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'git', + 'repositories', + repoName, + 'items', + ].join('/'); + + const queryParams = [`path=${path}`]; + + if (ref) { + queryParams.push(`version=${ref}`); + } + + parsedUrl.search = queryParams.join('&'); + + parsedUrl.protocol = 'https'; + + return parsedUrl.toString(); + } catch (e) { + throw new Error(`Incorrect URL: ${url}, ${e}`); + } +} + +/** + * Given a URL pointing to a path on a provider, returns a URL that is suitable + * for downloading the subtree. + * + * @param url A URL pointing to a path + */ +export function getAzureDownloadUrl(url: string): string { + const { + name: repoName, + owner: project, + organization, + protocol, + resource, + filepath, + } = parseGitUrl(url); + + // scopePath will limit the downloaded content + // /docs will only download the docs folder and everything below it + // /docs/index.md will only download index.md but put it in the root of the archive + const scopePath = filepath + ? `&scopePath=${encodeURIComponent(filepath)}` + : ''; + + return `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`; +} + +/** + * Gets the request options necessary to make requests to a given provider. + * + * @param config The relevant provider config + */ +export function getAzureRequestOptions( + config: AzureIntegrationConfig, + additionalHeaders?: Record, +): RequestInit { + const headers: HeadersInit = additionalHeaders + ? { ...additionalHeaders } + : {}; + + if (config.token) { + const buffer = Buffer.from(`:${config.token}`, 'utf8'); + headers.Authorization = `Basic ${buffer.toString('base64')}`; + } + + return { headers }; +} diff --git a/packages/integration/src/azure/index.ts b/packages/integration/src/azure/index.ts index ede0c88a81..365e4cdcdc 100644 --- a/packages/integration/src/azure/index.ts +++ b/packages/integration/src/azure/index.ts @@ -19,3 +19,8 @@ export { readAzureIntegrationConfigs, } from './config'; export type { AzureIntegrationConfig } from './config'; +export { + getAzureDownloadUrl, + getAzureFileFetchUrl, + getAzureRequestOptions, +} from './core'; diff --git a/packages/integration/src/bitbucket/core.test.ts b/packages/integration/src/bitbucket/core.test.ts new file mode 100644 index 0000000000..8d9f956c6b --- /dev/null +++ b/packages/integration/src/bitbucket/core.test.ts @@ -0,0 +1,100 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BitbucketIntegrationConfig } from './config'; +import { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core'; + +describe('bitbucket core', () => { + describe('getBitbucketRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: BitbucketIntegrationConfig = { + host: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: BitbucketIntegrationConfig = { + host: '', + apiBaseUrl: '', + }; + expect( + (getBitbucketRequestOptions(withToken).headers as any).Authorization, + ).toEqual('Bearer A'); + expect( + (getBitbucketRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + + it('insert basic auth when needed', () => { + const withUsernameAndPassword: BitbucketIntegrationConfig = { + host: '', + apiBaseUrl: '', + username: 'some-user', + appPassword: 'my-secret', + }; + const withoutUsernameAndPassword: BitbucketIntegrationConfig = { + host: '', + apiBaseUrl: '', + }; + expect( + (getBitbucketRequestOptions(withUsernameAndPassword).headers as any) + .Authorization, + ).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA=='); + expect( + (getBitbucketRequestOptions(withoutUsernameAndPassword).headers as any) + .Authorization, + ).toBeUndefined(); + }); + }); + + describe('getBitbucketFileFetchUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: BitbucketIntegrationConfig = { host: '', apiBaseUrl: '' }; + expect(() => getBitbucketFileFetchUrl('a/b', config)).toThrow( + /Incorrect URL: a\/b/, + ); + }); + + it('happy path for Bitbucket Cloud', () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }; + expect( + getBitbucketFileFetchUrl( + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config, + ), + ).toEqual( + 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', + ); + }); + + it('happy path for Bitbucket Server', () => { + const config: BitbucketIntegrationConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://bitbucket.mycompany.net/rest/api/1.0', + }; + expect( + getBitbucketFileFetchUrl( + 'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml', + config, + ), + ).toEqual( + 'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml?at=', + ); + }); + }); +}); diff --git a/packages/integration/src/bitbucket/core.ts b/packages/integration/src/bitbucket/core.ts new file mode 100644 index 0000000000..a522e7ce9f --- /dev/null +++ b/packages/integration/src/bitbucket/core.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import parseGitUrl from 'git-url-parse'; +import { BitbucketIntegrationConfig } from './config'; + +/** + * Given a URL pointing to a file on a provider, returns a URL that is suitable + * for fetching the contents of the data. + * + * Converts + * from: https://bitbucket.org/orgname/reponame/src/master/file.yaml + * to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml + * + * @param url A URL pointing to a file + * @param config The relevant provider config + */ +export function getBitbucketFileFetchUrl( + url: string, + config: BitbucketIntegrationConfig, +): string { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUrl(url); + if ( + !owner || + !name || + (filepathtype !== 'browse' && + filepathtype !== 'raw' && + filepathtype !== 'src') + ) { + throw new Error('Invalid Bitbucket URL or file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + + if (config.host === 'bitbucket.org') { + if (!ref) { + throw new Error('Invalid Bitbucket URL or file path'); + } + return `${config.apiBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`; + } + return `${config.apiBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`; + } catch (e) { + throw new Error(`Incorrect URL: ${url}, ${e}`); + } +} + +/** + * Gets the request options necessary to make requests to a given provider. + * + * @param config The relevant provider config + */ +export function getBitbucketRequestOptions( + config: BitbucketIntegrationConfig, +): RequestInit { + const headers: HeadersInit = {}; + + if (config.token) { + headers.Authorization = `Bearer ${config.token}`; + } else if (config.username && config.appPassword) { + const buffer = Buffer.from( + `${config.username}:${config.appPassword}`, + 'utf8', + ); + headers.Authorization = `Basic ${buffer.toString('base64')}`; + } + + return { + headers, + }; +} diff --git a/packages/integration/src/bitbucket/index.ts b/packages/integration/src/bitbucket/index.ts index 897c00d160..b8d37220db 100644 --- a/packages/integration/src/bitbucket/index.ts +++ b/packages/integration/src/bitbucket/index.ts @@ -19,3 +19,4 @@ export { readBitbucketIntegrationConfigs, } from './config'; export type { BitbucketIntegrationConfig } from './config'; +export { getBitbucketFileFetchUrl, getBitbucketRequestOptions } from './core'; diff --git a/packages/integration/src/github/core.test.ts b/packages/integration/src/github/core.test.ts new file mode 100644 index 0000000000..03235acfbb --- /dev/null +++ b/packages/integration/src/github/core.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GitHubIntegrationConfig } from './config'; +import { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; + +describe('github core', () => { + describe('getGitHubRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: GitHubIntegrationConfig = { + host: '', + rawBaseUrl: '', + token: 'A', + }; + const withoutToken: GitHubIntegrationConfig = { + host: '', + rawBaseUrl: '', + }; + expect( + (getGitHubRequestOptions(withToken).headers as any).Authorization, + ).toEqual('token A'); + expect( + (getGitHubRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + }); + + describe('getGitHubFileFetchUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: GitHubIntegrationConfig = { host: '', apiBaseUrl: '' }; + expect(() => getGitHubFileFetchUrl('a/b', config)).toThrow( + /Incorrect URL: a\/b/, + ); + }); + + it('happy path for github api', () => { + const config: GitHubIntegrationConfig = { + host: 'github.com', + apiBaseUrl: 'https://api.github.com', + }; + expect( + getGitHubFileFetchUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ); + expect( + getGitHubFileFetchUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ); + }); + + it('happy path for ghe api', () => { + const config: GitHubIntegrationConfig = { + host: 'ghe.mycompany.net', + apiBaseUrl: 'https://ghe.mycompany.net/api/v3', + }; + expect( + getGitHubFileFetchUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + 'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname', + ); + }); + + it('happy path for github raw', () => { + const config: GitHubIntegrationConfig = { + host: 'github.com', + rawBaseUrl: 'https://raw.githubusercontent.com', + }; + expect( + getGitHubFileFetchUrl( + 'https://github.com/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual( + 'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml', + ); + }); + + it('happy path for ghe raw', () => { + const config: GitHubIntegrationConfig = { + host: 'ghe.mycompany.net', + rawBaseUrl: 'https://ghe.mycompany.net/raw', + }; + expect( + getGitHubFileFetchUrl( + 'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml', + config, + ), + ).toEqual('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'); + }); + }); +}); diff --git a/packages/integration/src/github/core.ts b/packages/integration/src/github/core.ts new file mode 100644 index 0000000000..239692e962 --- /dev/null +++ b/packages/integration/src/github/core.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import parseGitUrl from 'git-url-parse'; +import { GitHubIntegrationConfig } from './config'; + +/** + * Given a URL pointing to a file on a provider, returns a URL that is suitable + * for fetching the contents of the data. + * + * Converts + * from: https://github.com/a/b/blob/branchname/path/to/c.yaml + * to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname + * or: https://raw.githubusercontent.com/a/b/branchname/c.yaml + * + * @param url A URL pointing to a file + * @param config The relevant provider config + */ +export function getGitHubFileFetchUrl( + url: string, + config: GitHubIntegrationConfig, +): string { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUrl(url); + if ( + !owner || + !name || + !ref || + (filepathtype !== 'blob' && filepathtype !== 'raw') + ) { + throw new Error('Invalid GitHub URL or file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + if (chooseEndpoint(config) === 'api') { + return `${config.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`; + } + return `${config.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`; + } catch (e) { + throw new Error(`Incorrect URL: ${url}, ${e}`); + } +} + +/** + * Gets the request options necessary to make requests to a given provider. + * + * @param config The relevant provider config + */ +export function getGitHubRequestOptions( + config: GitHubIntegrationConfig, +): RequestInit { + const headers: HeadersInit = {}; + + if (chooseEndpoint(config) === 'api') { + headers.Accept = 'application/vnd.github.v3.raw'; + } + if (config.token) { + headers.Authorization = `token ${config.token}`; + } + + return { headers }; +} + +export function chooseEndpoint(config: GitHubIntegrationConfig): 'api' | 'raw' { + if (config.apiBaseUrl && (config.token || !config.rawBaseUrl)) { + return 'api'; + } + return 'raw'; +} diff --git a/packages/integration/src/github/index.ts b/packages/integration/src/github/index.ts index 2099dd42e3..5f97f6980a 100644 --- a/packages/integration/src/github/index.ts +++ b/packages/integration/src/github/index.ts @@ -19,3 +19,4 @@ export { readGitHubIntegrationConfigs, } from './config'; export type { GitHubIntegrationConfig } from './config'; +export { getGitHubFileFetchUrl, getGitHubRequestOptions } from './core'; diff --git a/packages/integration/src/gitlab/core.test.ts b/packages/integration/src/gitlab/core.test.ts new file mode 100644 index 0000000000..43fea72e0b --- /dev/null +++ b/packages/integration/src/gitlab/core.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { GitLabIntegrationConfig } from './config'; +import { getGitLabFileFetchUrl } from './core'; + +const worker = setupServer(); + +describe('gitlab core', () => { + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); + afterAll(() => worker.close()); + afterEach(() => worker.resetHandlers()); + + beforeEach(() => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + ); + }); + + const configWithToken: GitLabIntegrationConfig = { + host: 'g.com', + token: '0123456789', + }; + + const configWithNoToken: GitLabIntegrationConfig = { + host: 'g.com', + }; + + describe('getGitLabFileFetchUrl', () => { + it.each([ + // Project URLs + { + config: configWithNoToken, + url: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + result: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }, + { + config: configWithToken, + url: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + result: + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }, + { + config: configWithNoToken, + url: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + result: + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }, + // Raw URLs + { + config: configWithNoToken, + url: 'https://gitlab.example.com/a/b/blob/master/c.yaml', + result: 'https://gitlab.example.com/a/b/raw/master/c.yaml', + }, + ])('should handle happy path %#', async ({ config, url, result }) => { + await expect(getGitLabFileFetchUrl(url, config)).resolves.toBe(result); + }); + }); +}); diff --git a/packages/integration/src/gitlab/core.ts b/packages/integration/src/gitlab/core.ts new file mode 100644 index 0000000000..63dc3fdb3b --- /dev/null +++ b/packages/integration/src/gitlab/core.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GitLabIntegrationConfig } from './config'; +import fetch from 'cross-fetch'; + +/** + * Given a URL pointing to a file on a provider, returns a URL that is suitable + * for fetching the contents of the data. + * + * Converts + * from: https://gitlab.example.com/a/b/blob/master/c.yaml + * to: https://gitlab.example.com/a/b/raw/master/c.yaml + * -or- + * from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + * to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch + * + * @param url A URL pointing to a file + * @param config The relevant provider config + */ +export async function getGitLabFileFetchUrl( + url: string, + config: GitLabIntegrationConfig, +): Promise { + // TODO(Rugvip): From the old GitlabReaderProcessor; used + // the existence of /-/blob/ to switch the logic. Don't know if this + // makes sense and it might require some more work. + if (url.includes('/-/blob/')) { + const projectID = await getProjectId(url, config); + return buildProjectUrl(url, projectID).toString(); + } + return buildRawUrl(url).toString(); +} + +/** + * Gets the request options necessary to make requests to a given provider. + * + * @param config The relevant provider config + */ +export function getGitLabRequestOptions( + config: GitLabIntegrationConfig, +): RequestInit { + const { token = '' } = config; + return { + headers: { + 'PRIVATE-TOKEN': token, + }, + }; +} + +// Converts +// from: https://gitlab.example.com/a/b/blob/master/c.yaml +// to: https://gitlab.example.com/a/b/raw/master/c.yaml +export function buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitLab URL'); + } + + // Replace 'blob' with 'raw' + url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join('/'); + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } +} + +// Converts +// from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath +// to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch +export function buildProjectUrl(target: string, projectID: Number): URL { + try { + const url = new URL(target); + + const branchAndFilePath = url.pathname.split('/-/blob/')[1]; + const [branch, ...filePath] = branchAndFilePath.split('/'); + + url.pathname = [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(filePath.join('/')), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } +} + +// Convert +// from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath +// to: The project ID that corresponds to the URL +export async function getProjectId( + target: string, + config: GitLabIntegrationConfig, +): Promise { + const url = new URL(target); + + if (!url.pathname.includes('/-/blob/')) { + throw new Error('Please provide full path to yaml file from Gitlab'); + } + + try { + const repo = url.pathname.split('/-/blob/')[0]; + + // Convert + // to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo + const repoIDLookup = new URL( + `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( + repo.replace(/^\//, ''), + )}`, + ); + const response = await fetch( + repoIDLookup.toString(), + getGitLabRequestOptions(config), + ); + const projectIDJson = await response.json(); + const projectID = Number(projectIDJson.id); + + return projectID; + } catch (e) { + throw new Error(`Could not get GitLab project ID for: ${target}, ${e}`); + } +} diff --git a/packages/integration/src/gitlab/index.ts b/packages/integration/src/gitlab/index.ts index 0801914fd4..8dc4e90764 100644 --- a/packages/integration/src/gitlab/index.ts +++ b/packages/integration/src/gitlab/index.ts @@ -19,3 +19,4 @@ export { readGitLabIntegrationConfigs, } from './config'; export type { GitLabIntegrationConfig } from './config'; +export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core';