From 419559d0e7c5f7c71d9737a368a7b548ecc263a7 Mon Sep 17 00:00:00 2001 From: ahmed Date: Mon, 4 Jul 2022 11:08:19 +0200 Subject: [PATCH 1/8] consider hosted gitlab with relative path Signed-off-by: ahmed --- .../src/reading/GitlabUrlReader.ts | 14 +++++-- .../src/gitlab/GitLabIntegration.test.ts | 1 + .../integration/src/gitlab/config.test.ts | 5 +++ packages/integration/src/gitlab/config.ts | 16 +++++++- packages/integration/src/gitlab/core.test.ts | 16 ++++++++ packages/integration/src/gitlab/core.ts | 39 +++++++++++++------ 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index 2387dadf5b..f681adb4d4 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -37,7 +37,7 @@ import { ReadUrlResponse, ReadUrlOptions, } from './types'; -import { trimEnd } from 'lodash'; +import { trimEnd, trimStart } from 'lodash'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; /** @@ -117,13 +117,19 @@ export class GitlabUrlReader implements UrlReader { const { etag, signal } = options ?? {}; const { ref, full_name, filepath } = parseGitUrl(url); + // Considering self hosted gitlab with relative + const repo_full_name = trimStart( + full_name, + this.integration.config.relativePath ?? '', + ); + // Use GitLab API to get the default branch // encodeURIComponent is required for GitLab API // https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding const projectGitlabResponse = await fetch( new URL( `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( - full_name, + repo_full_name, )}`, ).toString(), getGitLabRequestOptions(this.integration.config), @@ -150,7 +156,7 @@ export class GitlabUrlReader implements UrlReader { const commitsGitlabResponse = await fetch( new URL( `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( - full_name, + repo_full_name, )}/repository/commits?${commitsReqParams.toString()}`, ).toString(), { @@ -181,7 +187,7 @@ export class GitlabUrlReader implements UrlReader { // https://docs.gitlab.com/ee/api/repositories.html#get-file-archive const archiveGitLabResponse = await fetch( `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( - full_name, + repo_full_name, )}/repository/archive?sha=${branch}`, { ...getGitLabRequestOptions(this.integration.config), diff --git a/packages/integration/src/gitlab/GitLabIntegration.test.ts b/packages/integration/src/gitlab/GitLabIntegration.test.ts index 575ce75308..ca84110438 100644 --- a/packages/integration/src/gitlab/GitLabIntegration.test.ts +++ b/packages/integration/src/gitlab/GitLabIntegration.test.ts @@ -28,6 +28,7 @@ describe('GitLabIntegration', () => { token: 't', apiBaseUrl: 'https://h.com/api/v4', baseUrl: 'https://h.com', + relativePath: '', }, ], }, diff --git a/packages/integration/src/gitlab/config.test.ts b/packages/integration/src/gitlab/config.test.ts index db03285206..b642b3c7ae 100644 --- a/packages/integration/src/gitlab/config.test.ts +++ b/packages/integration/src/gitlab/config.test.ts @@ -66,6 +66,7 @@ describe('readGitLabIntegrationConfig', () => { token: 't', apiBaseUrl: 'https://a.com', baseUrl: 'https://baseurl.for.me/gitlab', + relativePath: '/gitlab', }); }); @@ -77,6 +78,7 @@ describe('readGitLabIntegrationConfig', () => { host: 'gitlab.com', apiBaseUrl: 'https://gitlab.com/api/v4', baseUrl: 'https://gitlab.com', + relativePath: '', }); }); @@ -89,6 +91,7 @@ describe('readGitLabIntegrationConfig', () => { host: 'gitlab.com', baseUrl: 'https://gitlab.com', apiBaseUrl: 'https://gitlab.com/api/v4', + relativePath: '', }); }); @@ -119,6 +122,7 @@ describe('readGitLabIntegrationConfig', () => { host: 'a.com', apiBaseUrl: 'https://a.com/api', baseUrl: 'https://a.com', + relativePath: '/', }); }); }); @@ -144,6 +148,7 @@ describe('readGitLabIntegrationConfigs', () => { token: 't', apiBaseUrl: 'https://a.com/api/v4', baseUrl: 'https://a.com', + relativePath: '/', }); }); diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index 71b4cd868f..89f6b6a822 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -54,6 +54,14 @@ export type GitLabIntegrationConfig = { * If no baseUrl is provided, it will default to `https://${host}` */ baseUrl: string; + + /** + * The relative path for gitlab self hosted installations + * described here https://docs.gitlab.com/ee/install/relative_url.html + * + * If no baseUrl is provided, it will default to `/` + */ + relativePath?: string; }; /** @@ -69,7 +77,7 @@ export function readGitLabIntegrationConfig( let apiBaseUrl = config.getOptionalString('apiBaseUrl'); const token = config.getOptionalString('token'); let baseUrl = config.getOptionalString('baseUrl'); - + let relativePath = ''; if (apiBaseUrl) { apiBaseUrl = trimEnd(apiBaseUrl, '/'); } else if (host === GITLAB_HOST) { @@ -96,7 +104,11 @@ export function readGitLabIntegrationConfig( ); } - return { host, token, apiBaseUrl, baseUrl }; + if (host !== GITLAB_HOST) { + relativePath = new URL(baseUrl).pathname; + } + + return { host, token, apiBaseUrl, baseUrl, relativePath }; } /** diff --git a/packages/integration/src/gitlab/core.test.ts b/packages/integration/src/gitlab/core.test.ts index d01406fe1c..e923c0ee0c 100644 --- a/packages/integration/src/gitlab/core.test.ts +++ b/packages/integration/src/gitlab/core.test.ts @@ -39,12 +39,22 @@ describe('gitlab core', () => { token: '0123456789', apiBaseUrl: '', baseUrl: '', + relativePath: '', }; const configWithNoToken: GitLabIntegrationConfig = { host: 'g.com', apiBaseUrl: '', baseUrl: '', + relativePath: '', + }; + + const configWithSelfHosted: GitLabIntegrationConfig = { + host: 'g.com', + token: '0123456789', + apiBaseUrl: '', + baseUrl: '', + relativePath: '/gitlab', }; describe('getGitLabFileFetchUrl with .yaml extension', () => { @@ -121,6 +131,12 @@ describe('gitlab core', () => { result: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', }, + { + config: configWithSelfHosted, + url: 'https://gitlab.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yml', + result: + 'https://gitlab.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', + }, { config: configWithNoToken, // Works with non URI encoded link diff --git a/packages/integration/src/gitlab/core.ts b/packages/integration/src/gitlab/core.ts index ba0f9c83de..64b350eb67 100644 --- a/packages/integration/src/gitlab/core.ts +++ b/packages/integration/src/gitlab/core.ts @@ -17,6 +17,9 @@ import { GitLabIntegrationConfig } from './config'; import fetch from 'cross-fetch'; import { InputError } from '@backstage/errors'; +import { relative } from 'path'; +import { trimStart } from 'lodash'; +// import { config } from 'process'; /** * Given a URL pointing to a file on a provider, returns a URL that is suitable @@ -44,7 +47,7 @@ export async function getGitLabFileFetchUrl( // makes sense and it might require some more work. if (url.includes('/-/blob/')) { const projectID = await getProjectId(url, config); - return buildProjectUrl(url, projectID).toString(); + return buildProjectUrl(url, projectID, config).toString(); } return buildRawUrl(url).toString(); } @@ -101,20 +104,26 @@ export function buildRawUrl(target: string): URL { // Converts // from: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath // to: https://gitlab.com/api/v4/projects/projectId/repository/files/filepath?ref=branch -export function buildProjectUrl(target: string, projectID: Number): URL { +export function buildProjectUrl( + target: string, + projectID: Number, + config: GitLabIntegrationConfig, +): URL { try { const url = new URL(target); const branchAndFilePath = url.pathname.split('/-/blob/')[1]; const [branch, ...filePath] = branchAndFilePath.split('/'); - url.pathname = [ - '/api/v4/projects', - projectID, - 'repository/files', - encodeURIComponent(decodeURIComponent(filePath.join('/'))), - 'raw', - ].join('/'); + url.pathname = + [config.relativePath] + + [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(decodeURIComponent(filePath.join('/'))), + 'raw', + ].join('/'); url.search = `?ref=${branch}`; return url; @@ -137,19 +146,27 @@ export async function getProjectId( } try { - const repo = url.pathname.split('/-/blob/')[0]; + let repo = url.pathname.split('/-/blob/')[0]; + + // Ignore relative path if it's not set + const relativePath = config.relativePath ?? ''; + + // Should replace first match only + repo = repo.replace(relativePath, ''); // Convert // to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo const repoIDLookup = new URL( - `${url.origin}/api/v4/projects/${encodeURIComponent( + `${url.origin}${relativePath}/api/v4/projects/${encodeURIComponent( repo.replace(/^\//, ''), )}`, ); + const response = await fetch( repoIDLookup.toString(), getGitLabRequestOptions(config), ); + const data = await response.json(); if (!response.ok) { From 954a94f52f6ad568aa73cf015677a098a776dae6 Mon Sep 17 00:00:00 2001 From: ahmed Date: Mon, 4 Jul 2022 11:20:40 +0200 Subject: [PATCH 2/8] add changeset Signed-off-by: ahmed --- .changeset/tall-pants-clap.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/tall-pants-clap.md diff --git a/.changeset/tall-pants-clap.md b/.changeset/tall-pants-clap.md new file mode 100644 index 0000000000..1472e0a208 --- /dev/null +++ b/.changeset/tall-pants-clap.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Support self-hosted gitlab installations with relative url From 6e77ce6605b5105d93ae898d3dc2bba309eedb2c Mon Sep 17 00:00:00 2001 From: ahmed Date: Mon, 4 Jul 2022 17:23:17 +0200 Subject: [PATCH 3/8] remove unused imports Signed-off-by: ahmed --- packages/integration/src/gitlab/core.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/integration/src/gitlab/core.ts b/packages/integration/src/gitlab/core.ts index 64b350eb67..5432bd90ee 100644 --- a/packages/integration/src/gitlab/core.ts +++ b/packages/integration/src/gitlab/core.ts @@ -17,9 +17,6 @@ import { GitLabIntegrationConfig } from './config'; import fetch from 'cross-fetch'; import { InputError } from '@backstage/errors'; -import { relative } from 'path'; -import { trimStart } from 'lodash'; -// import { config } from 'process'; /** * Given a URL pointing to a file on a provider, returns a URL that is suitable From c7aa786be4c8409f8579ca56deacc902b62e58fe Mon Sep 17 00:00:00 2001 From: ahmed Date: Tue, 5 Jul 2022 15:05:56 +0200 Subject: [PATCH 4/8] update api-report.md Signed-off-by: ahmed --- packages/integration/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 3ffb131900..78d170a0e0 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -443,6 +443,7 @@ export type GitLabIntegrationConfig = { apiBaseUrl: string; token?: string; baseUrl: string; + relativePath?: string; }; // @public From b0da37a6364f63c0c21ea0150d83d946e85c3535 Mon Sep 17 00:00:00 2001 From: ahmed Date: Tue, 5 Jul 2022 15:08:34 +0200 Subject: [PATCH 5/8] changeset typo Signed-off-by: ahmed --- .changeset/tall-pants-clap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/tall-pants-clap.md b/.changeset/tall-pants-clap.md index 1472e0a208..5a1714c106 100644 --- a/.changeset/tall-pants-clap.md +++ b/.changeset/tall-pants-clap.md @@ -3,4 +3,4 @@ '@backstage/integration': patch --- -Support self-hosted gitlab installations with relative url +Support self-hosted gitlab installations with relative URL. From f0f2e4f6b0bfbc699cee1d023c55647491e81218 Mon Sep 17 00:00:00 2001 From: ahmed Date: Wed, 6 Jul 2022 21:55:48 +0200 Subject: [PATCH 6/8] move relative path to a function Signed-off-by: ahmed --- .../src/reading/GitlabUrlReader.ts | 8 +++-- packages/integration/api-report.md | 6 +++- .../src/gitlab/GitLabIntegration.test.ts | 1 - .../integration/src/gitlab/config.test.ts | 5 --- packages/integration/src/gitlab/config.ts | 32 +++++++++-------- packages/integration/src/gitlab/core.test.ts | 35 ++++++++++++++----- packages/integration/src/gitlab/core.ts | 13 ++++--- packages/integration/src/gitlab/index.ts | 1 + 8 files changed, 64 insertions(+), 37 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index f681adb4d4..c4cb0fb661 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -16,6 +16,7 @@ import { getGitLabFileFetchUrl, + getGitLabIntegrationRelativePath, getGitLabRequestOptions, GitLabIntegration, ScmIntegrations, @@ -118,11 +119,12 @@ export class GitlabUrlReader implements UrlReader { const { ref, full_name, filepath } = parseGitUrl(url); // Considering self hosted gitlab with relative - const repo_full_name = trimStart( - full_name, - this.integration.config.relativePath ?? '', + const relativePath = getGitLabIntegrationRelativePath( + this.integration.config, ); + const repo_full_name = trimStart(full_name, relativePath); + // Use GitLab API to get the default branch // encodeURIComponent is required for GitLab API // https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 78d170a0e0..8d8d017095 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -342,6 +342,11 @@ export function getGitLabFileFetchUrl( config: GitLabIntegrationConfig, ): Promise; +// @public +export function getGitLabIntegrationRelativePath( + config: GitLabIntegrationConfig, +): string; + // @public export function getGitLabRequestOptions(config: GitLabIntegrationConfig): { headers: Record; @@ -443,7 +448,6 @@ export type GitLabIntegrationConfig = { apiBaseUrl: string; token?: string; baseUrl: string; - relativePath?: string; }; // @public diff --git a/packages/integration/src/gitlab/GitLabIntegration.test.ts b/packages/integration/src/gitlab/GitLabIntegration.test.ts index ca84110438..575ce75308 100644 --- a/packages/integration/src/gitlab/GitLabIntegration.test.ts +++ b/packages/integration/src/gitlab/GitLabIntegration.test.ts @@ -28,7 +28,6 @@ describe('GitLabIntegration', () => { token: 't', apiBaseUrl: 'https://h.com/api/v4', baseUrl: 'https://h.com', - relativePath: '', }, ], }, diff --git a/packages/integration/src/gitlab/config.test.ts b/packages/integration/src/gitlab/config.test.ts index b642b3c7ae..db03285206 100644 --- a/packages/integration/src/gitlab/config.test.ts +++ b/packages/integration/src/gitlab/config.test.ts @@ -66,7 +66,6 @@ describe('readGitLabIntegrationConfig', () => { token: 't', apiBaseUrl: 'https://a.com', baseUrl: 'https://baseurl.for.me/gitlab', - relativePath: '/gitlab', }); }); @@ -78,7 +77,6 @@ describe('readGitLabIntegrationConfig', () => { host: 'gitlab.com', apiBaseUrl: 'https://gitlab.com/api/v4', baseUrl: 'https://gitlab.com', - relativePath: '', }); }); @@ -91,7 +89,6 @@ describe('readGitLabIntegrationConfig', () => { host: 'gitlab.com', baseUrl: 'https://gitlab.com', apiBaseUrl: 'https://gitlab.com/api/v4', - relativePath: '', }); }); @@ -122,7 +119,6 @@ describe('readGitLabIntegrationConfig', () => { host: 'a.com', apiBaseUrl: 'https://a.com/api', baseUrl: 'https://a.com', - relativePath: '/', }); }); }); @@ -148,7 +144,6 @@ describe('readGitLabIntegrationConfigs', () => { token: 't', apiBaseUrl: 'https://a.com/api/v4', baseUrl: 'https://a.com', - relativePath: '/', }); }); diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index 89f6b6a822..0e96a5e5e9 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -54,14 +54,6 @@ export type GitLabIntegrationConfig = { * If no baseUrl is provided, it will default to `https://${host}` */ baseUrl: string; - - /** - * The relative path for gitlab self hosted installations - * described here https://docs.gitlab.com/ee/install/relative_url.html - * - * If no baseUrl is provided, it will default to `/` - */ - relativePath?: string; }; /** @@ -77,7 +69,6 @@ export function readGitLabIntegrationConfig( let apiBaseUrl = config.getOptionalString('apiBaseUrl'); const token = config.getOptionalString('token'); let baseUrl = config.getOptionalString('baseUrl'); - let relativePath = ''; if (apiBaseUrl) { apiBaseUrl = trimEnd(apiBaseUrl, '/'); } else if (host === GITLAB_HOST) { @@ -104,11 +95,7 @@ export function readGitLabIntegrationConfig( ); } - if (host !== GITLAB_HOST) { - relativePath = new URL(baseUrl).pathname; - } - - return { host, token, apiBaseUrl, baseUrl, relativePath }; + return { host, token, apiBaseUrl, baseUrl }; } /** @@ -136,3 +123,20 @@ export function readGitLabIntegrationConfigs( return result; } + +/** + * Reads a GitLab integration config, and returns + * relative path. + * + * @param config - GitLabIntegrationConfig object + * @public + */ +export function getGitLabIntegrationRelativePath( + config: GitLabIntegrationConfig, +): string { + let relativePath = ''; + if (config.host !== GITLAB_HOST) { + relativePath = new URL(config.baseUrl).pathname; + } + return relativePath; +} diff --git a/packages/integration/src/gitlab/core.test.ts b/packages/integration/src/gitlab/core.test.ts index e923c0ee0c..d0ba87c653 100644 --- a/packages/integration/src/gitlab/core.test.ts +++ b/packages/integration/src/gitlab/core.test.ts @@ -35,26 +35,23 @@ describe('gitlab core', () => { }); const configWithToken: GitLabIntegrationConfig = { - host: 'g.com', + host: 'gitlab.com', token: '0123456789', apiBaseUrl: '', baseUrl: '', - relativePath: '', }; const configWithNoToken: GitLabIntegrationConfig = { - host: 'g.com', + host: 'gitlab.com', apiBaseUrl: '', baseUrl: '', - relativePath: '', }; const configWithSelfHosted: GitLabIntegrationConfig = { - host: 'g.com', + host: 'gitlab.mycompany.com', token: '0123456789', apiBaseUrl: '', - baseUrl: '', - relativePath: '/gitlab', + baseUrl: 'https://gitlab.mycompany.com/gitlab', }; describe('getGitLabFileFetchUrl with .yaml extension', () => { @@ -66,6 +63,12 @@ describe('gitlab core', () => { result: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', }, + { + config: configWithSelfHosted, + url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + result: + 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }, { config: configWithNoToken, // Works with non URI encoded link @@ -73,6 +76,13 @@ describe('gitlab core', () => { result: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yaml/raw?ref=branch', }, + { + config: configWithSelfHosted, + url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yaml', + result: + 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yaml/raw?ref=branch', + }, + { config: configWithNoToken, url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yaml', @@ -133,9 +143,9 @@ describe('gitlab core', () => { }, { config: configWithSelfHosted, - url: 'https://gitlab.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yml', + url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yml', result: - 'https://gitlab.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', + 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', }, { config: configWithNoToken, @@ -144,6 +154,13 @@ describe('gitlab core', () => { result: 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yml/raw?ref=branch', }, + { + config: configWithSelfHosted, + // Works with non URI encoded link + url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yml', + result: + 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yml/raw?ref=branch', + }, { config: configWithNoToken, url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yml', diff --git a/packages/integration/src/gitlab/core.ts b/packages/integration/src/gitlab/core.ts index 5432bd90ee..5f51a514bf 100644 --- a/packages/integration/src/gitlab/core.ts +++ b/packages/integration/src/gitlab/core.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { GitLabIntegrationConfig } from './config'; +import { + getGitLabIntegrationRelativePath, + GitLabIntegrationConfig, +} from './config'; import fetch from 'cross-fetch'; import { InputError } from '@backstage/errors'; @@ -42,6 +45,7 @@ export async function getGitLabFileFetchUrl( // TODO(Rugvip): From the old GitlabReaderProcessor; used // the existence of /-/blob/ to switch the logic. Don't know if this // makes sense and it might require some more work. + if (url.includes('/-/blob/')) { const projectID = await getProjectId(url, config); return buildProjectUrl(url, projectID, config).toString(); @@ -111,9 +115,10 @@ export function buildProjectUrl( const branchAndFilePath = url.pathname.split('/-/blob/')[1]; const [branch, ...filePath] = branchAndFilePath.split('/'); + const relativePath = getGitLabIntegrationRelativePath(config); url.pathname = - [config.relativePath] + + [relativePath] + [ '/api/v4/projects', projectID, @@ -145,8 +150,8 @@ export async function getProjectId( try { let repo = url.pathname.split('/-/blob/')[0]; - // Ignore relative path if it's not set - const relativePath = config.relativePath ?? ''; + // Get gitlab relative path + const relativePath = getGitLabIntegrationRelativePath(config); // Should replace first match only repo = repo.replace(relativePath, ''); diff --git a/packages/integration/src/gitlab/index.ts b/packages/integration/src/gitlab/index.ts index e8d6665a6f..1a5063c51c 100644 --- a/packages/integration/src/gitlab/index.ts +++ b/packages/integration/src/gitlab/index.ts @@ -17,6 +17,7 @@ export { readGitLabIntegrationConfig, readGitLabIntegrationConfigs, + getGitLabIntegrationRelativePath, } from './config'; export type { GitLabIntegrationConfig } from './config'; export { getGitLabFileFetchUrl, getGitLabRequestOptions } from './core'; From e823d78d90fc87a94a7496b80c6623a5a43a89a2 Mon Sep 17 00:00:00 2001 From: ahmed Date: Thu, 7 Jul 2022 01:32:17 +0200 Subject: [PATCH 7/8] fix a bug for self hosted without relative path Signed-off-by: ahmed --- packages/integration/src/gitlab/config.ts | 2 +- packages/integration/src/gitlab/core.test.ts | 43 +++++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/integration/src/gitlab/config.ts b/packages/integration/src/gitlab/config.ts index 0e96a5e5e9..301fe46f90 100644 --- a/packages/integration/src/gitlab/config.ts +++ b/packages/integration/src/gitlab/config.ts @@ -138,5 +138,5 @@ export function getGitLabIntegrationRelativePath( if (config.host !== GITLAB_HOST) { relativePath = new URL(config.baseUrl).pathname; } - return relativePath; + return trimEnd(relativePath, '/'); } diff --git a/packages/integration/src/gitlab/core.test.ts b/packages/integration/src/gitlab/core.test.ts index d0ba87c653..d043aba55a 100644 --- a/packages/integration/src/gitlab/core.test.ts +++ b/packages/integration/src/gitlab/core.test.ts @@ -47,13 +47,20 @@ describe('gitlab core', () => { baseUrl: '', }; - const configWithSelfHosted: GitLabIntegrationConfig = { + const configSelfHosteWithRelativePath: GitLabIntegrationConfig = { host: 'gitlab.mycompany.com', token: '0123456789', apiBaseUrl: '', baseUrl: 'https://gitlab.mycompany.com/gitlab', }; + const configSelfHostedWithoutRelativePath: GitLabIntegrationConfig = { + host: 'gitlab.mycompany.com', + token: '0123456789', + apiBaseUrl: '', + baseUrl: 'https://gitlab.mycompany.com', + }; + describe('getGitLabFileFetchUrl with .yaml extension', () => { it.each([ // Project URLs @@ -64,11 +71,17 @@ describe('gitlab core', () => { 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', }, { - config: configWithSelfHosted, + config: configSelfHosteWithRelativePath, url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', result: 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', }, + { + config: configSelfHostedWithoutRelativePath, + url: 'https://gitlab.mycompany.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + result: + 'https://gitlab.mycompany.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + }, { config: configWithNoToken, // Works with non URI encoded link @@ -77,12 +90,17 @@ describe('gitlab core', () => { 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yaml/raw?ref=branch', }, { - config: configWithSelfHosted, + config: configSelfHosteWithRelativePath, url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yaml', result: 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yaml/raw?ref=branch', }, - + { + config: configSelfHostedWithoutRelativePath, + url: 'https://gitlab.mycompany.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yaml', + result: + 'https://gitlab.mycompany.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yaml/raw?ref=branch', + }, { config: configWithNoToken, url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yaml', @@ -142,11 +160,17 @@ describe('gitlab core', () => { 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', }, { - config: configWithSelfHosted, + config: configSelfHosteWithRelativePath, url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yml', result: 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', }, + { + config: configSelfHostedWithoutRelativePath, + url: 'https://gitlab.mycompany.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yml', + result: + 'https://gitlab.mycompany.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yml/raw?ref=branch', + }, { config: configWithNoToken, // Works with non URI encoded link @@ -155,12 +179,19 @@ describe('gitlab core', () => { 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yml/raw?ref=branch', }, { - config: configWithSelfHosted, + config: configSelfHosteWithRelativePath, // Works with non URI encoded link url: 'https://gitlab.mycompany.com/gitlab/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yml', result: 'https://gitlab.mycompany.com/gitlab/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yml/raw?ref=branch', }, + { + config: configSelfHostedWithoutRelativePath, + // Works with non URI encoded link + url: 'https://gitlab.mycompany.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file with spaces.yml', + result: + 'https://gitlab.mycompany.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile%20with%20spaces.yml/raw?ref=branch', + }, { config: configWithNoToken, url: 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path%20with%20spaces/to/file.yml', From 6f9f3b6936c336159904b1eb667a872444c79bd2 Mon Sep 17 00:00:00 2001 From: ahmed Date: Fri, 8 Jul 2022 17:30:17 +0200 Subject: [PATCH 8/8] rename variable and add relativePath check Signed-off-by: ahmed --- .../src/reading/GitlabUrlReader.ts | 14 +++++++---- packages/integration/src/gitlab/core.ts | 24 ++++++++++--------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index c4cb0fb661..5bc234daef 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -118,12 +118,16 @@ export class GitlabUrlReader implements UrlReader { const { etag, signal } = options ?? {}; const { ref, full_name, filepath } = parseGitUrl(url); - // Considering self hosted gitlab with relative + let repoFullName = full_name; + const relativePath = getGitLabIntegrationRelativePath( this.integration.config, ); - const repo_full_name = trimStart(full_name, relativePath); + // Considering self hosted gitlab with relative + if (relativePath) { + repoFullName = trimStart(full_name, relativePath); + } // Use GitLab API to get the default branch // encodeURIComponent is required for GitLab API @@ -131,7 +135,7 @@ export class GitlabUrlReader implements UrlReader { const projectGitlabResponse = await fetch( new URL( `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( - repo_full_name, + repoFullName, )}`, ).toString(), getGitLabRequestOptions(this.integration.config), @@ -158,7 +162,7 @@ export class GitlabUrlReader implements UrlReader { const commitsGitlabResponse = await fetch( new URL( `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( - repo_full_name, + repoFullName, )}/repository/commits?${commitsReqParams.toString()}`, ).toString(), { @@ -189,7 +193,7 @@ export class GitlabUrlReader implements UrlReader { // https://docs.gitlab.com/ee/api/repositories.html#get-file-archive const archiveGitLabResponse = await fetch( `${this.integration.config.apiBaseUrl}/projects/${encodeURIComponent( - repo_full_name, + repoFullName, )}/repository/archive?sha=${branch}`, { ...getGitLabRequestOptions(this.integration.config), diff --git a/packages/integration/src/gitlab/core.ts b/packages/integration/src/gitlab/core.ts index 5f51a514bf..1fcf15a78f 100644 --- a/packages/integration/src/gitlab/core.ts +++ b/packages/integration/src/gitlab/core.ts @@ -117,15 +117,15 @@ export function buildProjectUrl( const [branch, ...filePath] = branchAndFilePath.split('/'); const relativePath = getGitLabIntegrationRelativePath(config); - url.pathname = - [relativePath] + - [ - '/api/v4/projects', - projectID, - 'repository/files', - encodeURIComponent(decodeURIComponent(filePath.join('/'))), - 'raw', - ].join('/'); + url.pathname = [ + ...(relativePath ? [relativePath] : []), + 'api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(decodeURIComponent(filePath.join('/'))), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; return url; @@ -153,8 +153,10 @@ export async function getProjectId( // Get gitlab relative path const relativePath = getGitLabIntegrationRelativePath(config); - // Should replace first match only - repo = repo.replace(relativePath, ''); + // Check relative path exist and replace it if it's the case. + if (relativePath) { + repo = repo.replace(relativePath, ''); + } // Convert // to: https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo