From 2ea30b685225b24dbb1128f879c95a8ee0ed6521 Mon Sep 17 00:00:00 2001 From: Max Morton Date: Fri, 11 Nov 2022 10:27:08 -0800 Subject: [PATCH] Update GitlabUrlReader to support API and Artifact URLs Signed-off-by: Max Morton --- .../src/reading/GitlabUrlReader.test.ts | 82 +++++++++++++++++++ .../src/reading/GitlabUrlReader.ts | 67 ++++++++++++++- 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 29ef3f4ce8..d88c6d8fb9 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -573,4 +573,86 @@ describe('GitlabUrlReader', () => { ).rejects.toThrow(NotModifiedError); }); }); + + describe('getGitlabFetchUrl', () => { + beforeEach(() => { + worker.use( + rest.get( + '*/api/v4/projects/group%2Fsubgroup%2Fproject', + (_, res, ctx) => res(ctx.status(200), ctx.json({ id: 12345 })), + ), + ); + }); + it('should fall back to getGitLabFileFetchUrl for blob urls', async () => { + await expect( + gitlabProcessor.getGitlabFetchUrl( + 'https://gitlab.com/group/subgroup/project/-/blob/branch/my/path/to/file.yaml', + ), + ).resolves.toEqual( + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch', + ); + }); + it('should work for job artifact urls', async () => { + await expect( + gitlabProcessor.getGitlabFetchUrl( + 'https://gitlab.com/group/subgroup/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob', + ), + ).resolves.toEqual( + 'https://gitlab.com/api/v4/projects/12345/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob', + ); + }); + it('should pass API urls naively', async () => { + const apiUrl = 'https://gitlab.com/api/v4/my/api/path'; + await expect(gitlabProcessor.getGitlabFetchUrl(apiUrl)).resolves.toEqual( + apiUrl, + ); + }); + it('should fail on unfamiliar or non-Gitlab urls', async () => { + await expect( + gitlabProcessor.getGitlabFetchUrl( + 'https://gitlab.com/some/random/endpoint', + ), + ).rejects.toThrow('Please provide full path to yaml file from GitLab'); + }); + }); + + describe('getGitlabArtfiactFetchUrl', () => { + beforeEach(() => { + worker.use( + rest.get( + '*/api/v4/projects/group%2Fsubgroup%2Fproject', + (_, res, ctx) => res(ctx.status(200), ctx.json({ id: 12345 })), + ), + ); + worker.use( + rest.get( + '*/api/v4/projects/groupA%2Fsubgroup%2Fproject', + (_, res, ctx) => res(ctx.status(404)), + ), + ); + }); + it('should reject urls that are not for the job artifacts API', async () => { + await expect( + gitlabProcessor.getGitlabArtifactFetchUrl( + 'https://gitlab.com/some/url', + ), + ).rejects.toThrow('Unable to process url as an GitLab artifact'); + }); + it('should work for job artifact urls', async () => { + await expect( + gitlabProcessor.getGitlabFetchUrl( + 'https://gitlab.com/group/subgroup/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob', + ), + ).resolves.toEqual( + 'https://gitlab.com/api/v4/projects/12345/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob', + ); + }); + it('errors in mapping the project ID should be captured', async () => { + await expect( + gitlabProcessor.getGitlabFetchUrl( + 'https://gitlab.com/groupA/subgroup/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob', + ), + ).rejects.toThrow(/^Unable to translate GitLab artifact URL:/); + }); + }); }); diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index eaa15a7208..f704901650 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -73,7 +73,7 @@ export class GitlabUrlReader implements UrlReader { options?: ReadUrlOptions, ): Promise { const { etag, signal } = options ?? {}; - const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config); + const builtUrl = await this.getGitlabFetchUrl(url); let response: Response; try { @@ -256,4 +256,69 @@ export class GitlabUrlReader implements UrlReader { const { host, token } = this.integration.config; return `gitlab{host=${host},authed=${Boolean(token)}}`; } + + async getGitlabFetchUrl(target: string): Promise { + // If the target is a raw API url then trust that no parsing is needed + if (target.includes('/api/v4/')) { + return target; + } + // If the target is for a job artifact then go down that path + if (target.includes('/-/jobs/artifacts/')) { + return this.getGitlabArtifactFetchUrl(target).then(value => + value.toString(), + ); + } + // Default to the old behavior of assuming the url is for a file + return getGitLabFileFetchUrl(target, this.integration.config); + } + + // convert urls of the form: + // https://example.com///-/jobs/artifacts//raw/?job= + // to urls of the form: + // https://example.com/api/v4/projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job= + async getGitlabArtifactFetchUrl(target: string): Promise { + const url = new URL(target); + if (!url.pathname.includes('/-/jobs/artifacts/')) { + throw new Error('Unable to process url as an GitLab artifact'); + } + try { + const [namespaceAndProject, ref] = + url.pathname.split('/-/jobs/artifacts/'); + const projectPath = new URL(url); + projectPath.pathname = namespaceAndProject; + const projectId = await this.resolveProjectToId(projectPath); + const relativePath = getGitLabIntegrationRelativePath( + this.integration.config, + ); + url.pathname = `${relativePath}/api/v4/projects/${projectId}/jobs/artifacts/${ref}`; + return url; + } catch (e) { + throw new Error( + `Unable to translate GitLab artifact URL: ${target}, ${e}`, + ); + } + } + + private async resolveProjectToId(pathToProject: URL): Promise { + let project = pathToProject.pathname; + // Check relative path exist and remove it if so + const relativePath = getGitLabIntegrationRelativePath( + this.integration.config, + ); + if (relativePath) { + project = project.replace(relativePath, ''); + } + // Trim an initial / if it exists + project = project.replace(/^\//, ''); + const result = await fetch( + `${ + pathToProject.origin + }${relativePath}/api/v4/projects/${encodeURIComponent(project)}`, + ); + const data = await result.json(); + if (!result.ok) { + throw new Error(`Gitlab error: ${data.error}, ${data.error_description}`); + } + return Number(data.id); + } }