Merge pull request #29707 from CptnFizzbin/integrations/gitlab-url-reader-token

(Integrations - GitLab) Bugfix: Pass user provided token through to gitlab url resolvers

.changeset/large-lemons-clap.md

@@ -0,0 +1,7 @@
+---
+'@backstage/backend-plugin-api': patch
+'@backstage/backend-defaults': patch
+'@backstage/integration': patch
+---
+
+update documentation

.changeset/twenty-kiwis-punch.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-defaults': patch
+---
+
+Bug fix: Pass user provided token through to gitlab url resolvers

packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts

@@ -26,7 +26,7 @@ import { setupServer } from 'msw/node';
 import path from 'path';
 import { GitlabUrlReader } from './GitlabUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
-import { NotModifiedError, NotFoundError } from '@backstage/errors';
+import { NotFoundError, NotModifiedError } from '@backstage/errors';
 import {
   GitLabIntegration,
   readGitLabIntegrationConfig,
@@ -248,6 +248,29 @@ describe('GitlabUrlReader', () => {
       const content = await result.buffer();
       expect(content.toString()).toBe('foo');
     });
+
+    it('should return the file when using a user token', async () => {
+      worker.use(
+        rest.get('*/api/v4/projects/user%2Fproject', (req, res, ctx) => {
+          if (req.headers.get('private-token') !== 'gl-user-token') {
+            return res(
+              ctx.status(403),
+              ctx.json({ message: 'Not Authorized' }),
+            );
+          }
+          return res(ctx.status(200), ctx.json({ id: 12345 }));
+        }),
+        rest.get('*', (_req, res, ctx) => {
+          return res(ctx.status(200), ctx.body('foo'));
+        }),
+      );
+      const result = await reader.readUrl(
+        'https://gitlab.com/user/project/-/blob/branch/my/path/to/file.yaml',
+        { token: 'gl-user-token' },
+      );
+      const content = await result.buffer();
+      expect(content.toString()).toBe('foo');
+    });
   });
 
   describe('readTree', () => {
@@ -276,102 +299,103 @@ describe('GitlabUrlReader', () => {
           id: 'sha456def',
         },
       ];
-    });
 
-    beforeEach(() => {
-      worker.use(
-        rest.get(
-          'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/archive',
-          (_, res, ctx) =>
-            res(
-              ctx.status(200),
-              ctx.set('Content-Type', 'application/zip'),
-              ctx.set(
-                'content-disposition',
-                'attachment; filename="mock-main-sha123abc.zip"',
+      const projectNames = ['backstage%2Fmock', 'user%2Fproject'];
+      projectNames.forEach(projectName => {
+        worker.use(
+          rest.get(
+            `https://gitlab.com/api/v4/projects/${projectName}/repository/archive`,
+            (_, res, ctx) =>
+              res(
+                ctx.status(200),
+                ctx.set('Content-Type', 'application/zip'),
+                ctx.set(
+                  'content-disposition',
+                  'attachment; filename="mock-main-sha123abc.zip"',
+                ),
+                ctx.body(archiveBuffer),
               ),
-              ctx.body(archiveBuffer),
-            ),
-        ),
-        rest.get(
-          'https://gitlab.com/api/v4/projects/backstage%2Fmock',
-          (_, res, ctx) =>
-            res(
-              ctx.status(200),
-              ctx.set('Content-Type', 'application/json'),
-              ctx.json(projectGitlabApiResponse),
-            ),
-        ),
-        rest.get(
-          'https://gitlab.com/api/v4/projects/backstage%2Fmock/repository/commits',
-          (req, res, ctx) => {
-            const refName = req.url.searchParams.get('ref_name');
-            if (refName === 'main') {
-              const filepath = req.url.searchParams.get('path');
-              if (filepath === 'testFilepath') {
+          ),
+          rest.get(
+            `https://gitlab.com/api/v4/projects/${projectName}`,
+            (_, res, ctx) =>
+              res(
+                ctx.status(200),
+                ctx.set('Content-Type', 'application/json'),
+                ctx.json(projectGitlabApiResponse),
+              ),
+          ),
+          rest.get(
+            `https://gitlab.com/api/v4/projects/${projectName}/repository/commits`,
+            (req, res, ctx) => {
+              const refName = req.url.searchParams.get('ref_name');
+              if (refName === 'main') {
+                const filepath = req.url.searchParams.get('path');
+                if (filepath === 'testFilepath') {
+                  return res(
+                    ctx.status(200),
+                    ctx.set('Content-Type', 'application/json'),
+                    ctx.json(specificPathCommitsGitlabApiResponse),
+                  );
+                }
                 return res(
                   ctx.status(200),
                   ctx.set('Content-Type', 'application/json'),
-                  ctx.json(specificPathCommitsGitlabApiResponse),
+                  ctx.json(commitsGitlabApiResponse),
                 );
               }
-              return res(
+              if (refName === 'branchDoesNotExist') {
+                return res(ctx.status(404));
+              }
+              return res();
+            },
+          ),
+          rest.get(
+            `https://gitlab.mycompany.com/api/v4/projects/${projectName}`,
+            (_, res, ctx) =>
+              res(
                 ctx.status(200),
                 ctx.set('Content-Type', 'application/json'),
-                ctx.json(commitsGitlabApiResponse),
-              );
-            }
-            if (refName === 'branchDoesNotExist') {
-              return res(ctx.status(404));
-            }
-            return res();
-          },
-        ),
-        rest.get(
-          'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock',
-          (_, res, ctx) =>
-            res(
-              ctx.status(200),
-              ctx.set('Content-Type', 'application/json'),
-              ctx.json(projectGitlabApiResponse),
-            ),
-        ),
-        rest.get(
-          'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/commits',
-          (req, res, ctx) => {
-            const refName = req.url.searchParams.get('ref_name');
-            if (refName === 'main') {
-              const filepath = req.url.searchParams.get('path');
-              if (filepath === 'testFilepath') {
+                ctx.json(projectGitlabApiResponse),
+              ),
+          ),
+          rest.get(
+            `https://gitlab.mycompany.com/api/v4/projects/${projectName}/repository/commits`,
+            (req, res, ctx) => {
+              const refName = req.url.searchParams.get('ref_name');
+              if (refName === 'main') {
+                const filepath = req.url.searchParams.get('path');
+                if (filepath === 'testFilepath') {
+                  return res(
+                    ctx.status(200),
+                    ctx.set('Content-Type', 'application/json'),
+                    ctx.json(specificPathCommitsGitlabApiResponse),
+                  );
+                }
                 return res(
                   ctx.status(200),
                   ctx.set('Content-Type', 'application/json'),
-                  ctx.json(specificPathCommitsGitlabApiResponse),
+                  ctx.json(commitsGitlabApiResponse),
                 );
               }
-              return res(
+              return res();
+            },
+          ),
+          rest.get(
+            `https://gitlab.mycompany.com/api/v4/projects/${projectName}/repository/archive`,
+            (_, res, ctx) =>
+              res(
                 ctx.status(200),
-                ctx.set('Content-Type', 'application/json'),
-                ctx.json(commitsGitlabApiResponse),
-              );
-            }
-            return res();
-          },
-        ),
-        rest.get(
-          'https://gitlab.mycompany.com/api/v4/projects/backstage%2Fmock/repository/archive',
-          (_, res, ctx) =>
-            res(
-              ctx.status(200),
-              ctx.set('Content-Type', 'application/zip'),
-              ctx.set(
-                'content-disposition',
-                'attachment; filename="mock-main-sha123abc.zip"',
+                ctx.set('Content-Type', 'application/zip'),
+                ctx.set(
+                  'content-disposition',
+                  'attachment; filename="mock-main-sha123abc.zip"',
+                ),
+                ctx.body(archiveBuffer),
               ),
-              ctx.body(archiveBuffer),
-            ),
-        ),
-      );
+          ),
+        );
+      });
     });
 
     it('returns the wanted files from an archive', async () => {
@@ -543,6 +567,28 @@ describe('GitlabUrlReader', () => {
       expect(mkDocsFile.toString()).toBe('site_name: Test\n');
       expect(indexMarkdownFile.toString()).toBe('# Test\n');
     });
+
+    it('should return the file when using a user token', async () => {
+      worker.use(
+        rest.get('*/api/v4/projects/user%2Fproject', (req, res, ctx) => {
+          if (req.headers.get('private-token') !== 'gl-user-token') {
+            return res(
+              ctx.status(403),
+              ctx.json({ message: 'Not Authorized' }),
+            );
+          }
+          return res(ctx.status(200), ctx.json({ id: 12345 }));
+        }),
+      );
+
+      const response = await gitlabProcessor.readTree(
+        'https://gitlab.com/user/project/tree/main',
+        { token: 'gl-user-token' },
+      );
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+    });
   });
 
   describe('search', () => {
@@ -690,6 +736,15 @@ describe('GitlabUrlReader', () => {
           '*/api/v4/projects/group%2Fsubgroup%2Fproject',
           (_, res, ctx) => res(ctx.status(200), ctx.json({ id: 12345 })),
         ),
+        rest.get('*/api/v4/projects/user%2Fproject', (req, res, ctx) => {
+          if (req.headers.get('private-token') !== 'gl-user-token') {
+            return res(
+              ctx.status(403),
+              ctx.json({ message: 'Not Authorized' }),
+            );
+          }
+          return res(ctx.status(200), ctx.json({ id: 12345 }));
+        }),
       );
     });
     it('should fall back to getGitLabFileFetchUrl for blob urls', async () => {
@@ -719,6 +774,16 @@ describe('GitlabUrlReader', () => {
         'Failed converting /some/random/endpoint to a project id. Url path must include /blob/.',
       );
     });
+    it('should resolve the project path using a user token', async () => {
+      await expect(
+        (gitlabProcessor as any).getGitlabFetchUrl(
+          'https://gitlab.com/user/project/-/blob/branch/my/path/to/file.yaml',
+          'gl-user-token',
+        ),
+      ).resolves.toEqual(
+        'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+      );
+    });
   });
 
   describe('getGitlabArtifactFetchUrl', () => {
@@ -728,12 +793,19 @@ describe('GitlabUrlReader', () => {
           '*/api/v4/projects/group%2Fsubgroup%2Fproject',
           (_, res, ctx) => res(ctx.status(200), ctx.json({ id: 12345 })),
         ),
-      );
-      worker.use(
         rest.get(
           '*/api/v4/projects/groupA%2Fsubgroup%2Fproject',
           (_, res, ctx) => res(ctx.status(404)),
         ),
+        rest.get('*/api/v4/projects/user%2Fproject', (req, res, ctx) => {
+          if (req.headers.get('private-token') !== 'gl-user-token') {
+            return res(
+              ctx.status(403),
+              ctx.json({ message: 'Not Authorized' }),
+            );
+          }
+          return res(ctx.status(200), ctx.json({ id: 12345 }));
+        }),
       );
     });
     it('should reject urls that are not for the job artifacts API', async () => {
@@ -765,20 +837,61 @@ describe('GitlabUrlReader', () => {
         ),
       ).rejects.toThrow(/^Unable to translate GitLab artifact URL:/);
     });
+    it('should resolve the project path using a user token', async () => {
+      await expect(
+        (gitlabProcessor as any).getGitlabArtifactFetchUrl(
+          new URL(
+            'https://gitlab.com/user/project/-/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
+          ),
+          'gl-user-token',
+        ),
+      ).resolves.toEqual(
+        new URL(
+          'https://gitlab.com/api/v4/projects/12345/jobs/artifacts/branch/raw/my/path/to/file.yaml?job=myJob',
+        ),
+      );
+    });
   });
 
   describe('resolveProjectToId', () => {
-    it('should resolve the project path to a valid project id', async () => {
+    beforeEach(() => {
       worker.use(
-        rest.get('*/api/v4/projects/some%2Fproject', (req, res, ctx) => {
+        rest.get('*/api/v4/projects/group%2Fproject', (req, res, ctx) => {
           // the private-token header must be included on API calls
-          expect(req.headers.get('private-token')).toBe('gl-dummy-token');
+          if (req.headers.get('private-token') !== 'gl-dummy-token') {
+            return res(
+              ctx.status(403),
+              ctx.json({ message: 'Not Authorized' }),
+            );
+          }
+          return res(ctx.status(200), ctx.json({ id: 12345 }));
+        }),
+        rest.get('*/api/v4/projects/user%2Fproject', (req, res, ctx) => {
+          // the private-token header must be included on API calls
+          if (req.headers.get('private-token') !== 'gl-user-token') {
+            return res(
+              ctx.status(403),
+              ctx.json({ message: 'Not Authorized' }),
+            );
+          }
           return res(ctx.status(200), ctx.json({ id: 12345 }));
         }),
       );
+    });
+
+    it('should resolve the project path to a valid project id', async () => {
       await expect(
         (gitlabProcessor as any).resolveProjectToId(
-          new URL('https://gitlab.com/some/project'),
+          new URL('https://gitlab.com/group/project'),
+        ),
+      ).resolves.toEqual(12345);
+    });
+
+    it('should resolve the project path to a valid project id using a user token', async () => {
+      await expect(
+        (gitlabProcessor as any).resolveProjectToId(
+          new URL('https://gitlab.com/user/project'),
+          'gl-user-token',
         ),
       ).resolves.toEqual(12345);
     });

packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts

@@ -32,18 +32,18 @@ import {
   NotModifiedError,
 } from '@backstage/errors';
 import {
-  GitLabIntegration,
-  ScmIntegrations,
   getGitLabFileFetchUrl,
   getGitLabIntegrationRelativePath,
   getGitLabRequestOptions,
+  GitLabIntegration,
+  ScmIntegrations,
 } from '@backstage/integration';
 import parseGitUrl from 'git-url-parse';
 import { trimEnd, trimStart } from 'lodash';
 import { Minimatch } from 'minimatch';
 import { Readable } from 'stream';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
-import { ReadTreeResponseFactory, ReaderFactory } from './types';
+import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import { parseLastModified } from './util';
 
 /**
@@ -79,7 +79,7 @@ export class GitlabUrlReader implements UrlReaderService {
   ): Promise<UrlReaderServiceReadUrlResponse> {
     const { etag, lastModifiedAfter, signal, token } = options ?? {};
     const isArtifact = url.includes('/-/jobs/artifacts/');
-    const builtUrl = await this.getGitlabFetchUrl(url);
+    const builtUrl = await this.getGitlabFetchUrl(url, token);
 
     let response: Response;
     try {
@@ -328,23 +328,32 @@ export class GitlabUrlReader implements UrlReaderService {
     return `gitlab{host=${host},authed=${Boolean(token)}}`;
   }
 
-  private async getGitlabFetchUrl(target: string): Promise<string> {
+  private async getGitlabFetchUrl(
+    target: string,
+    token?: string,
+  ): Promise<string> {
     // If the target is for a job artifact then go down that path
     const targetUrl = new URL(target);
     if (targetUrl.pathname.includes('/-/jobs/artifacts/')) {
-      return this.getGitlabArtifactFetchUrl(targetUrl).then(value =>
+      return this.getGitlabArtifactFetchUrl(targetUrl, token).then(value =>
         value.toString(),
       );
     }
     // Default to the old behavior of assuming the url is for a file
-    return getGitLabFileFetchUrl(target, this.integration.config);
+    return getGitLabFileFetchUrl(target, {
+      ...this.integration.config,
+      ...(token && { token }),
+    });
   }
 
   // convert urls of the form:
   //    https://example.com/<namespace>/<project>/-/jobs/artifacts/<ref>/raw/<path_to_file>?job=<job_name>
   // to urls of the form:
   //    https://example.com/api/v4/projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=<job_name>
-  private async getGitlabArtifactFetchUrl(target: URL): Promise<URL> {
+  private async getGitlabArtifactFetchUrl(
+    target: URL,
+    token?: string,
+  ): Promise<URL> {
     if (!target.pathname.includes('/-/jobs/artifacts/')) {
       throw new Error('Unable to process url as an GitLab artifact');
     }
@@ -353,7 +362,7 @@ export class GitlabUrlReader implements UrlReaderService {
         target.pathname.split('/-/jobs/artifacts/');
       const projectPath = new URL(target);
       projectPath.pathname = namespaceAndProject;
-      const projectId = await this.resolveProjectToId(projectPath);
+      const projectId = await this.resolveProjectToId(projectPath, token);
       const relativePath = getGitLabIntegrationRelativePath(
         this.integration.config,
       );
@@ -367,7 +376,10 @@ export class GitlabUrlReader implements UrlReaderService {
     }
   }
 
-  private async resolveProjectToId(pathToProject: URL): Promise<number> {
+  private async resolveProjectToId(
+    pathToProject: URL,
+    token?: string,
+  ): Promise<number> {
     let project = pathToProject.pathname;
     // Check relative path exist and remove it if so
     const relativePath = getGitLabIntegrationRelativePath(
@@ -382,7 +394,7 @@ export class GitlabUrlReader implements UrlReaderService {
       `${
         pathToProject.origin
       }${relativePath}/api/v4/projects/${encodeURIComponent(project)}`,
-      getGitLabRequestOptions(this.integration.config),
+      getGitLabRequestOptions(this.integration.config, token),
     );
     const data = await result.json();
     if (!result.ok) {

packages/backend-plugin-api/src/services/definitions/UrlReaderService.ts

@@ -209,7 +209,7 @@ export type UrlReaderServiceReadTreeOptions = {
    * @remarks
    *
    * By default all URL Readers will use the integrations config which is supplied
-   * when creating the Readers. Sometimes it might be desireable to use the already
+   * when creating the Readers. Sometimes it might be desirable to use the already
    * created URLReaders but with a different token, maybe that's supplied by the user
    * at runtime.
    */

packages/integration/src/gitlab/core.ts

@@ -49,6 +49,7 @@ export async function getGitLabFileFetchUrl(
  * Gets the request options necessary to make requests to a given provider.
  *
  * @param config - The relevant provider config
+ * @param token - An optional auth token to use for communicating with GitLab. By default uses the integration token
  * @public
  */
 export function getGitLabRequestOptions(