Merge pull request #22521 from backstage/blam/support-tokens-fetch

Support `token` in `URLReaders` for `fetch:*` actions in Scaffolder

.changeset/quick-ties-stare.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-plugin-api': patch
+'@backstage/backend-common': patch
+---
+
+Support `token` in `readTree`, `readUrl` and `search`

.changeset/ten-trainers-cough.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+'@backstage/plugin-scaffolder-node': minor
+---
+
+Support providing an overriding token for `fetch:template`, `fetch:plain` and `fetch:file` when interacting with upstream integrations

packages/backend-common/src/reading/GithubUrlReader.test.ts

@@ -46,7 +46,7 @@ const treeResponseFactory = DefaultReadTreeResponseFactory.create({
 
 const mockCredentialsProvider = {
   getCredentials: jest.fn().mockResolvedValue({ headers: {} }),
-} as unknown as GithubCredentialsProvider;
+} satisfies GithubCredentialsProvider;
 
 const githubProcessor = new GithubUrlReader(
   new GithubIntegration(
@@ -105,7 +105,7 @@ describe('GithubUrlReader', () => {
         otherheader: 'something',
       };
 
-      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
         headers: mockHeaders,
       });
 
@@ -146,7 +146,7 @@ describe('GithubUrlReader', () => {
         otherheader: 'something',
       };
 
-      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
         headers: mockHeaders,
       });
 
@@ -182,7 +182,7 @@ describe('GithubUrlReader', () => {
         otherheader: 'something',
       };
 
-      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
         headers: mockHeaders,
       });
 
@@ -241,7 +241,7 @@ describe('GithubUrlReader', () => {
     });
 
     it('should return etag and last-modified from the response', async () => {
-      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
         headers: {
           Authorization: 'bearer blah',
         },
@@ -271,6 +271,33 @@ describe('GithubUrlReader', () => {
       expect(response.etag).toBe('foo');
       expect(response.lastModifiedAt).toEqual(new Date('2021-01-01T00:00:00Z'));
     });
+
+    it('should override the token if its provided', async () => {
+      expect.assertions(1);
+
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
+        headers: {
+          Authorization: 'bearer blah',
+        },
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              'Bearer overridentoken',
+            );
+            return res(ctx.status(200));
+          },
+        ),
+      );
+
+      await gheProcessor.readUrl(
+        'https://github.com/backstage/mock/tree/blob/main',
+        { token: 'overridentoken' },
+      );
+    });
   });
 
   /*
@@ -431,7 +458,7 @@ describe('GithubUrlReader', () => {
         otherheader: 'something',
       };
 
-      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
         headers: mockHeaders,
       });
 
@@ -463,6 +490,44 @@ describe('GithubUrlReader', () => {
       );
     });
 
+    it('should override the token when provided', async () => {
+      expect.assertions(1);
+
+      const mockHeaders = {
+        Authorization: 'bearer blah',
+      };
+
+      mockCredentialsProvider.getCredentials.mockResolvedValue({
+        headers: mockHeaders,
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tarball/etag123abc',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              'Bearer overridentoken',
+            );
+
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.set(
+                'content-disposition',
+                'attachment; filename=backstage-mock-etag123.tar.gz',
+              ),
+              ctx.body(repoBuffer),
+            );
+          },
+        ),
+      );
+
+      await gheProcessor.readTree(
+        'https://ghe.github.com/backstage/mock/tree/main',
+        { token: 'overridentoken' },
+      );
+    });
+
     it('includes the subdomain in the github url', async () => {
       const response = await gheProcessor.readTree(
         'https://ghe.github.com/backstage/mock/tree/main/docs',
@@ -905,6 +970,34 @@ describe('GithubUrlReader', () => {
       await runTests(gheProcessor, 'https://ghe.github.com');
     });
 
+    it('passes through a token for the search request', async () => {
+      expect.assertions(1);
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/git/trees/etag123abc',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              'Bearer overridentoken',
+            );
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/json'),
+              ctx.json({
+                truncated: true,
+                tree: [],
+              } as Partial<GhTreeResponse>),
+            );
+          },
+        ),
+      );
+
+      await gheProcessor.search(
+        `https://ghe.github.com/backstage/mock/tree/main/**/*`,
+        { token: 'overridentoken' },
+      );
+    });
+
     // eslint-disable-next-line jest/expect-expect
     it('succeeds on ghe when going via readTree', async () => {
       worker.use(

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -92,13 +92,31 @@ export class GithubUrlReader implements UrlReader {
     return response.buffer();
   }
 
+  private getCredentials = async (
+    url: string,
+    options?: { token?: string },
+  ): Promise<GithubCredentials> => {
+    if (options?.token) {
+      return {
+        headers: {
+          Authorization: `Bearer ${options.token}`,
+        },
+        type: 'token',
+        token: options.token,
+      };
+    }
+
+    return await this.deps.credentialsProvider.getCredentials({
+      url,
+    });
+  };
+
   async readUrl(
     url: string,
     options?: ReadUrlOptions,
   ): Promise<ReadUrlResponse> {
-    const credentials = await this.deps.credentialsProvider.getCredentials({
-      url,
-    });
+    const credentials = await this.getCredentials(url, options);
+
     const ghUrl = getGithubFileFetchUrl(
       url,
       this.integration.config,
@@ -141,9 +159,7 @@ export class GithubUrlReader implements UrlReader {
     }
 
     const { filepath } = parseGitUrl(url);
-    const { headers } = await this.deps.credentialsProvider.getCredentials({
-      url,
-    });
+    const { headers } = await this.getCredentials(url, options);
 
     return this.doReadTree(
       repoDetails.repo.archive_url,
@@ -169,9 +185,7 @@ export class GithubUrlReader implements UrlReader {
     }
 
     const { filepath } = parseGitUrl(url);
-    const { headers } = await this.deps.credentialsProvider.getCredentials({
-      url,
-    });
+    const { headers } = await this.getCredentials(url, options);
 
     const files = await this.doSearch(
       url,

packages/backend-plugin-api/api-report.md

@@ -316,6 +316,7 @@ export type ReadTreeOptions = {
   ): boolean;
   etag?: string;
   signal?: AbortSignal;
+  token?: string;
 };
 
 // @public
@@ -343,6 +344,7 @@ export type ReadUrlOptions = {
   etag?: string;
   lastModifiedAfter?: Date;
   signal?: AbortSignal;
+  token?: string;
 };
 
 // @public
@@ -390,6 +392,7 @@ export interface SchedulerService extends PluginTaskScheduler {}
 export type SearchOptions = {
   etag?: string;
   signal?: AbortSignal;
+  token?: string;
 };
 
 // @public

packages/backend-plugin-api/src/services/definitions/UrlReaderService.ts

@@ -92,6 +92,18 @@ export type ReadUrlOptions = {
    * Not all reader implementations may take this field into account.
    */
   signal?: AbortSignal;
+
+  /**
+   * An optional token to use for authentication when reading the resources.
+   *
+   * @remarks
+   *
+   * By default all URL Readers will use the integrations config which is supplied
+   * when creating the Readers. Sometimes it might be desireable to use the already
+   * created URLReaders but with a different token, maybe that's supplied by the user
+   * at runtime.
+   */
+  token?: string;
 };
 
 /**
@@ -179,6 +191,18 @@ export type ReadTreeOptions = {
    * Not all reader implementations may take this field into account.
    */
   signal?: AbortSignal;
+
+  /**
+   * An optional token to use for authentication when reading the resources.
+   *
+   * @remarks
+   *
+   * By default all URL Readers will use the integrations config which is supplied
+   * when creating the Readers. Sometimes it might be desireable to use the already
+   * created URLReaders but with a different token, maybe that's supplied by the user
+   * at runtime.
+   */
+  token?: string;
 };
 
 /**
@@ -281,6 +305,18 @@ export type SearchOptions = {
    * Not all reader implementations may take this field into account.
    */
   signal?: AbortSignal;
+
+  /**
+   * An optional token to use for authentication when reading the resources.
+   *
+   * @remarks
+   *
+   * By default all URL Readers will use the integrations config which is supplied
+   * when creating the Readers. Sometimes it might be desireable to use the already
+   * created URLReaders but with a different token, maybe that's supplied by the user
+   * at runtime.
+   */
+  token?: string;
 };
 
 /**

plugins/scaffolder-backend/api-report.md

@@ -145,6 +145,7 @@ export function createFetchPlainAction(options: {
   {
     url: string;
     targetPath?: string | undefined;
+    token?: string | undefined;
   },
   JsonObject
 >;
@@ -157,6 +158,7 @@ export function createFetchPlainFileAction(options: {
   {
     url: string;
     targetPath: string;
+    token?: string | undefined;
   },
   JsonObject
 >;
@@ -179,6 +181,7 @@ export function createFetchTemplateAction(options: {
     replace?: boolean | undefined;
     trimBlocks?: boolean | undefined;
     lstripBlocks?: boolean | undefined;
+    token?: string | undefined;
   },
   JsonObject
 >;

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts

@@ -77,6 +77,26 @@ describe('fetch:plain', () => {
         targetPath: 'lol',
       },
     });
+
+    expect(fetchContents).toHaveBeenCalledWith(
+      expect.objectContaining({
+        outputPath: resolvePath(mockContext.workspacePath, 'lol'),
+        fetchUrl:
+          'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets',
+      }),
+    );
+  });
+
+  it('should fetch plain with token', async () => {
+    await action.handler({
+      ...mockContext,
+      input: {
+        url: 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets',
+        targetPath: 'lol',
+        token: 'mockToken',
+      },
+    });
+
     expect(fetchContents).toHaveBeenCalledWith(
       expect.objectContaining({
         outputPath: resolvePath(mockContext.workspacePath, 'lol'),

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts

@@ -36,7 +36,11 @@ export function createFetchPlainAction(options: {
 }) {
   const { reader, integrations } = options;
 
-  return createTemplateAction<{ url: string; targetPath?: string }>({
+  return createTemplateAction<{
+    url: string;
+    targetPath?: string;
+    token?: string;
+  }>({
     id: ACTION_ID,
     examples,
     description:
@@ -58,6 +62,12 @@ export function createFetchPlainAction(options: {
               'Target path within the working directory to download the contents to.',
             type: 'string',
           },
+          token: {
+            title: 'Token',
+            description:
+              'An optional token to use for authentication when reading the resources.',
+            type: 'string',
+          },
         },
       },
     },
@@ -75,6 +85,7 @@ export function createFetchPlainAction(options: {
         baseUrl: ctx.templateInfo?.baseUrl,
         fetchUrl: ctx.input.url,
         outputPath,
+        token: ctx.input.token,
       });
     },
   });

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.test.ts

@@ -69,6 +69,21 @@ describe('fetch:plain:file', () => {
     );
   });
 
+  it('passed through the token to fetchFile', async () => {
+    await action.handler({
+      ...mockContext,
+      input: {
+        url: 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png',
+        token: 'mockToken',
+        targetPath: 'lol',
+      },
+    });
+
+    expect(fetchFile).toHaveBeenCalledWith(
+      expect.objectContaining({ token: 'mockToken' }),
+    );
+  });
+
   it('should fetch plain', async () => {
     await action.handler({
       ...mockContext,

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.ts

@@ -33,7 +33,11 @@ export function createFetchPlainFileAction(options: {
 }) {
   const { reader, integrations } = options;
 
-  return createTemplateAction<{ url: string; targetPath: string }>({
+  return createTemplateAction<{
+    url: string;
+    targetPath: string;
+    token?: string;
+  }>({
     id: 'fetch:plain:file',
     description: 'Downloads single file and places it in the workspace.',
     examples,
@@ -54,6 +58,12 @@ export function createFetchPlainFileAction(options: {
               'Target path within the working directory to download the file as.',
             type: 'string',
           },
+          token: {
+            title: 'Token',
+            description:
+              'An optional token to use for authentication when reading the resources.',
+            type: 'string',
+          },
         },
       },
     },
@@ -73,6 +83,7 @@ export function createFetchPlainFileAction(options: {
         baseUrl: ctx.templateInfo?.baseUrl,
         fetchUrl: ctx.input.url,
         outputPath,
+        token: ctx.input.token,
       });
     },
   });

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts

@@ -371,6 +371,18 @@ describe('fetch:template', () => {
           fs.readlink(`${workspacePath}/target/brokenSymlink`),
         ).resolves.toEqual(`.${pathSep}not-a-real-file.txt`);
       });
+
+      it('passed through the token to the fetchContents call', async () => {
+        await action.handler(
+          mockContext({
+            token: 'mockToken',
+          }),
+        );
+
+        expect(mockFetchContents).toHaveBeenCalledWith(
+          expect.objectContaining({ token: 'mockToken' }),
+        );
+      });
     });
   });
 

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts

@@ -71,6 +71,7 @@ export function createFetchTemplateAction(options: {
     replace?: boolean;
     trimBlocks?: boolean;
     lstripBlocks?: boolean;
+    token?: string;
   }>({
     id: 'fetch:template',
     description:
@@ -134,6 +135,12 @@ export function createFetchTemplateAction(options: {
               'If set, replace files in targetPath instead of skipping existing ones.',
             type: 'boolean',
           },
+          token: {
+            title: 'Token',
+            description:
+              'An optional token to use for authentication when reading the resources.',
+            type: 'string',
+          },
         },
       },
     },
@@ -197,6 +204,7 @@ export function createFetchTemplateAction(options: {
         baseUrl: ctx.templateInfo?.baseUrl,
         fetchUrl: ctx.input.url,
         outputPath: templateDir,
+        token: ctx.input.token,
       });
 
       ctx.logger.info('Listing files and directories in template');

plugins/scaffolder-node/api-report.md

@@ -195,6 +195,7 @@ export function fetchContents(options: {
   baseUrl?: string;
   fetchUrl?: string;
   outputPath: string;
+  token?: string;
 }): Promise<void>;
 
 // @public
@@ -204,6 +205,7 @@ export function fetchFile(options: {
   baseUrl?: string;
   fetchUrl?: string;
   outputPath: string;
+  token?: string;
 }): Promise<void>;
 
 // @public (undocumented)

plugins/scaffolder-node/src/actions/fetch.test.ts

@@ -124,6 +124,20 @@ describe('fetchContents helper', () => {
       expect(fs.ensureDir).toHaveBeenCalledWith('foo');
       expect(dirFunction).toHaveBeenCalledWith({ targetDir: 'foo' });
     });
+
+    it('should pass through the token provided through to the URL reader', async () => {
+      await fetchContents({
+        ...options,
+        outputPath: 'mydir/foo',
+        fetchUrl: 'https://github.com/backstage/foo',
+        token: 'mockToken',
+      });
+
+      expect(readTree).toHaveBeenCalledWith(
+        'https://github.com/backstage/foo',
+        expect.objectContaining({ token: 'mockToken' }),
+      );
+    });
   });
 
   describe('fetch file', () => {
@@ -211,5 +225,19 @@ describe('fetchContents helper', () => {
       expect(fs.ensureDir).toHaveBeenCalledWith('mydir');
       expect(fs.outputFile).toHaveBeenCalledWith('mydir/foo', 'test');
     });
+
+    it('should pass through the token provided through to the URL reader', async () => {
+      await fetchFile({
+        ...options,
+        outputPath: 'mydir/foo',
+        fetchUrl: 'https://github.com/backstage/foo',
+        token: 'mockToken',
+      });
+
+      expect(readUrl).toHaveBeenCalledWith(
+        'https://github.com/backstage/foo',
+        expect.objectContaining({ token: 'mockToken' }),
+      );
+    });
   });
 });

plugins/scaffolder-node/src/actions/fetch.ts

@@ -32,8 +32,16 @@ export async function fetchContents(options: {
   baseUrl?: string;
   fetchUrl?: string;
   outputPath: string;
+  token?: string;
 }) {
-  const { reader, integrations, baseUrl, fetchUrl = '.', outputPath } = options;
+  const {
+    reader,
+    integrations,
+    baseUrl,
+    fetchUrl = '.',
+    outputPath,
+    token,
+  } = options;
 
   const fetchUrlIsAbsolute = isFetchUrlAbsolute(fetchUrl);
 
@@ -45,7 +53,7 @@ export async function fetchContents(options: {
   } else {
     const readUrl = getReadUrl(fetchUrl, baseUrl, integrations);
 
-    const res = await reader.readTree(readUrl);
+    const res = await reader.readTree(readUrl, { token });
     await fs.ensureDir(outputPath);
     await res.dir({ targetDir: outputPath });
   }
@@ -63,8 +71,16 @@ export async function fetchFile(options: {
   baseUrl?: string;
   fetchUrl?: string;
   outputPath: string;
+  token?: string;
 }) {
-  const { reader, integrations, baseUrl, fetchUrl = '.', outputPath } = options;
+  const {
+    reader,
+    integrations,
+    baseUrl,
+    fetchUrl = '.',
+    outputPath,
+    token,
+  } = options;
 
   const fetchUrlIsAbsolute = isFetchUrlAbsolute(fetchUrl);
 
@@ -76,7 +92,7 @@ export async function fetchFile(options: {
   } else {
     const readUrl = getReadUrl(fetchUrl, baseUrl, integrations);
 
-    const res = await reader.readUrl(readUrl);
+    const res = await reader.readUrl(readUrl, { token });
     await fs.ensureDir(path.dirname(outputPath));
     const buffer = await res.buffer();
     await fs.outputFile(outputPath, buffer.toString());