feat: reworking some of the preparers to support the integration config and add deprecation for old scaffolder config

packages/integration/src/gitlab/config.ts

@@ -31,7 +31,7 @@ export type GitLabIntegrationConfig = {
 
   /**
    * The base URL of the API of this provider, e.g. "https://gitlab.com/api/v4",
-   * with no trailing slash.
+   * with no trailing slash.s
    *
    * May be omitted specifically for GitLab; then it will be deduced.
    *

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts

@@ -33,6 +33,8 @@ describe('AzurePreparer', () => {
     clone: jest.fn(),
   };
 
+  const logger = getVoidLogger();
+
   jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
 
   let mockEntity: TemplateEntityV1alpha1;
@@ -78,31 +80,55 @@ describe('AzurePreparer', () => {
     };
   });
 
-  it('initializes git client with the correct arguments if an access token is provided for a repository', async () => {
+  // TODO(blam): Here's a test that will fail when the deprecation is complete
+  it('calls the clone command with deprecated token', async () => {
     const preparer = new AzurePreparer(
       new ConfigReader({
         scaffolder: {
           azure: {
             api: {
-              token: 'fake-token',
+              token: 'fake-azure-token',
             },
           },
         },
       }),
+      { logger },
     );
-    const logger = getVoidLogger();
-    await preparer.prepare(mockEntity, { logger });
+
+    await preparer.prepare(mockEntity);
 
     expect(Git.fromAuth).toHaveBeenCalledWith({
-      username: 'notempty',
-      password: 'fake-token',
       logger,
+      password: 'fake-azure-token',
+      username: 'notempty',
     });
   });
-  it('calls the clone command with the correct arguments for a repository', async () => {
-    const preparer = new AzurePreparer(new ConfigReader({}));
 
-    await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+  it('calls the clone command with token from integrations config', async () => {
+    const preparer = new AzurePreparer(
+      new ConfigReader({
+        integrations: {
+          azure: [
+            { host: 'dev.azure.com', token: 'fake-azure-token-integration' },
+          ],
+        },
+      }),
+      { logger },
+    );
+
+    await preparer.prepare(mockEntity);
+
+    expect(Git.fromAuth).toHaveBeenCalledWith({
+      logger,
+      password: 'fake-azure-token-integration',
+      username: 'notempty',
+    });
+  });
+
+  it('calls the clone command with the correct arguments for a repository', async () => {
+    const preparer = new AzurePreparer(new ConfigReader({}), { logger });
+
+    await preparer.prepare(mockEntity);
 
     expect(mockGitClient.clone).toHaveBeenCalledWith({
       url:
@@ -112,10 +138,10 @@ describe('AzurePreparer', () => {
   });
 
   it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
-    const preparer = new AzurePreparer(new ConfigReader({}));
+    const preparer = new AzurePreparer(new ConfigReader({}), { logger });
     delete mockEntity.spec.path;
 
-    await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+    await preparer.prepare(mockEntity);
 
     expect(mockGitClient.clone).toHaveBeenCalledWith({
       url:
@@ -125,12 +151,10 @@ describe('AzurePreparer', () => {
   });
 
   it('return the temp directory with the path to the folder if it is specified', async () => {
-    const preparer = new AzurePreparer(new ConfigReader({}));
+    const preparer = new AzurePreparer(new ConfigReader({}), { logger });
     mockEntity.spec.path = './template/test/1/2/3';
 
-    const response = await preparer.prepare(mockEntity, {
-      logger: getVoidLogger(),
-    });
+    const response = await preparer.prepare(mockEntity);
 
     expect(response.split('\\').join('/')).toMatch(
       /\/template\/test\/1\/2\/3$/,
@@ -138,11 +162,10 @@ describe('AzurePreparer', () => {
   });
 
   it('return the working directory with the path to the folder if it is specified', async () => {
-    const preparer = new AzurePreparer(new ConfigReader({}));
+    const preparer = new AzurePreparer(new ConfigReader({}), { logger });
     mockEntity.spec.path = './template/test/1/2/3';
 
     const response = await preparer.prepare(mockEntity, {
-      logger: getVoidLogger(),
       workingDirectory: '/workDir',
     });
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts

@@ -22,22 +22,46 @@ import { InputError, Git } from '@backstage/backend-common';
 import { PreparerBase, PreparerOptions } from './types';
 import GitUriParser from 'git-url-parse';
 import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+import {
+  readAzureIntegrationConfigs,
+  AzureIntegrationConfig,
+} from '@backstage/integration';
 
 export class AzurePreparer implements PreparerBase {
-  private readonly privateToken: string;
+  private readonly integrations: AzureIntegrationConfig[];
+  private readonly scaffolderToken: string | undefined;
+  private readonly logger: Logger;
 
-  constructor(config: Config) {
-    this.privateToken =
-      config.getOptionalString('scaffolder.azure.api.token') ?? '';
+  constructor(config: Config, { logger }: { logger: Logger }) {
+    this.logger = logger;
+    this.integrations = readAzureIntegrationConfigs(
+      config.getOptionalConfigArray('integrations.azure') ?? [],
+    );
+
+    if (!this.integrations.length) {
+      this.logger.warn(
+        'Integrations for Azure in Scaffolder are not set. This will cause errors in a future release. Please migrate to using integrations config and specifying tokens under hostnames',
+      );
+    }
+
+    this.scaffolderToken = config.getOptionalString(
+      'scaffolder.azure.api.token',
+    );
+
+    if (this.scaffolderToken) {
+      this.logger.warn(
+        "DEPRECATION: Using the token format under 'scaffolder.azure.api.token' will not be respected in future releases. Please consider using integrations config instead",
+      );
+    }
   }
 
   async prepare(
     template: TemplateEntityV1alpha1,
-    opts: PreparerOptions,
+    opts?: PreparerOptions,
   ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
     const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
-    const { logger } = opts;
 
     if (!['azure/api', 'url'].includes(protocol)) {
       throw new InputError(
@@ -57,15 +81,17 @@ export class AzurePreparer implements PreparerBase {
       template.spec.path ?? '.',
     );
 
+    const token = this.getToken(parsedGitLocation.resource);
+
     // Username can be anything but the empty string according to:
     // https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
-    const git = this.privateToken
+    const git = token
       ? Git.fromAuth({
-          password: this.privateToken,
+          password: token,
           username: 'notempty',
-          logger,
+          logger: this.logger,
         })
-      : Git.fromAuth({ logger });
+      : Git.fromAuth({ logger: this.logger });
 
     await git.clone({
       url: repositoryCheckoutUrl,
@@ -74,4 +100,11 @@ export class AzurePreparer implements PreparerBase {
 
     return path.resolve(tempDir, templateDirectory);
   }
+
+  private getToken(host: string): string | undefined {
+    return (
+      this.scaffolderToken ||
+      this.integrations.find(c => c.host === host)?.token
+    );
+  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -26,12 +26,14 @@ import {
   LOCATION_ANNOTATION,
 } from '@backstage/catalog-model';
 import { getVoidLogger, Git } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
 
 describe('GitHubPreparer', () => {
   let mockEntity: TemplateEntityV1alpha1;
   const mockGitClient = {
     clone: jest.fn(),
   };
+  const logger = getVoidLogger();
 
   jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
 
@@ -77,9 +79,18 @@ describe('GitHubPreparer', () => {
     };
   });
   it('calls the clone command with the correct arguments for a repository', async () => {
-    const preparer = new GithubPreparer();
+    const preparer = new GithubPreparer(
+      new ConfigReader({
+        scaffolder: {
+          github: {
+            token: 'fake-token',
+          },
+        },
+      }),
+      { logger },
+    );
 
-    await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+    await preparer.prepare(mockEntity);
 
     expect(mockGitClient.clone).toHaveBeenCalledWith({
       url: 'https://github.com/benjdlambert/backstage-graphql-template',
@@ -123,15 +134,42 @@ describe('GitHubPreparer', () => {
     );
   });
 
-  it('calls the clone command with the token when provided', async () => {
-    const preparer = new GithubPreparer({ token: 'abc' });
-    const logger = getVoidLogger();
+  it('calls the clone command with deprecated token', async () => {
+    const preparer = new GithubPreparer(
+      new ConfigReader({
+        scaffolder: {
+          github: {
+            token: 'fake-token',
+          },
+        },
+      }),
+      { logger },
+    );
 
-    await preparer.prepare(mockEntity, { logger });
+    await preparer.prepare(mockEntity);
 
     expect(Git.fromAuth).toHaveBeenCalledWith({
       logger,
-      username: 'abc',
+      username: 'fake-token',
+      password: 'x-oauth-basic',
+    });
+  });
+
+  it('calls the clone command with token from integrations config', async () => {
+    const preparer = new GithubPreparer(
+      new ConfigReader({
+        integrations: {
+          github: [{ host: 'github.com', token: 'fake-me' }],
+        },
+      }),
+      { logger },
+    );
+
+    await preparer.prepare(mockEntity);
+
+    expect(Git.fromAuth).toHaveBeenCalledWith({
+      logger,
+      username: 'fake-me',
       password: 'x-oauth-basic',
     });
   });

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -20,22 +20,46 @@ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError, Git } from '@backstage/backend-common';
 import { PreparerBase, PreparerOptions } from './types';
+import { Logger } from 'winston';
 import GitUriParser from 'git-url-parse';
+import { Config } from '@backstage/config';
+import {
+  GitHubIntegrationConfig,
+  readGitHubIntegrationConfigs,
+} from '@backstage/integration';
 
 export class GithubPreparer implements PreparerBase {
-  token?: string;
+  private readonly integrations: GitHubIntegrationConfig[];
+  private readonly scaffolderToken: string | undefined;
+  private readonly logger: Logger;
 
-  constructor(params: { token?: string } = {}) {
-    this.token = params.token;
+  constructor(config: Config, { logger }: { logger: Logger }) {
+    this.logger = logger;
+    this.integrations = readGitHubIntegrationConfigs(
+      config.getOptionalConfigArray('integrations.github') ?? [],
+    );
+
+    if (!this.integrations.length) {
+      this.logger.warn(
+        'Integrations for Github in Scaffolder are not set. This will cause errors in a future release. Please migrate to using integrations config and specifying tokens under hostnames',
+      );
+    }
+
+    this.scaffolderToken = config.getOptionalString('scaffolder.github.token');
+
+    if (this.scaffolderToken) {
+      this.logger.warn(
+        "DEPRECATION: Using the token format under 'scaffolder.github.token' will not be respected in future releases. Please consider using integrations config instead",
+      );
+    }
   }
 
   async prepare(
     template: TemplateEntityV1alpha1,
-    opts: PreparerOptions,
+    opts?: PreparerOptions,
   ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
     const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
-    const { logger } = opts;
 
     if (!['github', 'url'].includes(protocol)) {
       throw new InputError(
@@ -57,13 +81,15 @@ export class GithubPreparer implements PreparerBase {
 
     const checkoutLocation = path.resolve(tempDir, templateDirectory);
 
-    const git = this.token
+    const token = this.getToken(parsedGitLocation.resource);
+
+    const git = token
       ? Git.fromAuth({
-          username: this.token,
+          username: token,
           password: 'x-oauth-basic',
-          logger,
+          logger: this.logger,
         })
-      : Git.fromAuth({ logger });
+      : Git.fromAuth({ logger: this.logger });
 
     await git.clone({
       url: repositoryCheckoutUrl,
@@ -72,4 +98,10 @@ export class GithubPreparer implements PreparerBase {
 
     return checkoutLocation;
   }
+  private getToken(host: string): string | undefined {
+    return (
+      this.scaffolderToken ||
+      this.integrations.find(c => c.host === host)?.token
+    );
+  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts

@@ -70,6 +70,7 @@ describe('GitLabPreparer', () => {
   const mockGitClient = {
     clone: jest.fn(),
   };
+  const logger = getVoidLogger();
 
   jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
 
@@ -79,10 +80,10 @@ describe('GitLabPreparer', () => {
 
   ['gitlab', 'gitlab/api'].forEach(protocol => {
     it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => {
-      const preparer = new GitlabPreparer(new ConfigReader({}));
+      const preparer = new GitlabPreparer(new ConfigReader({}), { logger });
       mockEntity = mockEntityWithProtocol(protocol);
 
-      await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+      await preparer.prepare(mockEntity);
 
       expect(mockGitClient.clone).toHaveBeenCalledWith({
         url: 'https://gitlab.com/benjdlambert/backstage-graphql-template',
@@ -102,11 +103,11 @@ describe('GitLabPreparer', () => {
             ],
           },
         }),
+        { logger },
       );
       mockEntity = mockEntityWithProtocol(protocol);
-      const logger = getVoidLogger();
 
-      await preparer.prepare(mockEntity, { logger });
+      await preparer.prepare(mockEntity);
 
       expect(Git.fromAuth).toHaveBeenCalledWith({
         logger,
@@ -122,11 +123,11 @@ describe('GitLabPreparer', () => {
             gitlab: { api: { token: 'fake-token' } },
           },
         }),
+        { logger },
       );
       mockEntity = mockEntityWithProtocol(protocol);
-      const logger = getVoidLogger();
 
-      await preparer.prepare(mockEntity, { logger });
+      await preparer.prepare(mockEntity);
 
       expect(Git.fromAuth).toHaveBeenCalledWith({
         logger,
@@ -136,11 +137,11 @@ describe('GitLabPreparer', () => {
     });
 
     it(`calls the clone command with the correct arguments for a repository when no path is provided using the ${protocol} protocol`, async () => {
-      const preparer = new GitlabPreparer(new ConfigReader({}));
+      const preparer = new GitlabPreparer(new ConfigReader({}), { logger });
       mockEntity = mockEntityWithProtocol(protocol);
       delete mockEntity.spec.path;
 
-      await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+      await preparer.prepare(mockEntity);
 
       expect(mockGitClient.clone).toHaveBeenCalledWith({
         url: 'https://gitlab.com/benjdlambert/backstage-graphql-template',
@@ -149,23 +150,19 @@ describe('GitLabPreparer', () => {
     });
 
     it(`return the temp directory with the path to the folder if it is specified using the ${protocol} protocol`, async () => {
-      const preparer = new GitlabPreparer(new ConfigReader({}));
+      const preparer = new GitlabPreparer(new ConfigReader({}), { logger });
       mockEntity = mockEntityWithProtocol(protocol);
       mockEntity.spec.path = './template/test/1/2/3';
-      const response = await preparer.prepare(mockEntity, {
-        logger: getVoidLogger(),
-      });
-
+      const response = await preparer.prepare(mockEntity);
       expect(response.split('\\').join('/')).toMatch(
         /\/template\/test\/1\/2\/3$/,
       );
     });
 
     it('return the working directory with the path to the folder if it is specified', async () => {
-      const preparer = new GitlabPreparer(new ConfigReader({}));
+      const preparer = new GitlabPreparer(new ConfigReader({}), { logger });
       mockEntity.spec.path = './template/test/1/2/3';
       const response = await preparer.prepare(mockEntity, {
-        logger: getVoidLogger(),
         workingDirectory: '/workDir',
       });
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts

@@ -26,26 +26,41 @@ import os from 'os';
 import path from 'path';
 import { parseLocationAnnotation } from '../helpers';
 import { PreparerBase, PreparerOptions } from './types';
+import { Logger } from 'winston';
 
 export class GitlabPreparer implements PreparerBase {
   private readonly integrations: GitLabIntegrationConfig[];
   private readonly scaffolderToken: string | undefined;
+  private readonly logger: Logger;
 
-  constructor(config: Config) {
+  constructor(config: Config, { logger }: { logger: Logger }) {
+    this.logger = logger;
     this.integrations = readGitLabIntegrationConfigs(
       config.getOptionalConfigArray('integrations.gitlab') ?? [],
     );
+
+    if (!this.integrations.length) {
+      this.logger.warn(
+        'Integrations for GitLab in Scaffolder are not set. This will cause errors in a future release. Please migrate to using integrations config and specifying tokens under hostnames',
+      );
+    }
+
     this.scaffolderToken = config.getOptionalString(
       'scaffolder.gitlab.api.token',
     );
+
+    if (this.scaffolderToken) {
+      this.logger.warn(
+        "DEPRECATION: Using the token format under 'scaffolder.gitlab.api.token' will not be respected in future releases. Please consider using integrations config instead",
+      );
+    }
   }
 
   async prepare(
     template: TemplateEntityV1alpha1,
-    opts: PreparerOptions,
+    opts?: PreparerOptions,
   ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
-    const { logger } = opts;
     const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
 
     if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) {
@@ -71,9 +86,9 @@ export class GitlabPreparer implements PreparerBase {
       ? Git.fromAuth({
           password: token,
           username: 'oauth2',
-          logger,
+          logger: this.logger,
         })
-      : Git.fromAuth({ logger });
+      : Git.fromAuth({ logger: this.logger });
 
     await git.clone({
       url: repositoryCheckoutUrl,

plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts

@@ -77,31 +77,15 @@ export class Preparers implements PreparerBuilder {
     const preparers = new Preparers(typeDetector);
 
     const filePreparer = new FilePreparer();
-    const gitlabPreparer = new GitlabPreparer(config);
-    const azurePreparer = new AzurePreparer(config);
+    const gitlabPreparer = new GitlabPreparer(config, { logger });
+    const azurePreparer = new AzurePreparer(config, { logger });
+    const githubPreparer = new GithubPreparer(config, { logger });
 
     preparers.register('file', filePreparer);
     preparers.register('gitlab', gitlabPreparer);
     preparers.register('gitlab/api', gitlabPreparer);
     preparers.register('azure/api', azurePreparer);
-
-    const githubConfig = config.getOptionalConfig('scaffolder.github');
-    if (githubConfig) {
-      try {
-        const githubToken = githubConfig.getString('token');
-        const githubPreparer = new GithubPreparer({ token: githubToken });
-
-        preparers.register('github', githubPreparer);
-      } catch (e) {
-        if (process.env.NODE_ENV !== 'development') {
-          throw new Error(
-            `Failed to initialize github scaffolding provider, ${e.message}`,
-          );
-        }
-
-        logger.warn(`Skipping github scaffolding provider, ${e.message}`);
-      }
-    }
+    preparers.register('github', githubPreparer);
 
     return preparers;
   }

plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts

@@ -14,15 +14,13 @@
  * limitations under the License.
  */
 import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { Logger } from 'winston';
 import { RemoteProtocol } from '../types';
 
 export type PreparerOptions = {
-  logger: Logger;
   workingDirectory?: string;
 };
 
-export type PreparerBase = {
+export interface PreparerBase {
   /**
    * Given an Entity definition from the Service Catalog, go and prepare a directory
    * with contents from the remote location in temporary storage and return the path
@@ -30,9 +28,9 @@ export type PreparerBase = {
    */
   prepare(
     template: TemplateEntityV1alpha1,
-    opts: PreparerOptions,
+    opts?: PreparerOptions,
   ): Promise<string>;
-};
+}
 
 export type PreparerBuilder = {
   register(protocol: RemoteProtocol, preparer: PreparerBase): void;