Merge pull request #3117 from spotify/rugvip/urlscaffodler

scaffolder-backend: move config into plugin and add support for url location types

.changeset/chilly-emus-fetch.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Added .fromConfig static factories for Preparers and Publishers + read integrations config to support url location types

app-config.yaml

@@ -156,7 +156,6 @@ catalog:
 
 scaffolder:
   github:
-    host: https://github.com
     token:
       $env: GITHUB_TOKEN
     visibility: public # or 'internal' or 'private'

docs/features/software-templates/installation.md

@@ -111,44 +111,8 @@ export default async function createPlugin({
   templaters.register('cookiecutter', cookiecutterTemplater);
   templaters.register('cra', craTemplater);
 
-  const filePreparer = new FilePreparer();
-  const githubPreparer = new GithubPreparer();
-  const gitlabPreparer = new GitlabPreparer(config);
-  const preparers = new Preparers();
-
-  preparers.register('file', filePreparer);
-  preparers.register('github', githubPreparer);
-  preparers.register('gitlab', gitlabPreparer);
-  preparers.register('gitlab/api', gitlabPreparer);
-
-  const publishers = new Publishers();
-
-  const githubToken = config.getString('scaffolder.github.token');
-  const repoVisibility = config.getString(
-    'scaffolder.github.visibility',
-  ) as RepoVisibilityOptions;
-
-  const githubClient = new Octokit({ auth: githubToken });
-  const githubPublisher = new GithubPublisher({
-    client: githubClient,
-    token: githubToken,
-    repoVisibility,
-  });
-  publishers.register('file', githubPublisher);
-  publishers.register('github', githubPublisher);
-
-  const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
-
-  if (gitLabConfig) {
-    const gitLabToken = gitLabConfig.getString('token');
-    const gitLabClient = new Gitlab({
-      host: gitLabConfig.getOptionalString('baseUrl'),
-      token: gitLabToken,
-    });
-    const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
-    publishers.register('gitlab', gitLabPublisher);
-    publishers.register('gitlab/api', gitLabPublisher);
-  }
+  const preparers = await Preparers.fromConfig(config, { logger });
+  const publishers = await Publishers.fromConfig(config, { logger });
 
   const dockerClient = new Docker();
   return await createRouter({

packages/backend/src/plugins/scaffolder.ts

@@ -17,22 +17,11 @@
 import {
   CookieCutter,
   createRouter,
-  FilePreparer,
-  GithubPreparer,
-  GitlabPreparer,
-  AzurePreparer,
   Preparers,
   Publishers,
-  GithubPublisher,
-  GitlabPublisher,
-  AzurePublisher,
   CreateReactAppTemplater,
   Templaters,
-  RepoVisibilityOptions,
 } from '@backstage/plugin-scaffolder-backend';
-import { Octokit } from '@octokit/rest';
-import { Gitlab } from '@gitbeaker/node';
-import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
 import type { PluginEnvironment } from '../types';
 import Docker from 'dockerode';
 
@@ -46,111 +35,11 @@ export default async function createPlugin({
   templaters.register('cookiecutter', cookiecutterTemplater);
   templaters.register('cra', craTemplater);
 
-  const filePreparer = new FilePreparer();
-
-  const gitlabPreparer = new GitlabPreparer(config);
-  const azurePreparer = new AzurePreparer(config);
-  const preparers = new Preparers();
-
-  preparers.register('file', filePreparer);
-  preparers.register('gitlab', gitlabPreparer);
-  preparers.register('gitlab/api', gitlabPreparer);
-  preparers.register('azure/api', azurePreparer);
-
-  const publishers = new Publishers();
-
-  const githubConfig = config.getOptionalConfig('scaffolder.github');
-
-  if (githubConfig) {
-    try {
-      const repoVisibility = githubConfig.getString(
-        'visibility',
-      ) as RepoVisibilityOptions;
-
-      const githubToken = githubConfig.getString('token');
-      const githubHost =
-        githubConfig.getOptionalString('host') ?? 'https://github.com';
-      const githubClient = new Octokit({
-        auth: githubToken,
-        baseUrl: githubHost,
-      });
-      const githubPublisher = new GithubPublisher({
-        client: githubClient,
-        token: githubToken,
-        repoVisibility,
-      });
-
-      const githubPreparer = new GithubPreparer({ token: githubToken });
-
-      preparers.register('github', githubPreparer);
-      publishers.register('file', githubPublisher);
-      publishers.register('github', githubPublisher);
-    } catch (e) {
-      const providerName = 'github';
-      if (process.env.NODE_ENV !== 'development') {
-        throw new Error(
-          `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
-        );
-      }
-
-      logger.warn(
-        `Skipping ${providerName} scaffolding provider, ${e.message}`,
-      );
-    }
-  }
-
-  const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
-  if (gitLabConfig) {
-    try {
-      const gitLabToken = gitLabConfig.getString('token');
-      const gitLabClient = new Gitlab({
-        host: gitLabConfig.getOptionalString('baseUrl'),
-        token: gitLabToken,
-      });
-      const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
-      publishers.register('gitlab', gitLabPublisher);
-      publishers.register('gitlab/api', gitLabPublisher);
-    } catch (e) {
-      const providerName = 'gitlab';
-      if (process.env.NODE_ENV !== 'development') {
-        throw new Error(
-          `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
-        );
-      }
-
-      logger.warn(
-        `Skipping ${providerName} scaffolding provider, ${e.message}`,
-      );
-    }
-  }
-
-  const azureConfig = config.getOptionalConfig('scaffolder.azure');
-  if (azureConfig) {
-    try {
-      const baseUrl = azureConfig.getString('baseUrl');
-      const azureToken = azureConfig.getConfig('api').getString('token');
-
-      const authHandler = getPersonalAccessTokenHandler(azureToken);
-      const webApi = new WebApi(baseUrl, authHandler);
-      const azureClient = await webApi.getGitApi();
-
-      const azurePublisher = new AzurePublisher(azureClient, azureToken);
-      publishers.register('azure/api', azurePublisher);
-    } catch (e) {
-      const providerName = 'azure';
-      if (process.env.NODE_ENV !== 'development') {
-        throw new Error(
-          `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
-        );
-      }
-
-      logger.warn(
-        `Skipping ${providerName} scaffolding provider, ${e.message}`,
-      );
-    }
-  }
+  const preparers = await Preparers.fromConfig(config, { logger });
+  const publishers = await Publishers.fromConfig(config, { logger });
 
   const dockerClient = new Docker();
+
   return await createRouter({
     preparers,
     templaters,

plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts

@@ -13,11 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { parseLocationAnnotation } from './helpers';
+import {
+  makeDeprecatedLocationTypeDetector,
+  parseLocationAnnotation,
+} from './helpers';
 import {
   TemplateEntityV1alpha1,
   LOCATION_ANNOTATION,
 } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
 
 describe('Helpers', () => {
   describe('parseLocationAnnotation', () => {
@@ -253,4 +257,29 @@ describe('Helpers', () => {
       });
     });
   });
+
+  describe('makeDeprecatedLocationTypeDetector', () => {
+    it('detects deprecated location types', () => {
+      const detector = makeDeprecatedLocationTypeDetector(
+        new ConfigReader({
+          integrations: {
+            github: [{ host: 'derp.com' }, { host: 'foo.com' }],
+            gitlab: [{ host: 'derp.org' }, { host: 'foo.org' }],
+            azure: [{ host: 'derp.net' }, { host: 'foo.net' }],
+          },
+        }),
+      );
+
+      expect(detector('http://lol:wut@derp.com/wat')).toBe('github');
+      expect(detector('https://foo.com/wat')).toBe('github');
+      expect(detector('http://derp.org:80/wat')).toBe('gitlab');
+      expect(detector('https://foo.org/wat')).toBe('gitlab');
+      expect(detector('http://not.derp.net')).toBe(undefined);
+      expect(detector('http://derp.net')).toBe('azure/api');
+      expect(detector('http://derp.net:8080/wat')).toBe('azure/api');
+      expect(detector('http://github.com')).toBe('github');
+      expect(detector('http://gitlab.com')).toBe('gitlab');
+      expect(detector('http://dev.azure.com')).toBe('azure/api');
+    });
+  });
 });

plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts

@@ -17,6 +17,7 @@ import {
   TemplateEntityV1alpha1,
   LOCATION_ANNOTATION,
 } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
 import { InputError } from '@backstage/backend-common';
 import { RemoteProtocol } from './types';
 
@@ -54,3 +55,39 @@ export const parseLocationAnnotation = (
     location,
   };
 };
+
+export type DeprecatedLocationTypeDetector = (
+  url: string,
+) => string | undefined;
+
+// The reason for the existence of this is to help in migration to using mostly locations
+// of type 'url'. This allows us to detect the deprecated location type based on the host,
+// which we in turn can use to select out preparer or publisher.
+//
+// TODO(Rugvip): This should be removed in the future once we fully migrate to using
+//               integrations configuration for the scaffolder.
+export function makeDeprecatedLocationTypeDetector(
+  config: Config,
+): DeprecatedLocationTypeDetector {
+  const hostMap = new Map();
+
+  // These are installed by default by the integrations
+  hostMap.set('github.com', 'github');
+  hostMap.set('gitlab.com', 'gitlab');
+  hostMap.set('dev.azure.com', 'azure/api');
+
+  config.getOptionalConfigArray('integrations.github')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'github');
+  });
+  config.getOptionalConfigArray('integrations.gitlab')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'gitlab');
+  });
+  config.getOptionalConfigArray('integrations.azure')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'azure/api');
+  });
+
+  return (url: string): string | undefined => {
+    const parsed = new URL(url);
+    return hostMap.get(parsed.hostname);
+  };
+}

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts

@@ -35,9 +35,9 @@ export class AzurePreparer implements PreparerBase {
   async prepare(template: TemplateEntityV1alpha1): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
 
-    if (protocol !== 'azure/api') {
+    if (!['azure/api', 'url'].includes(protocol)) {
       throw new InputError(
-        `Wrong location protocol: ${protocol}, should be 'azure/api'`,
+        `Wrong location protocol: ${protocol}, should be 'url'`,
       );
     }
     const templateId = template.metadata.name;

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -34,9 +34,9 @@ export class GithubPreparer implements PreparerBase {
     const { protocol, location } = parseLocationAnnotation(template);
     const { token } = this;
 
-    if (protocol !== 'github') {
+    if (!['github', 'url'].includes(protocol)) {
       throw new InputError(
-        `Wrong location protocol: ${protocol}, should be 'github'`,
+        `Wrong location protocol: ${protocol}, should be 'url'`,
       );
     }
     const templateId = template.metadata.name;

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts

@@ -36,9 +36,9 @@ export class GitlabPreparer implements PreparerBase {
   async prepare(template: TemplateEntityV1alpha1): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
 
-    if (['gitlab', 'gitlab/api'].indexOf(protocol) < 0) {
+    if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) {
       throw new InputError(
-        `Wrong location protocol: ${protocol}, should be 'gitlab' or 'gitlab/api'`,
+        `Wrong location protocol: ${protocol}, should be 'url'`,
       );
     }
     const templateId = template.metadata.name;

plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts

@@ -14,26 +14,85 @@
  * limitations under the License.
  */
 
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
 import { PreparerBase, PreparerBuilder } from './types';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { parseLocationAnnotation } from '../helpers';
+import {
+  DeprecatedLocationTypeDetector,
+  makeDeprecatedLocationTypeDetector,
+  parseLocationAnnotation,
+} from '../helpers';
 import { RemoteProtocol } from '../types';
+import { FilePreparer } from './file';
+import { GitlabPreparer } from './gitlab';
+import { AzurePreparer } from './azure';
+import { GithubPreparer } from './github';
 
 export class Preparers implements PreparerBuilder {
   private preparerMap = new Map<RemoteProtocol, PreparerBase>();
 
+  constructor(private readonly typeDetector?: DeprecatedLocationTypeDetector) {}
+
   register(protocol: RemoteProtocol, preparer: PreparerBase) {
     this.preparerMap.set(protocol, preparer);
   }
 
   get(template: TemplateEntityV1alpha1): PreparerBase {
-    const { protocol } = parseLocationAnnotation(template);
+    const { protocol, location } = parseLocationAnnotation(template);
+
     const preparer = this.preparerMap.get(protocol);
 
     if (!preparer) {
+      if ((protocol as string) === 'url') {
+        const type = this.typeDetector?.(location);
+        const detected = type && this.preparerMap.get(type as RemoteProtocol);
+        if (detected) {
+          return detected;
+        }
+        throw new Error(`No preparer integration found for url "${location}"`);
+      }
       throw new Error(`No preparer registered for type: "${protocol}"`);
     }
 
     return preparer;
   }
+
+  static async fromConfig(
+    config: Config,
+    { logger }: { logger: Logger },
+  ): Promise<PreparerBuilder> {
+    const typeDetector = makeDeprecatedLocationTypeDetector(config);
+
+    const preparers = new Preparers(typeDetector);
+
+    const filePreparer = new FilePreparer();
+    const gitlabPreparer = new GitlabPreparer(config);
+    const azurePreparer = new AzurePreparer(config);
+
+    preparers.register('file', filePreparer);
+    preparers.register('gitlab', gitlabPreparer);
+    preparers.register('gitlab/api', gitlabPreparer);
+    preparers.register('azure/api', azurePreparer);
+
+    const githubConfig = config.getOptionalConfig('scaffolder.github');
+    if (githubConfig) {
+      try {
+        const githubToken = githubConfig.getString('token');
+        const githubPreparer = new GithubPreparer({ token: githubToken });
+
+        preparers.register('github', githubPreparer);
+      } catch (e) {
+        if (process.env.NODE_ENV !== 'development') {
+          throw new Error(
+            `Failed to initialize github scaffolding provider, ${e.message}`,
+          );
+        }
+
+        logger.warn(`Skipping github scaffolding provider, ${e.message}`);
+      }
+    }
+
+    return preparers;
+  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts

@@ -14,26 +14,145 @@
  * limitations under the License.
  */
 
+import { Logger } from 'winston';
+import { Octokit } from '@octokit/rest';
+import { Gitlab } from '@gitbeaker/node';
+import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
+import { Config } from '@backstage/config';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { parseLocationAnnotation } from '../helpers';
+import {
+  DeprecatedLocationTypeDetector,
+  makeDeprecatedLocationTypeDetector,
+  parseLocationAnnotation,
+} from '../helpers';
 import { PublisherBase, PublisherBuilder } from './types';
 import { RemoteProtocol } from '../types';
+import { GithubPublisher, RepoVisibilityOptions } from './github';
+import { GitlabPublisher } from './gitlab';
+import { AzurePublisher } from './azure';
 
 export class Publishers implements PublisherBuilder {
   private publisherMap = new Map<RemoteProtocol, PublisherBase>();
 
+  constructor(private readonly typeDetector?: DeprecatedLocationTypeDetector) {}
+
   register(protocol: RemoteProtocol, publisher: PublisherBase) {
     this.publisherMap.set(protocol, publisher);
   }
 
   get(template: TemplateEntityV1alpha1): PublisherBase {
-    const { protocol } = parseLocationAnnotation(template);
+    const { protocol, location } = parseLocationAnnotation(template);
     const publisher = this.publisherMap.get(protocol);
 
     if (!publisher) {
+      if ((protocol as string) === 'url') {
+        const type = this.typeDetector?.(location);
+        const detected = type && this.publisherMap.get(type as RemoteProtocol);
+        if (detected) {
+          return detected;
+        }
+        throw new Error(`No preparer integration found for url "${location}"`);
+      }
       throw new Error(`No publisher registered for type: "${protocol}"`);
     }
 
     return publisher;
   }
+
+  static async fromConfig(
+    config: Config,
+    { logger }: { logger: Logger },
+  ): Promise<PublisherBuilder> {
+    const typeDetector = makeDeprecatedLocationTypeDetector(config);
+    const publishers = new Publishers(typeDetector);
+
+    const githubConfig = config.getOptionalConfig('scaffolder.github');
+    if (githubConfig) {
+      try {
+        const repoVisibility = githubConfig.getString(
+          'visibility',
+        ) as RepoVisibilityOptions;
+
+        const githubToken = githubConfig.getString('token');
+        const githubHost =
+          githubConfig.getOptionalString('host') ?? 'https://api.github.com';
+        const githubClient = new Octokit({
+          auth: githubToken,
+          baseUrl: githubHost,
+        });
+        const githubPublisher = new GithubPublisher({
+          client: githubClient,
+          token: githubToken,
+          repoVisibility,
+        });
+
+        publishers.register('file', githubPublisher);
+        publishers.register('github', githubPublisher);
+      } catch (e) {
+        const providerName = 'github';
+        if (process.env.NODE_ENV !== 'development') {
+          throw new Error(
+            `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
+          );
+        }
+
+        logger.warn(
+          `Skipping ${providerName} scaffolding provider, ${e.message}`,
+        );
+      }
+    }
+
+    const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
+    if (gitLabConfig) {
+      try {
+        const gitLabToken = gitLabConfig.getString('token');
+        const gitLabClient = new Gitlab({
+          host: gitLabConfig.getOptionalString('baseUrl'),
+          token: gitLabToken,
+        });
+        const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
+        publishers.register('gitlab', gitLabPublisher);
+        publishers.register('gitlab/api', gitLabPublisher);
+      } catch (e) {
+        const providerName = 'gitlab';
+        if (process.env.NODE_ENV !== 'development') {
+          throw new Error(
+            `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
+          );
+        }
+
+        logger.warn(
+          `Skipping ${providerName} scaffolding provider, ${e.message}`,
+        );
+      }
+    }
+
+    const azureConfig = config.getOptionalConfig('scaffolder.azure');
+    if (azureConfig) {
+      try {
+        const baseUrl = azureConfig.getString('baseUrl');
+        const azureToken = azureConfig.getConfig('api').getString('token');
+
+        const authHandler = getPersonalAccessTokenHandler(azureToken);
+        const webApi = new WebApi(baseUrl, authHandler);
+        const azureClient = await webApi.getGitApi();
+
+        const azurePublisher = new AzurePublisher(azureClient, azureToken);
+        publishers.register('azure/api', azurePublisher);
+      } catch (e) {
+        const providerName = 'azure';
+        if (process.env.NODE_ENV !== 'development') {
+          throw new Error(
+            `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
+          );
+        }
+
+        logger.warn(
+          `Skipping ${providerName} scaffolding provider, ${e.message}`,
+        );
+      }
+    }
+
+    return publishers;
+  }
 }