scaffolder-backend: add Preparers.fromConfig + integration type detection

packages/backend/src/plugins/scaffolder.ts

@@ -17,10 +17,6 @@
 import {
   CookieCutter,
   createRouter,
-  FilePreparer,
-  GithubPreparer,
-  GitlabPreparer,
-  AzurePreparer,
   Preparers,
   Publishers,
   GithubPublisher,
@@ -46,16 +42,7 @@ export default async function createPlugin({
   templaters.register('cookiecutter', cookiecutterTemplater);
   templaters.register('cra', craTemplater);
 
-  const filePreparer = new FilePreparer();
-
-  const gitlabPreparer = new GitlabPreparer(config);
-  const azurePreparer = new AzurePreparer(config);
-  const preparers = new Preparers();
-
-  preparers.register('file', filePreparer);
-  preparers.register('gitlab', gitlabPreparer);
-  preparers.register('gitlab/api', gitlabPreparer);
-  preparers.register('azure/api', azurePreparer);
+  const preparers = await Preparers.fromConfig(config, { logger });
 
   const publishers = new Publishers();
 
@@ -80,9 +67,6 @@ export default async function createPlugin({
         repoVisibility,
       });
 
-      const githubPreparer = new GithubPreparer({ token: githubToken });
-
-      preparers.register('github', githubPreparer);
       publishers.register('file', githubPublisher);
       publishers.register('github', githubPublisher);
     } catch (e) {

plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts

@@ -13,11 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { parseLocationAnnotation } from './helpers';
+import {
+  makeDeprecatedLocationTypeDetector,
+  parseLocationAnnotation,
+} from './helpers';
 import {
   TemplateEntityV1alpha1,
   LOCATION_ANNOTATION,
 } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
 
 describe('Helpers', () => {
   describe('parseLocationAnnotation', () => {
@@ -253,4 +257,29 @@ describe('Helpers', () => {
       });
     });
   });
+
+  describe('makeDeprecatedLocationTypeDetector', () => {
+    it('detects deprecated location types', () => {
+      const detector = makeDeprecatedLocationTypeDetector(
+        new ConfigReader({
+          integrations: {
+            github: [{ host: 'derp.com' }, { host: 'foo.com' }],
+            gitlab: [{ host: 'derp.org' }, { host: 'foo.org' }],
+            azure: [{ host: 'derp.net' }, { host: 'foo.net' }],
+          },
+        }),
+      );
+
+      expect(detector('http://lol:wut@derp.com/wat')).toBe('github');
+      expect(detector('https://foo.com/wat')).toBe('github');
+      expect(detector('http://derp.org:80/wat')).toBe('gitlab');
+      expect(detector('https://foo.org/wat')).toBe('gitlab');
+      expect(detector('http://not.derp.net')).toBe(undefined);
+      expect(detector('http://derp.net')).toBe('azure/api');
+      expect(detector('http://derp.net:8080/wat')).toBe('azure/api');
+      expect(detector('http://github.com')).toBe('github');
+      expect(detector('http://gitlab.com')).toBe('gitlab');
+      expect(detector('http://dev.azure.com')).toBe('azure/api');
+    });
+  });
 });

plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts

@@ -17,6 +17,7 @@ import {
   TemplateEntityV1alpha1,
   LOCATION_ANNOTATION,
 } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
 import { InputError } from '@backstage/backend-common';
 import { RemoteProtocol } from './types';
 
@@ -54,3 +55,39 @@ export const parseLocationAnnotation = (
     location,
   };
 };
+
+export type DeprecatedLocationTypeDetector = (
+  url: string,
+) => string | undefined;
+
+// The reason for the existence of this is to help in migration to using mostly locations
+// of type 'url'. This allows us to detect the deprecated location type based on the host,
+// which we in turn can use to select out preparer or publisher.
+//
+// TODO(Rugvip): This should be removed in the future once we fully migrate to using
+//               integrations configuration for the scaffolder.
+export function makeDeprecatedLocationTypeDetector(
+  config: Config,
+): DeprecatedLocationTypeDetector {
+  const hostMap = new Map();
+
+  // These are installed by default by the integrations
+  hostMap.set('github.com', 'github');
+  hostMap.set('gitlab.com', 'gitlab');
+  hostMap.set('dev.azure.com', 'azure/api');
+
+  config.getOptionalConfigArray('integrations.github')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'github');
+  });
+  config.getOptionalConfigArray('integrations.gitlab')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'gitlab');
+  });
+  config.getOptionalConfigArray('integrations.azure')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'azure/api');
+  });
+
+  return (url: string): string | undefined => {
+    const parsed = new URL(url);
+    return hostMap.get(parsed.hostname);
+  };
+}

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts

@@ -35,9 +35,9 @@ export class AzurePreparer implements PreparerBase {
   async prepare(template: TemplateEntityV1alpha1): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
 
-    if (protocol !== 'azure/api') {
+    if (!['azure/api', 'url'].includes(protocol)) {
       throw new InputError(
-        `Wrong location protocol: ${protocol}, should be 'azure/api'`,
+        `Wrong location protocol: ${protocol}, should be 'url'`,
       );
     }
     const templateId = template.metadata.name;

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -34,9 +34,9 @@ export class GithubPreparer implements PreparerBase {
     const { protocol, location } = parseLocationAnnotation(template);
     const { token } = this;
 
-    if (protocol !== 'github') {
+    if (!['github', 'url'].includes(protocol)) {
       throw new InputError(
-        `Wrong location protocol: ${protocol}, should be 'github'`,
+        `Wrong location protocol: ${protocol}, should be 'url'`,
       );
     }
     const templateId = template.metadata.name;

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts

@@ -36,9 +36,9 @@ export class GitlabPreparer implements PreparerBase {
   async prepare(template: TemplateEntityV1alpha1): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
 
-    if (['gitlab', 'gitlab/api'].indexOf(protocol) < 0) {
+    if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) {
       throw new InputError(
-        `Wrong location protocol: ${protocol}, should be 'gitlab' or 'gitlab/api'`,
+        `Wrong location protocol: ${protocol}, should be 'url'`,
       );
     }
     const templateId = template.metadata.name;

plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts

@@ -14,26 +14,85 @@
  * limitations under the License.
  */
 
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
 import { PreparerBase, PreparerBuilder } from './types';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { parseLocationAnnotation } from '../helpers';
+import {
+  DeprecatedLocationTypeDetector,
+  makeDeprecatedLocationTypeDetector,
+  parseLocationAnnotation,
+} from '../helpers';
 import { RemoteProtocol } from '../types';
+import { FilePreparer } from './file';
+import { GitlabPreparer } from './gitlab';
+import { AzurePreparer } from './azure';
+import { GithubPreparer } from './github';
 
 export class Preparers implements PreparerBuilder {
   private preparerMap = new Map<RemoteProtocol, PreparerBase>();
 
+  constructor(private readonly typeDetector?: DeprecatedLocationTypeDetector) {}
+
   register(protocol: RemoteProtocol, preparer: PreparerBase) {
     this.preparerMap.set(protocol, preparer);
   }
 
   get(template: TemplateEntityV1alpha1): PreparerBase {
-    const { protocol } = parseLocationAnnotation(template);
+    const { protocol, location } = parseLocationAnnotation(template);
+
     const preparer = this.preparerMap.get(protocol);
 
     if (!preparer) {
+      if ((protocol as string) === 'url') {
+        const type = this.typeDetector?.(location);
+        const preparer2 = type && this.preparerMap.get(type as RemoteProtocol);
+        if (preparer2) {
+          return preparer2;
+        }
+        throw new Error(`No preparer integration found for url "${location}"`);
+      }
       throw new Error(`No preparer registered for type: "${protocol}"`);
     }
 
     return preparer;
   }
+
+  static async fromConfig(
+    config: Config,
+    { logger }: { logger: Logger },
+  ): Promise<PreparerBuilder> {
+    const typeDetector = makeDeprecatedLocationTypeDetector(config);
+
+    const preparers = new Preparers(typeDetector);
+
+    const filePreparer = new FilePreparer();
+    const gitlabPreparer = new GitlabPreparer(config);
+    const azurePreparer = new AzurePreparer(config);
+
+    preparers.register('file', filePreparer);
+    preparers.register('gitlab', gitlabPreparer);
+    preparers.register('gitlab/api', gitlabPreparer);
+    preparers.register('azure/api', azurePreparer);
+
+    const githubConfig = config.getOptionalConfig('scaffolder.github');
+    if (githubConfig) {
+      try {
+        const githubToken = githubConfig.getString('token');
+        const githubPreparer = new GithubPreparer({ token: githubToken });
+
+        preparers.register('github', githubPreparer);
+      } catch (e) {
+        if (process.env.NODE_ENV !== 'development') {
+          throw new Error(
+            `Failed to initialize github scaffolding provider, ${e.message}`,
+          );
+        }
+
+        logger.warn(`Skipping github scaffolding provider, ${e.message}`);
+      }
+    }
+
+    return preparers;
+  }
 }