feat(scaffolder): added the ability to process github remote sources, without authenication for now at least

plugins/scaffolder-backend/package.json

@@ -30,14 +30,18 @@
     "express": "^4.17.1",
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.0",
+    "git-url-parse": "^11.1.2",
     "globby": "^11.0.0",
     "helmet": "^3.22.0",
     "morgan": "^1.10.0",
+    "nodegit": "0.26.5",
     "winston": "^3.2.1"
   },
   "devDependencies": {
     "@backstage/cli": "^0.1.1-alpha.9",
     "@types/fs-extra": "^9.0.1",
+    "@types/git-url-parse": "^9.0.0",
+    "@types/nodegit": "0.26.5",
     "@types/supertest": "^2.0.8",
     "supertest": "^4.0.2",
     "yaml": "^1.10.0"

plugins/scaffolder-backend/scripts/mock-data

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
 
-curl \
---location \
---request POST 'localhost:7000/catalog/locations' \
---header 'Content-Type: application/json' \
---data-raw "{\"type\": \"file\", \"target\": \"$(pwd)/sample-templates/react-ssr-template/template.yaml\"}"
+# curl \
+# --location \
+# --request POST 'localhost:7000/catalog/locations' \
+# --header 'Content-Type: application/json' \
+# --data-raw "{\"type\": \"file\", \"target\": \"$(pwd)/sample-templates/react-ssr-template/template.yaml\"}"
 
+
+curl \
+    --location \
+    --request POST 'localhost:7000/catalog/locations' \
+    --header 'Content-Type: application/json' \
+    --data-raw "{\"type\": \"github\", \"target\": \"https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml\"}"

plugins/scaffolder-backend/src/scaffolder/prepare/github.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs-extra';
+import path from 'path';
+import os from 'os';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { parseLocationAnnotation } from './helpers';
+import { InputError } from '@backstage/backend-common';
+import { PreparerBase } from './types';
+import GitUriParser from 'git-url-parse';
+import { Clone, CheckoutOptions } from 'nodegit';
+
+export class GithubPreparer implements PreparerBase {
+  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+    const { protocol, location } = parseLocationAnnotation(template);
+
+    if (protocol !== 'github') {
+      throw new InputError(
+        `Wrong location protocol: ${protocol}, should be 'github'`,
+      );
+    }
+    const templateId = template.metadata.name;
+
+    const parsedGitLocation = GitUriParser(location);
+    const repositoryCheckoutUrl = parsedGitLocation.toString('https');
+    const tempDir = await fs.promises.mkdtemp(
+      path.join(os.tmpdir(), templateId),
+    );
+
+    const templateDirectory = path.join(
+      `${path.dirname(parsedGitLocation.filepath)}`,
+      template.spec.path ?? '.',
+    );
+
+    const checkoutOptions = new CheckoutOptions();
+    checkoutOptions.paths = [templateDirectory];
+
+    await Clone.clone(repositoryCheckoutUrl, tempDir, {
+      checkoutOpts: checkoutOptions,
+      // TODO(blam): Maybe need some auth here?
+    });
+
+    return path.resolve(tempDir, templateDirectory);
+  }
+}

plugins/scaffolder-backend/src/scaffolder/prepare/index.ts

@@ -17,3 +17,4 @@ export * from './preparers';
 export * from './types';
 export * from './helpers';
 export * from './file';
+export * from './github';

plugins/scaffolder-backend/src/scaffolder/prepare/types.ts

@@ -29,4 +29,4 @@ export type PreparerBuilder = {
   get(template: TemplateEntityV1alpha1): PreparerBase;
 };
 
-export type RemoteProtocol = 'file';
+export type RemoteProtocol = 'file' | 'github';

plugins/scaffolder-backend/src/service/router.ts

@@ -44,19 +44,20 @@ export async function createRouter(
       kind: 'Template',
       metadata: {
         annotations: {
-          'backstage.io/managed-by-location': `file:${__dirname}/../../sample-templates/react-ssr-template/template.yaml`,
+          'backstage.io/managed-by-location':
+            'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
         },
-        name: 'react-ssr-template',
-        title: 'React SSR Template',
+        name: 'graphql-starter',
+        title: 'GraphQL Service',
         description:
-          'Next.js application skeleton for creating isomorphic web applications.',
-        uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
-        etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
+          'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+        uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+        etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
         generation: 1,
       },
       spec: {
         type: 'cookiecutter',
-        path: '.',
+        path: './template',
       },
     };