Merge pull request #4194 from backstage/orkohunter/update-templates-store-path

templates: Update Software Templates to use updated cookiecutter.storePath

.changeset/orange-roses-visit.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Fixing issues with templating and full URL's as `storePath`'s

docs/features/software-templates/extending/create-your-own-publisher.md

@@ -57,7 +57,7 @@ That type looks like the following:
 export type PublisherBase = {
   publish(opts: {
     entity: TemplateEntityV1alpha1;
-    values: RequiredTemplateValues & Record<string, JsonValue>;
+    values: TemplaterValues;
     directory: string;
   }): Promise<{ remoteUrl: string }>;
 };

docs/features/software-templates/extending/create-your-own-templater.md

@@ -61,7 +61,7 @@ That type looks like the following:
 ```ts
 export type TemplaterRunOptions = {
   directory: string;
-  values: RequiredTemplateValues & Record<string, JsonValue>;
+  values: TemplaterValues;
   logStream?: Writable;
   dockerClient: Docker;
 };

plugins/scaffolder-backend/package.json

@@ -38,6 +38,7 @@
     "@octokit/rest": "^18.0.12",
     "@types/dockerode": "^3.2.1",
     "@types/express": "^4.17.6",
+    "@types/git-url-parse": "^9.0.0",
     "azure-devops-node-api": "^10.1.1",
     "command-exists-promise": "^2.0.2",
     "compression": "^1.7.4",
@@ -61,7 +62,6 @@
     "@backstage/cli": "^0.4.7",
     "@backstage/test-utils": "^0.1.5",
     "@types/fs-extra": "^9.0.1",
-    "@types/git-url-parse": "^9.0.0",
     "@types/supertest": "^2.0.8",
     "supertest": "^4.0.2",
     "yaml": "^1.10.0",

plugins/scaffolder-backend/sample-templates/docs-template/{{cookiecutter.component_id}}/catalog-info.yaml

@@ -4,8 +4,8 @@ metadata:
   name: {{cookiecutter.component_id | jsonify}}
   description: {{cookiecutter.description | jsonify}}
   annotations:
-    github.com/project-slug: {{cookiecutter.storePath | jsonify}}
-    backstage.io/techdocs-ref: github:https://github.com/{{cookiecutter.storePath}}
+    github.com/project-slug: {{cookiecutter.destination.git.owner + "/" + cookiecutter.destination.git.name}}
+    backstage.io/techdocs-ref: url:{{cookiecutter.storePath}}
 spec:
   type: documentation
   lifecycle: experimental

plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/catalog-info.yaml

@@ -4,8 +4,8 @@ metadata:
   name: {{cookiecutter.component_id | jsonify}}
   description: {{cookiecutter.description | jsonify}}
   annotations:
-    github.com/project-slug: {{cookiecutter.storePath | jsonify}}
-    backstage.io/techdocs-ref: github:https://github.com/{{cookiecutter.storePath}}
+    github.com/project-slug: {{cookiecutter.destination.git.owner + "/" + cookiecutter.destination.git.name}}
+    backstage.io/techdocs-ref: url:{{cookiecutter.storePath}}
 spec:
   type: website
   lifecycle: experimental

plugins/scaffolder-backend/sample-templates/springboot-grpc-template/{{cookiecutter.component_id}}/catalog-info.yaml

@@ -4,8 +4,8 @@ metadata:
   name: {{cookiecutter.component_id | jsonify}}
   description: {{cookiecutter.description | jsonify}}
   annotations:
-    github.com/project-slug: {{cookiecutter.storePath | jsonify}}
-    backstage.io/techdocs-ref: github:https://github.com/{{cookiecutter.storePath}}
+    github.com/project-slug: {{cookiecutter.destination.git.owner + "/" + cookiecutter.destination.git.name}}
+    backstage.io/techdocs-ref: url:{{cookiecutter.storePath}}
 spec:
   type: service
   lifecycle: experimental

plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts

@@ -17,6 +17,7 @@ import { JobProcessor } from './processor';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { StageInput } from './types';
 import { RequiredTemplateValues } from '../stages/templater';
+import parseGitUrl from 'git-url-parse';
 
 describe('JobProcessor', () => {
   const mockEntity: TemplateEntityV1alpha1 = {
@@ -61,7 +62,10 @@ describe('JobProcessor', () => {
 
   const mockValues: RequiredTemplateValues = {
     owner: 'blobby',
-    storePath: 'backstage/mock-repo',
+    storePath: 'https://github.com/backstage/mock-repo',
+    destination: {
+      git: parseGitUrl('https://github.com/backstage/mock-repo'),
+    },
   };
 
   describe('create', () => {

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 import { Processor, Job, StageContext, StageInput } from './types';
-import { JsonValue } from '@backstage/config';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import * as uuid from 'uuid';
 import Docker from 'dockerode';
-import { RequiredTemplateValues, TemplaterBase } from '../stages/templater';
+import { TemplaterValues, TemplaterBase } from '../stages/templater';
 import { PreparerBuilder } from '../stages/prepare';
 import { makeLogStream } from './logger';
 
@@ -42,7 +41,7 @@ export class JobProcessor implements Processor {
     stages,
   }: {
     entity: TemplateEntityV1alpha1;
-    values: RequiredTemplateValues & Record<string, JsonValue>;
+    values: TemplaterValues;
     stages: StageInput[];
   }): Job {
     const id = uuid.v4();

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -15,15 +15,14 @@
  */
 import type { Writable } from 'stream';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { JsonValue } from '@backstage/config';
-import { RequiredTemplateValues } from '../stages/templater';
+import { TemplaterValues } from '../stages/templater';
 import { Logger } from 'winston';
 
 // Context will be a mutable object which is passed between stages
 // To share data, but also thinking that we can pass in functions here too
 // To maybe create sub steps or fail the entire thing, or skip stages down the line.
 export type StageContext<T = {}> = {
-  values: RequiredTemplateValues & Record<string, JsonValue>;
+  values: TemplaterValues;
   entity: TemplateEntityV1alpha1;
   logger: Logger;
   logStream: Writable;
@@ -58,7 +57,7 @@ export type Processor = {
     stages,
   }: {
     entity: TemplateEntityV1alpha1;
-    values: RequiredTemplateValues & Record<string, JsonValue>;
+    values: TemplaterValues;
     stages: StageInput[];
   }): Job;
 

plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts

@@ -13,8 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { RequiredTemplateValues } from '../templater';
-import { JsonValue } from '@backstage/config';
+import { TemplaterValues } from '../templater';
 import { Logger } from 'winston';
 
 /**
@@ -32,7 +31,7 @@ export type PublisherBase = {
 };
 
 export type PublisherOptions = {
-  values: RequiredTemplateValues & Record<string, JsonValue>;
+  values: TemplaterValues;
   directory: string;
   logger: Logger;
 };

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts

@@ -26,6 +26,7 @@ import path from 'path';
 import { RunDockerContainerOptions, RunCommandOptions } from './helpers';
 import { PassThrough } from 'stream';
 import Docker from 'dockerode';
+import parseGitUrl from 'git-url-parse';
 
 const commandExists = require('command-exists-promise');
 
@@ -56,16 +57,21 @@ describe('CookieCutter Templater', () => {
 
     const values = {
       owner: 'blobby',
-      storePath: 'backstage/end-repo',
+      storePath: 'https://github.com/org/repo',
       description: 'description',
       component_id: 'newthing',
+      destination: {
+        git: parseGitUrl('https://github.com/org/repo'),
+      },
     };
 
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
 
     const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
 
-    expect(cookieCutterJson).toEqual(expect.objectContaining(values));
+    expect(cookieCutterJson).toEqual(
+      expect.objectContaining(JSON.parse(JSON.stringify(values))),
+    );
   });
 
   it('should merge any value that is in the cookiecutter.json path already', async () => {
@@ -78,15 +84,24 @@ describe('CookieCutter Templater', () => {
 
     const values = {
       owner: 'blobby',
-      storePath: 'backstage/end-repo',
+      storePath: 'https://github.com/org/repo',
       component_id: 'something',
+      destination: {
+        git: parseGitUrl('https://github.com/org/repo'),
+      },
     };
 
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
 
     const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
 
-    expect(cookieCutterJson).toEqual({ ...existingJson, ...values });
+    expect(cookieCutterJson).toEqual({
+      ...existingJson,
+      ...values,
+      destination: {
+        git: expect.objectContaining({ organization: 'org', name: 'repo' }),
+      },
+    });
   });
 
   it('should throw an error if the cookiecutter json is malformed and not missing', async () => {
@@ -96,7 +111,10 @@ describe('CookieCutter Templater', () => {
 
     const values = {
       owner: 'blobby',
-      storePath: 'backstage/end-repo',
+      storePath: 'https://github.com/org/repo',
+      destination: {
+        git: parseGitUrl('https://github.com/org/repo'),
+      },
     };
 
     await expect(
@@ -109,8 +127,11 @@ describe('CookieCutter Templater', () => {
 
     const values = {
       owner: 'blobby',
-      storePath: 'backstage/end-repo',
+      storePath: 'https://github.com/org/repo',
       component_id: 'newthing',
+      destination: {
+        git: parseGitUrl('https://github.com/org/repo'),
+      },
     };
 
     await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
@@ -137,8 +158,11 @@ describe('CookieCutter Templater', () => {
 
     const values = {
       owner: 'blobby',
-      storePath: 'backstage/end-repo',
+      storePath: 'https://github.com/org/repo',
       component_id: 'newthing',
+      destination: {
+        git: parseGitUrl('https://github.com/org/repo'),
+      },
     };
 
     const { resultDir } = await cookie.run({
@@ -157,8 +181,11 @@ describe('CookieCutter Templater', () => {
 
     const values = {
       owner: 'blobby',
-      storePath: 'backstage/end-repo',
+      storePath: 'https://github.com/org/repo',
       component_id: 'newthing',
+      destination: {
+        git: parseGitUrl('https://github.com/org/repo'),
+      },
     };
 
     await cookie.run({
@@ -203,8 +230,11 @@ describe('CookieCutter Templater', () => {
 
       const values = {
         owner: 'blobby',
-        storePath: 'backstage/end-repo',
+        storePath: 'https://github.com/org/repo',
         component_id: 'newthing',
+        destination: {
+          git: parseGitUrl('https://github.com/org/repo'),
+        },
       };
 
       await cookie.run({
@@ -243,7 +273,10 @@ describe('CookieCutter Templater', () => {
           directory: tempdir,
           values: {
             owner: 'blobby',
-            storePath: 'backstage/end-repo',
+            storePath: 'https://github.com/org/repo',
+            destination: {
+              git: parseGitUrl('https://github.com/org/repo'),
+            },
           },
           logStream: stream,
           dockerClient: mockDocker,

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts

@@ -18,6 +18,7 @@ import { JsonValue } from '@backstage/config';
 import { runDockerContainer, runCommand } from './helpers';
 import { TemplaterBase, TemplaterRunOptions } from '.';
 import path from 'path';
+
 import { TemplaterRunResult } from './types';
 
 const commandExists = require('command-exists-promise');

plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts

@@ -70,7 +70,9 @@ export class CreateReactAppTemplater implements TemplaterBase {
         `${finalDir}/.github/workflows/main.yml`,
       );
 
-      extraAnnotations[GITHUB_ACTIONS_ANNOTATION] = options.values.storePath;
+      extraAnnotations[
+        GITHUB_ACTIONS_ANNOTATION
+      ] = `${options.values?.destination?.git?.owner}/${options.values?.destination?.git?.name}`;
     }
 
     const componentInfo = {

plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts

@@ -15,8 +15,8 @@
  */
 import type { Writable } from 'stream';
 import Docker from 'dockerode';
-import { JsonValue } from '@backstage/config';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import gitUrlParse from 'git-url-parse';
 
 /**
  * Currently the required template values. The owner
@@ -25,8 +25,13 @@ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 export type RequiredTemplateValues = {
   owner: string;
   storePath: string;
+  destination?: {
+    git?: gitUrlParse.GitUrl;
+  };
 };
 
+export type TemplaterValues = RequiredTemplateValues & Record<string, any>;
+
 /**
  * The returned directory from the templater which is ready
  * to pass to the next stage of the scaffolder which is publishing
@@ -42,7 +47,7 @@ export type TemplaterRunResult = {
  */
 export type TemplaterRunOptions = {
   directory: string;
-  values: RequiredTemplateValues & Record<string, JsonValue>;
+  values: TemplaterValues;
   logStream?: Writable;
   dockerClient: Docker;
 };

plugins/scaffolder-backend/src/service/router.test.ts

@@ -107,10 +107,14 @@ describe('createRouter - working directory', () => {
     });
 
     const app = express().use(router);
-    await request(app).post('/v1/jobs').send({
-      templateName: '',
-      values: {},
-    });
+    await request(app)
+      .post('/v1/jobs')
+      .send({
+        templateName: '',
+        values: {
+          storePath: 'https://github.com/backstage/good',
+        },
+      });
 
     expect(mockPrepare).toBeCalledWith(expect.anything(), {
       logger: expect.anything(),
@@ -130,10 +134,14 @@ describe('createRouter - working directory', () => {
     });
 
     const app = express().use(router);
-    await request(app).post('/v1/jobs').send({
-      templateName: '',
-      values: {},
-    });
+    await request(app)
+      .post('/v1/jobs')
+      .send({
+        templateName: '',
+        values: {
+          storePath: 'https://github.com/backstage/goodrepo',
+        },
+      });
 
     expect(mockPrepare).toBeCalledWith(expect.anything(), {
       logger: expect.anything(),
@@ -200,10 +208,14 @@ describe('createRouter', () => {
 
   describe('POST /v1/jobs', () => {
     it('rejects template values which do not match the template schema definition', async () => {
-      const response = await request(app).post('/v1/jobs').send({
-        templateName: '',
-        values: {},
-      });
+      const response = await request(app)
+        .post('/v1/jobs')
+        .send({
+          templateName: '',
+          values: {
+            storePath: 'https://github.com/backstage/backstage',
+          },
+        });
 
       expect(response.status).toEqual(400);
     });

plugins/scaffolder-backend/src/service/router.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Config, JsonValue } from '@backstage/config';
+import { Config } from '@backstage/config';
 import fs from 'fs-extra';
 import Docker from 'dockerode';
 import express from 'express';
@@ -23,15 +23,16 @@ import { Logger } from 'winston';
 import {
   JobProcessor,
   PreparerBuilder,
-  RequiredTemplateValues,
   StageContext,
   TemplaterBuilder,
+  TemplaterValues,
   PublisherBuilder,
   parseLocationAnnotation,
   FilePreparer,
 } from '../scaffolder';
 import { CatalogEntityClient } from '../lib/catalog';
 import { validate, ValidatorResult } from 'jsonschema';
+import parseGitUrl from 'git-url-parse';
 
 export interface RouterOptions {
   preparers: PreparerBuilder;
@@ -109,8 +110,12 @@ export async function createRouter(
     })
     .post('/v1/jobs', async (req, res) => {
       const templateName: string = req.body.templateName;
-      const values: RequiredTemplateValues & Record<string, JsonValue> =
-        req.body.values;
+      const values: TemplaterValues = {
+        ...req.body.values,
+        destination: {
+          git: parseGitUrl(req.body.values.storePath),
+        },
+      };
 
       const template = await entityClient.findTemplate(templateName);
 
@@ -123,7 +128,6 @@ export async function createRouter(
         res.status(400).json({ errors: validationResult.errors });
         return;
       }
-
       const job = jobProcessor.create({
         entity: template,
         values,