Merge pull request #4956 from vinayvinay/vinay-fetch-cookiecutter-more-inputs

Adds inputs for fetch:cookiecutter action: copyWithoutRender, extensions, imageName

.changeset/orange-flowers-act.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Add inputs for action `fetch:cookiecutter`: copyWithoutRender, extensions, imageName

contrib/docker/cookiecutter-with-jinja2-extensions/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.7
+
+RUN apk add --update \
+	git \
+	python \
+	python-dev \
+	py-pip \
+	g++ && \
+	pip install cookiecutter jinja2_custom_filters_extension && \
+	apk del g++ py-pip python-dev && \
+	rm -rf /var/cache/apk/*

contrib/docker/cookiecutter-with-jinja2-extensions/README.md

@@ -0,0 +1,73 @@
+# Using Cookiecutter with Jinja2 extensions
+
+Jinja2 extensions can be used with the scaffolder's `fetch:cookiecutter` built-in action to add filters, tests, or to extend the parser.
+
+Using Cookiecutter extensions is a two-step process:
+
+- [Installing the extension](#installing-the-extension), and
+- [Instructing Cookiecutter to use the extension](#instructing-cookiecutter-to-use-the-extension)
+
+### Installing the extension
+
+This step depends on how the scaffolder is setup to use Cookiecutter:
+
+- Using a local Cookiecutter, or
+- Using a Cookiecutter Docker image, e.g. [spotify/backstage-cookiecutter](https://github.com/backstage/backstage/blob/37e35b91/plugins/scaffolder-backend/scripts/Cookiecutter.dockerfile).
+
+Say we want to install [`jinja2_custom_filters_extension`](https://pypi.org/project/jinja2-custom-filters-extension/) to use the `upper_case_first_letter` filter in a Cookiecutter template.
+
+#### Using a local Cookiecutter
+
+The scaffolder is able to execute a locally installed Cookiecutter, and doesn't pull a Docker image in that case. If that's your setup, just ensure that the Jinja2 extensions, available via `pip` are installed alongside Cookiecutter, e.g. if Cookiecutter is baked into a custom Backstage image using `pip` and a `requirements.txt`:
+
+In the custom Backstage Dockerfile:
+
+```Dockerfile
+...
+RUN pip3 install -r requirements.txt
+...
+```
+
+In requirements.txt:
+
+```python
+...
+cookiecutter==1.7.2
+jinja2_custom_filters_extension==0.0.2
+...
+```
+
+#### Using a Cookiecutter Docker image
+
+If the scaffolder doesn't find a local Cookiecutter, it pulls down the `spotify/backstage-cookiecutter` image. You can create a custom Cookiecutter image based on that, install extensions into it, and specify that customised image as an input `imageName` to the `fetch:cookiecutter` action:
+
+```yaml
+steps:
+  - id: fetch-base
+    name: Fetch Base
+    action: fetch:cookiecutter
+    input:
+      url: https://github.com/spotify/cookiecutter-golang
+      imageName: 'foo/custom-built-cookiecutter-image-with-extensions'
+```
+
+See for example, the [`Dockerfile`](./Dockerfile) in this directory.
+
+### Instructing Cookiecutter to use the extension
+
+Cookiecutter enables extensions mentioned in `cookiecutter.json`. `fetch:cookiecutter` generates a `cookiecutter.json`, deriving its values from `inputs` to `fetch:cookiecutter` in the scaffolder [Template](https://backstage.io/docs/features/software-templates/writing-templates), as:
+
+```yaml
+steps:
+  - id: fetch-base
+    name: Fetch Base
+    action: fetch:cookiecutter
+    input:
+      extensions:
+        - jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension
+      url: https://github.com/spotify/cookiecutter-golang
+      values:
+        name: '{{ parameters.name }}'
+```
+
+Cookiecutter enables a few extensions by default. See the official Cookiecutter documentation for [Template Extensions](https://cookiecutter.readthedocs.io/en/1.7.2/advanced/template_extensions.html) for a list of such extensions, and more information.

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts

@@ -108,6 +108,58 @@ describe('fetch:cookiecutter', () => {
     });
   });
 
+  it('should execute the cookiecutter templater with optional inputs if they are present and valid', async () => {
+    await action.handler({
+      ...mockContext,
+      input: {
+        ...mockContext.input,
+        copyWithoutRender: ['goreleaser.yml'],
+        extensions: [
+          'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension',
+        ],
+        imageName: 'foo/cookiecutter-image-with-extensions',
+      },
+    });
+
+    expect(cookiecutterTemplater.run).toHaveBeenCalledWith({
+      workspacePath: mockTmpDir,
+      dockerClient: mockDockerClient,
+      logStream: mockContext.logStream,
+      values: {
+        ...mockContext.input.values,
+        _copy_without_render: ['goreleaser.yml'],
+        _extensions: [
+          'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension',
+        ],
+        imageName: 'foo/cookiecutter-image-with-extensions',
+      },
+    });
+  });
+
+  it('should throw if copyWithoutRender is not an Array', async () => {
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: {
+          ...mockContext.input,
+          copyWithoutRender: 'xyz',
+        },
+      }),
+    ).rejects.toThrow(/copyWithoutRender must be an Array/);
+  });
+
+  it('should throw if extensions is not an Array', async () => {
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: {
+          ...mockContext.input,
+          extensions: 'xyz',
+        },
+      }),
+    ).rejects.toThrow(/extensions must be an Array/);
+  });
+
   it('should throw if there is no cookiecutter templater initialized', async () => {
     const templatersWithoutCookiecutter = new Templaters();
 

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts

@@ -37,6 +37,9 @@ export function createFetchCookiecutterAction(options: {
     url: string;
     targetPath?: string;
     values: JsonObject;
+    copyWithoutRender?: string[];
+    extensions?: string[];
+    imageName?: string;
   }>({
     id: 'fetch:cookiecutter',
     description:
@@ -63,6 +66,30 @@ export function createFetchCookiecutterAction(options: {
             description: 'Values to pass on to cookiecutter for templating',
             type: 'object',
           },
+          copyWithoutRender: {
+            title: 'Copy Without Render',
+            description:
+              'Avoid rendering directories and files in the template',
+            type: 'array',
+            items: {
+              type: 'string',
+            },
+          },
+          extensions: {
+            title: 'Template Extensions',
+            description:
+              "Jinja2 extensions to add filters, tests, globals or extend the parser. Extensions must be installed in the container or on the host where Cookiecutter executes. See the contrib directory in Backstage's repo for more information",
+            type: 'array',
+            items: {
+              type: 'string',
+            },
+          },
+          imageName: {
+            title: 'Cookiecutter Docker image',
+            description:
+              "Specify a custom Docker image to run cookiecutter, to override the default: 'spotify/backstage-cookiecutter'. This can be used to execute cookiecutter with Template Extensions. Used only when a local cookiecutter is not found.",
+            type: 'string',
+          },
         },
       },
     },
@@ -76,6 +103,18 @@ export function createFetchCookiecutterAction(options: {
       );
       const resultDir = resolvePath(workDir, 'result');
 
+      if (
+        ctx.input.copyWithoutRender &&
+        !Array.isArray(ctx.input.copyWithoutRender)
+      ) {
+        throw new InputError(
+          'Fetch action input copyWithoutRender must be an Array',
+        );
+      }
+      if (ctx.input.extensions && !Array.isArray(ctx.input.extensions)) {
+        throw new InputError('Fetch action input extensions must be an Array');
+      }
+
       await fetchContents({
         reader,
         integrations,
@@ -85,13 +124,19 @@ export function createFetchCookiecutterAction(options: {
       });
 
       const cookiecutter = templaters.get('cookiecutter');
+      const values = {
+        ...(ctx.input.values as TemplaterValues),
+        _copy_without_render: ctx.input.copyWithoutRender,
+        _extensions: ctx.input.extensions,
+        imageName: ctx.input.imageName,
+      };
 
       // Will execute the template in ./template and put the result in ./result
       await cookiecutter.run({
         workspacePath: workDir,
         dockerClient,
         logStream: ctx.logStream,
-        values: ctx.input.values as TemplaterValues,
+        values,
       });
 
       // Finally move the template result into the task workspace

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts

@@ -168,6 +168,29 @@ describe('CookieCutter Templater', () => {
     });
   });
 
+  it('should run the docker container mentioned in configs, overriding the default', async () => {
+    const values = {
+      owner: 'blobby',
+      storePath: 'https://github.com/org/repo',
+      imageName: 'foo/cookiecutter-image-with-extensions',
+    };
+
+    jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any);
+
+    const templater = new CookieCutter();
+    await templater.run({
+      workspacePath: 'tempdir',
+      values,
+      dockerClient: mockDocker,
+    });
+
+    expect(runDockerContainer).toHaveBeenCalledWith(
+      expect.objectContaining({
+        imageName: 'foo/cookiecutter-image-with-extensions',
+      }),
+    );
+  });
+
   it('should pass through the streamer to the run docker helper', async () => {
     const stream = new PassThrough();
 

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts

@@ -52,9 +52,10 @@ export class CookieCutter implements TemplaterBase {
     // First lets grab the default cookiecutter.json file
     const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir);
 
+    const { imageName, ...valuesForCookieCutterJson } = values;
     const cookieInfo = {
       ...cookieCutterJson,
-      ...values,
+      ...valuesForCookieCutterJson,
     };
 
     await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo);
@@ -74,7 +75,7 @@ export class CookieCutter implements TemplaterBase {
       });
     } else {
       await runDockerContainer({
-        imageName: 'spotify/backstage-cookiecutter',
+        imageName: imageName || 'spotify/backstage-cookiecutter',
         args: [
           'cookiecutter',
           '--no-input',