From b1dc16baa14863d974aa627067b9f19462831ce4 Mon Sep 17 00:00:00 2001 From: Vinay P Date: Mon, 15 Mar 2021 12:45:06 +0000 Subject: [PATCH 1/4] copyWithoutRender, extensions inputs for fetch:cookiecutter as discussed in this GitHub issue: https://github.com/backstage/backstage/issues/4920 Signed-off-by: Vinay P --- .../builtin/fetch/cookiecutter.test.ts | 50 +++++++++++++++++++ .../actions/builtin/fetch/cookiecutter.ts | 39 ++++++++++++++- .../stages/templater/cookiecutter.test.ts | 23 +++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts index 0de7d94fda..940c0f1e08 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -108,6 +108,56 @@ describe('fetch:cookiecutter', () => { }); }); + it('should execute the cookiecutter templater with optional inputs if they are present and valid', async () => { + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + copyWithoutRender: ['goreleaser.yml'], + extensions: [ + 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', + ], + }, + }); + + expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ + workspacePath: mockTmpDir, + dockerClient: mockDockerClient, + logStream: mockContext.logStream, + values: { + ...mockContext.input.values, + _copy_without_render: ['goreleaser.yml'], + _extensions: [ + 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', + ], + }, + }); + }); + + it('should throw if copyWithoutRender is not an Array', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + ...mockContext.input, + copyWithoutRender: 'xyz', + }, + }), + ).rejects.toThrow(/copyWithoutRender must be an Array/); + }); + + it('should throw if extensions is not an Array', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + ...mockContext.input, + extensions: 'xyz', + }, + }), + ).rejects.toThrow(/extensions must be an Array/); + }); + it('should throw if there is no cookiecutter templater initialized', async () => { const templatersWithoutCookiecutter = new Templaters(); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 814146cc50..404d134f6d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -37,6 +37,8 @@ export function createFetchCookiecutterAction(options: { url: string; targetPath?: string; values: JsonObject; + copyWithoutRender?: string[]; + extensions?: string[]; }>({ id: 'fetch:cookiecutter', description: @@ -63,6 +65,24 @@ export function createFetchCookiecutterAction(options: { description: 'Values to pass on to cookiecutter for templating', type: 'object', }, + copyWithoutRender: { + title: 'Copy Without Render', + description: + 'Avoid rendering directories and files in the template', + type: 'array', + items: { + type: 'string', + }, + }, + extensions: { + title: 'Template Extensions', + description: + "Jinja2 extensions to add filters, tests, globals or extend the parser. Extensions must be installed in the container or on the host where Cookiecutter executes. See the contrib directory in Backstage's repo for more information", + type: 'array', + items: { + type: 'string', + }, + }, }, }, }, @@ -76,6 +96,18 @@ export function createFetchCookiecutterAction(options: { ); const resultDir = resolvePath(workDir, 'result'); + if ( + ctx.input.copyWithoutRender && + !Array.isArray(ctx.input.copyWithoutRender) + ) { + throw new InputError( + 'Fetch action input copyWithoutRender must be an Array', + ); + } + if (ctx.input.extensions && !Array.isArray(ctx.input.extensions)) { + throw new InputError('Fetch action input extensions must be an Array'); + } + await fetchContents({ reader, integrations, @@ -85,13 +117,18 @@ export function createFetchCookiecutterAction(options: { }); const cookiecutter = templaters.get('cookiecutter'); + const values = { + ...(ctx.input.values as TemplaterValues), + _copy_without_render: ctx.input.copyWithoutRender, + _extensions: ctx.input.extensions, + }; // Will execute the template in ./template and put the result in ./result await cookiecutter.run({ workspacePath: workDir, dockerClient, logStream: ctx.logStream, - values: ctx.input.values as TemplaterValues, + values, }); // Finally move the template result into the task workspace diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts index 763572fda5..7806f089ff 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts @@ -168,6 +168,29 @@ describe('CookieCutter Templater', () => { }); }); + it('should run the docker container mentioned in configs, overriding the default', async () => { + const values = { + owner: 'blobby', + storePath: 'https://github.com/org/repo', + imageName: 'foo/cookiecutter-image-with-extensions', + }; + + jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); + + const templater = new CookieCutter(); + await templater.run({ + workspacePath: 'tempdir', + values, + dockerClient: mockDocker, + }); + + expect(runDockerContainer).toHaveBeenCalledWith( + expect.objectContaining({ + imageName: 'foo/cookiecutter-image-with-extensions', + }), + ); + }); + it('should pass through the streamer to the run docker helper', async () => { const stream = new PassThrough(); From 91e87c0554055ce830b4a7e79ec97c6289e9e452 Mon Sep 17 00:00:00 2001 From: Vinay P Date: Mon, 15 Mar 2021 13:35:27 +0000 Subject: [PATCH 2/4] Add changeset Signed-off-by: Vinay P --- .changeset/orange-flowers-act.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/orange-flowers-act.md diff --git a/.changeset/orange-flowers-act.md b/.changeset/orange-flowers-act.md new file mode 100644 index 0000000000..ba83f8401e --- /dev/null +++ b/.changeset/orange-flowers-act.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Add inputs for action `fetch:cookiecutter`: copyWithoutRender, extensions, imageName From 3105b46afa8088670160f4f68e4a5de270c59741 Mon Sep 17 00:00:00 2001 From: Vinay P Date: Thu, 18 Mar 2021 12:19:28 +0000 Subject: [PATCH 3/4] Document use of Jinja2 extensions with fetch:cookiecutter Signed-off-by: Vinay P --- .../Dockerfile | 11 ++++ .../README.md | 63 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 contrib/docker/cookiecutter-with-jinja2-extensions/Dockerfile create mode 100644 contrib/docker/cookiecutter-with-jinja2-extensions/README.md diff --git a/contrib/docker/cookiecutter-with-jinja2-extensions/Dockerfile b/contrib/docker/cookiecutter-with-jinja2-extensions/Dockerfile new file mode 100644 index 0000000000..4f5f72bdf8 --- /dev/null +++ b/contrib/docker/cookiecutter-with-jinja2-extensions/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine:3.7 + +RUN apk add --update \ + git \ + python \ + python-dev \ + py-pip \ + g++ && \ + pip install cookiecutter jinja2_custom_filters_extension && \ + apk del g++ py-pip python-dev && \ + rm -rf /var/cache/apk/* diff --git a/contrib/docker/cookiecutter-with-jinja2-extensions/README.md b/contrib/docker/cookiecutter-with-jinja2-extensions/README.md new file mode 100644 index 0000000000..6b990f77a3 --- /dev/null +++ b/contrib/docker/cookiecutter-with-jinja2-extensions/README.md @@ -0,0 +1,63 @@ +# Using Cookiecutter with Jinja2 extensions + +Jinja2 extensions can be used with the scaffolder's `fetch:cookiecutter` built-in action to add filters, tests, or to extend the parser. + +Using Cookiecutter extensions is a two-step process: + +- [Installing the extension](#installing-the-extension), and +- [Instructing Cookiecutter to use the extension](#instructing-cookiecutter-to-use-the-extension) + +### Installing the extension + +This step depends on how the scaffolder is setup to use Cookiecutter: + +- Using a local Cookiecutter, or +- Using a Cookiecutter Docker image, e.g. [spotify/backstage-cookiecutter](https://github.com/backstage/backstage/blob/37e35b91/plugins/scaffolder-backend/scripts/Cookiecutter.dockerfile). + +Say we want to install [`jinja2_custom_filters_extension`](https://pypi.org/project/jinja2-custom-filters-extension/) to use the `upper_case_first_letter` filter in a Cookiecutter template. + +#### Using a local Cookiecutter + +The scaffolder is able to execute a locally installed Cookiecutter, and doesn't pull a Docker image in that case. If that's your setup, just ensure that the Jinja2 extensions, available via `pip` are installed alongside Cookiecutter, e.g. if Cookiecutter is baked into a custom Backstage image using `pip` and a `requirements.txt`: + +In the custom Backstage Dockerfile: + +```Dockerfile +... +RUN pip3 install -r requirements.txt +... +``` + +In requirements.txt: + +```python +... +cookiecutter==1.7.2 +jinja2_custom_filters_extension==0.0.2 +... +``` + +#### Using a Cookiecutter Docker image + +If the scaffolder doesn't find a local Cookiecutter, it pulls down the `spotify/backstage-cookiecutter` image. You can create a custom Cookiecutter image based on that, install extensions into it, and use that instead. See for example, the [`Dockerfile`](./Dockerfile) in this directory. + +**Note:** The [`imageName`](https://github.com/vinayvinay/backstage/blob/37e35b91/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts#L77) that gets pulled isn't currently configurable. So, for the time being, you might have to customise the `scaffolder-backend` plugin to fetch your customised Cookiecutter Docker image. + +### Instructing Cookiecutter to use the extension + +Cookiecutter enables extensions mentioned in `cookiecutter.json`. `fetch:cookiecutter` generates a `cookiecutter.json`, deriving its values from `inputs` to `fetch:cookiecutter` in the scaffolder [Template](https://backstage.io/docs/features/software-templates/writing-templates), as: + +```yaml +steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + extensions: + - jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension + url: https://github.com/spotify/cookiecutter-golang + values: + name: '{{ parameters.name }}' +``` + +Cookiecutter enables a few extensions by default. See the official Cookiecutter documentation for [Template Extensions](https://cookiecutter.readthedocs.io/en/1.7.2/advanced/template_extensions.html) for a list of such extensions, and more information. From 673c4f7d5aef19beef2772baa8415130e64534eb Mon Sep 17 00:00:00 2001 From: Vinay P Date: Thu, 18 Mar 2021 13:23:30 +0000 Subject: [PATCH 4/4] Allow overriding cookiecutter's docker imageName .. to allow running custom-built images that have cookiecutter installed alongwith extensions, for example. https://cookiecutter.readthedocs.io/en/1.7.2/advanced/template_extensions.html related to: https://github.com/backstage/backstage/pull/4956#discussion_r596846365 Signed-off-by: Vinay P --- .../cookiecutter-with-jinja2-extensions/README.md | 14 ++++++++++++-- .../actions/builtin/fetch/cookiecutter.test.ts | 2 ++ .../actions/builtin/fetch/cookiecutter.ts | 8 ++++++++ .../scaffolder/stages/templater/cookiecutter.ts | 5 +++-- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/contrib/docker/cookiecutter-with-jinja2-extensions/README.md b/contrib/docker/cookiecutter-with-jinja2-extensions/README.md index 6b990f77a3..470694513a 100644 --- a/contrib/docker/cookiecutter-with-jinja2-extensions/README.md +++ b/contrib/docker/cookiecutter-with-jinja2-extensions/README.md @@ -39,9 +39,19 @@ jinja2_custom_filters_extension==0.0.2 #### Using a Cookiecutter Docker image -If the scaffolder doesn't find a local Cookiecutter, it pulls down the `spotify/backstage-cookiecutter` image. You can create a custom Cookiecutter image based on that, install extensions into it, and use that instead. See for example, the [`Dockerfile`](./Dockerfile) in this directory. +If the scaffolder doesn't find a local Cookiecutter, it pulls down the `spotify/backstage-cookiecutter` image. You can create a custom Cookiecutter image based on that, install extensions into it, and specify that customised image as an input `imageName` to the `fetch:cookiecutter` action: -**Note:** The [`imageName`](https://github.com/vinayvinay/backstage/blob/37e35b91/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts#L77) that gets pulled isn't currently configurable. So, for the time being, you might have to customise the `scaffolder-backend` plugin to fetch your customised Cookiecutter Docker image. +```yaml +steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: https://github.com/spotify/cookiecutter-golang + imageName: 'foo/custom-built-cookiecutter-image-with-extensions' +``` + +See for example, the [`Dockerfile`](./Dockerfile) in this directory. ### Instructing Cookiecutter to use the extension diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts index 940c0f1e08..628107155d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -117,6 +117,7 @@ describe('fetch:cookiecutter', () => { extensions: [ 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', ], + imageName: 'foo/cookiecutter-image-with-extensions', }, }); @@ -130,6 +131,7 @@ describe('fetch:cookiecutter', () => { _extensions: [ 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', ], + imageName: 'foo/cookiecutter-image-with-extensions', }, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 404d134f6d..fca791c7bd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -39,6 +39,7 @@ export function createFetchCookiecutterAction(options: { values: JsonObject; copyWithoutRender?: string[]; extensions?: string[]; + imageName?: string; }>({ id: 'fetch:cookiecutter', description: @@ -83,6 +84,12 @@ export function createFetchCookiecutterAction(options: { type: 'string', }, }, + imageName: { + title: 'Cookiecutter Docker image', + description: + "Specify a custom Docker image to run cookiecutter, to override the default: 'spotify/backstage-cookiecutter'. This can be used to execute cookiecutter with Template Extensions. Used only when a local cookiecutter is not found.", + type: 'string', + }, }, }, }, @@ -121,6 +128,7 @@ export function createFetchCookiecutterAction(options: { ...(ctx.input.values as TemplaterValues), _copy_without_render: ctx.input.copyWithoutRender, _extensions: ctx.input.extensions, + imageName: ctx.input.imageName, }; // Will execute the template in ./template and put the result in ./result diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index f0fbf4a708..e60c5efb4d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -52,9 +52,10 @@ export class CookieCutter implements TemplaterBase { // First lets grab the default cookiecutter.json file const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir); + const { imageName, ...valuesForCookieCutterJson } = values; const cookieInfo = { ...cookieCutterJson, - ...values, + ...valuesForCookieCutterJson, }; await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); @@ -74,7 +75,7 @@ export class CookieCutter implements TemplaterBase { }); } else { await runDockerContainer({ - imageName: 'spotify/backstage-cookiecutter', + imageName: imageName || 'spotify/backstage-cookiecutter', args: [ 'cookiecutter', '--no-input',