diff --git a/.changeset/poor-otters-buy.md b/.changeset/poor-otters-buy.md new file mode 100644 index 0000000000..b60cbddc1b --- /dev/null +++ b/.changeset/poor-otters-buy.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Add new `fetch:template` action which handles the same responsibilities as `fetch:cookiecutter` without the external dependency on `cookiecutter`. For information on migrating from `fetch:cookiecutter` to `fetch:template`, see the [migration guide](https://backstage.io/docs/features/software-templates/builtin-actions#migrating-from-fetch-cookiecutter-to-fetch-template) in the docs. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index c095717def..e6af881f1e 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -169,6 +169,7 @@ nohoist nonces noop npm +nunjucks nvarchar nvm OAuth diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 988af89e47..fb17f4a434 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -654,7 +654,7 @@ spec: steps: - id: fetch-base name: Fetch Base - action: fetch:cookiecutter + action: fetch:template input: url: ./template values: diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index d185558949..2cdf6f1b35 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -51,7 +51,7 @@ spec: steps: - id: fetch-base name: Fetch Base - action: fetch:cookiecutter + action: fetch:template input: url: ./template values: diff --git a/docs/features/software-templates/builtin-actions.md b/docs/features/software-templates/builtin-actions.md index 14dc7ef861..af3ae97b21 100644 --- a/docs/features/software-templates/builtin-actions.md +++ b/docs/features/software-templates/builtin-actions.md @@ -14,3 +14,54 @@ Azure, GitLab and Bitbucket. A list of all registered actions can be found under `/create/actions`. For local development you should be able to reach them at `http://localhost:3000/create/actions`. + +### Migrating from `fetch:cookiecutter` to `fetch:template` + +The `fetch:template` action is a new action with a similar API to +`fetch:cookiecutter` but no dependency on `cookiecutter`. There are two options +for migrating templates that use `fetch:cookiecutter` to use `fetch:template`: + +#### Using `cookiecutterCompat` mode + +The new `fetch:template` action has a `cookiecutterCompat` flag which should +allow most templates built for `fetch:cookiecutter` to work without any changes. + +1. Update action name in `template.yaml`. The name should be changed from + `fetch:cookiecutter` to `fetch:template`. +2. Set `cookiecutterCompat` to `true` in the `fetch:template` step input in + `template.yaml`. + +```diff + steps: + - id: fetch-base + name: Fetch Base +- action: fetch:cookiecutter ++ action: fetch:template + input: + url: ./skeleton ++ cookiecutterCompat: true + values: +``` + +#### Manual migration + +If you prefer, you can manually migrate your templates to avoid the need for +enabling cookiecutter compatibility mode, which will result in slightly less +verbose template variables expressions. + +1. Update action name in `template.yaml`. The name should be changed from + `fetch:cookiecutter` to `fetch:template`. +2. Update variable syntax in file names and content. `fetch:cookiecutter` + expects variables to be enclosed in `{{` `}}` and prefixed with + `cookiecutter.`, while `fetch:template` expects variables to be enclosed in + `${{` `}}` and prefixed with `values.`. For example, a reference to variable + `myInputVariable` would need to be migrated from + `{{ cookiecutter.myInputVariable }}` to `${{ values.myInputVariable }}`. +3. Replace uses of `jsonify` with `dump`. The + [`jsonify` filter](https://cookiecutter.readthedocs.io/en/latest/advanced/template_extensions.html#jsonify-extension) + is built in to `cookiecutter`, and is not available by default when using + `fetch:template`. The + [`dump` filter](https://mozilla.github.io/nunjucks/templating.html#dump) is + the equivalent filter in nunjucks, so an expression like + `{{ cookiecutter.myAwesomeList | jsonify }}` should be migrated to + `${{ values.myAwesomeList | dump }}`. diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index 529376eb89..7b475f2fa1 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -62,7 +62,7 @@ spec: steps: - id: fetch-base name: Fetch Base - action: fetch:cookiecutter + action: fetch:template input: url: ./template values: @@ -289,8 +289,8 @@ template. These follow the same standard format: - id: fetch-base # A unique id for the step name: Fetch Base # A title displayed in the frontend if: '{{ parameters.name }}' # Optional condition, skip the step if not truthy - action: fetch:cookiecutter # an action to call - input: # input that is passed as arguments to the action handler + action: fetch:template # An action to call + input: # Input that is passed as arguments to the action handler url: ./template values: name: '{{ parameters.name }}' @@ -317,20 +317,20 @@ output: ### The templating syntax -You might have noticed in the examples that there are `{{ }}`, and these are a -`handlebars` templates for linking and glueing all these different parts of -`yaml` together. All the form inputs from the `parameters` section, when passed -to the steps will be available by using the template syntax -`{{ parameters.something }}`. This is great for passing the values from the form -into different steps and reusing these input variables. To pass arrays or -objects use the syntax `{{ json paramaters.something }}` where -`paramaters.something` is of type `object` or `array` in the `jsonSchema`, such -as the `nicknames` parameter in the previous example. +You might have noticed variables wrapped in `{{ }}` in the examples. These are +`handlebars` template strings for linking and gluing the different parts of the +template together. All the form inputs from the `parameters` section will be +available by using this template syntax (for example, +`{{ parameters.firstName }}` inserts the value of `firstName` from the +parameters). This is great for passing the values from the form into different +steps and reusing these input variables. To pass arrays or objects use the +`json` custom [helper](https://handlebarsjs.com/guide/expressions.html#helpers). +For example, `{{ json parameters.nicknames }}` will insert the result of calling +`JSON.stringify` on the value of the `nicknames` parameter. As you can see above in the `Outputs` section, `actions` and `steps` can also -output things. So you can grab that output by using -`steps.$stepId.output.$property`. +output things. You can grab that output using `steps.$stepId.output.$property`. You can read more about all the `inputs` and `outputs` defined in the actions in -code part of the `JSONSchema` or you can read more about our built in ones +code part of the `JSONSchema`, or you can read more about our built in ones [here](./builtin-actions.md). diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index e8e8697291..cca05b3333 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -74,6 +74,12 @@ export function createFetchPlainAction(options: { integrations: ScmIntegrations; }): TemplateAction; +// @public (undocumented) +export function createFetchTemplateAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; +}): TemplateAction; + // @public (undocumented) export const createFilesystemDeleteAction: () => TemplateAction; diff --git a/plugins/scaffolder-backend/fixtures/test-nested-template/public/react-logo192.png b/plugins/scaffolder-backend/fixtures/test-nested-template/public/react-logo192.png new file mode 100644 index 0000000000..fc44b0a379 Binary files /dev/null and b/plugins/scaffolder-backend/fixtures/test-nested-template/public/react-logo192.png differ diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 16e66eb2e8..7a280f7255 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -51,12 +51,14 @@ "globby": "^11.0.0", "handlebars": "^4.7.6", "helmet": "^4.0.0", + "isbinaryfile": "^4.0.8", "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", "knex": "^0.95.1", "lodash": "^4.17.21", "luxon": "^1.26.0", "morgan": "^1.10.0", + "nunjucks": "^3.2.3", "octokit-plugin-create-pull-request": "^3.9.3", "uuid": "^8.2.0", "winston": "^3.2.1", @@ -69,6 +71,7 @@ "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", "@types/mock-fs": "^4.13.0", + "@types/nunjucks": "^3.1.4", "@types/supertest": "^2.0.8", "jest-when": "^3.1.0", "mock-fs": "^4.13.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 08d7bbf824..6541625035 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -24,7 +24,11 @@ import { } from './catalog'; import { createDebugLogAction } from './debug'; -import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch'; +import { + createFetchCookiecutterAction, + createFetchPlainAction, + createFetchTemplateAction, +} from './fetch'; import { createFilesystemDeleteAction, createFilesystemRenameAction, @@ -62,6 +66,10 @@ export const createBuiltinActions = (options: { integrations, containerRunner, }), + createFetchTemplateAction({ + integrations, + reader, + }), createPublishGithubAction({ integrations, config, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts index 9b160b1dc3..557190dbbc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts @@ -26,7 +26,7 @@ export function createDebugLogAction() { return createTemplateAction<{ message?: string; listWorkspace?: boolean }>({ id: 'debug:log', description: - 'Writes a message into the log or list all files in the workspace.', + 'Writes a message into the log or lists all files in the workspace.', schema: { input: { type: 'object', diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 2bf22ce4fd..048b2bba64 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -136,7 +136,7 @@ export function createFetchCookiecutterAction(options: { }>({ id: 'fetch:cookiecutter', description: - 'Downloads a template from the given URL into the workspace, and runs cookiecutter on it.', + "Downloads a template from the given URL into the workspace, and runs cookiecutter on it. This action is deprecated in favor of 'fetch:template'. See https://backstage.io/docs/features/software-templates/builtin-actions#migrating-from-fetch-cookiecutter-to-fetch-template for more details.", schema: { input: { type: 'object', diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts index fa6d23c032..fea32df39c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -16,4 +16,5 @@ export { createFetchPlainAction } from './plain'; export { createFetchCookiecutterAction } from './cookiecutter'; +export { createFetchTemplateAction } from './template'; export { fetchContents } from './helpers'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts new file mode 100644 index 0000000000..bd2beae47a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts @@ -0,0 +1,295 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import { join as joinPath, resolve as resolvePath } from 'path'; +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { PassThrough } from 'stream'; +import { fetchContents } from './helpers'; +import { ActionContext, TemplateAction } from '../../types'; +import { createFetchTemplateAction, FetchTemplateInput } from './template'; + +jest.mock('./helpers', () => ({ + fetchContents: jest.fn(), +})); + +const aBinaryFile = fs.readFileSync( + resolvePath( + 'src', + '../fixtures/test-nested-template/public/react-logo192.png', + ), +); + +const mockFetchContents = fetchContents as jest.MockedFunction< + typeof fetchContents +>; + +describe('fetch:template', () => { + let action: TemplateAction; + + const workspacePath = os.tmpdir(); + const createTemporaryDirectory: jest.MockedFunction< + ActionContext['createTemporaryDirectory'] + > = jest.fn(() => + Promise.resolve( + joinPath(workspacePath, `${createTemporaryDirectory.mock.calls.length}`), + ), + ); + + const logger = getVoidLogger(); + + const mockContext = (inputPatch: Partial = {}) => ({ + baseUrl: 'base-url', + input: { + url: './skeleton', + targetPath: './target', + values: { + test: 'value', + }, + ...inputPatch, + }, + output: jest.fn(), + logStream: new PassThrough(), + logger, + workspacePath, + createTemporaryDirectory, + }); + + beforeEach(() => { + mockFs(); + + action = createFetchTemplateAction({ + reader: (Symbol('UrlReader') as unknown) as UrlReader, + integrations: (Symbol('Integrations') as unknown) as ScmIntegrations, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it(`returns a TemplateAction with the id 'fetch:template'`, () => { + expect(action.id).toEqual('fetch:template'); + }); + + describe('handler', () => { + it('throws if output directory is outside the workspace', async () => { + await expect(() => + action.handler(mockContext({ targetPath: '../' })), + ).rejects.toThrowError( + /relative path is not allowed to refer to a directory outside its parent/i, + ); + }); + + it('throws if copyWithoutRender parameter is not an array', async () => { + await expect(() => + action.handler( + mockContext({ copyWithoutRender: ('abc' as unknown) as string[] }), + ), + ).rejects.toThrowError(/copyWithoutRender must be an array/i); + }); + + describe('with valid input', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + 'empty-dir-${{ values.count }}': {}, + 'static.txt': 'static content', + '${{ values.name }}.txt': 'static content', + subdir: { + 'templated-content.txt': + '${{ values.name }}: ${{ values.count }}', + }, + '.${{ values.name }}': '${{ values.itemList | dump }}', + 'a-binary-file.png': aBinaryFile, + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('uses fetchContents to retrieve the template content', () => { + expect(mockFetchContents).toHaveBeenCalledWith( + expect.objectContaining({ + baseUrl: context.baseUrl, + fetchUrl: context.input.url, + }), + ); + }); + + it('copies files with no templating in names or content successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with templated names successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with templated content successfully', async () => { + await expect( + fs.readFile( + `${workspacePath}/target/subdir/templated-content.txt`, + 'utf-8', + ), + ).resolves.toEqual('test-project: 1234'); + }); + + it('processes dotfiles', async () => { + await expect( + fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'), + ).resolves.toEqual('["first","second","third"]'); + }); + + it('copies empty directories', async () => { + await expect( + fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'), + ).resolves.toEqual([]); + }); + + it('copies binary files as-is without processing them', async () => { + await expect( + fs.readFile(`${workspacePath}/target/a-binary-file.png`), + ).resolves.toEqual(aBinaryFile); + }); + }); + + describe('copyWithoutRender', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + }, + copyWithoutRender: ['.unprocessed'], + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + processed: { + 'templated-content-${{ values.name }}.txt': + '${{ values.count }}', + }, + '.unprocessed': { + 'templated-content-${{ values.name }}.txt': + '${{ values.count }}', + }, + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('ignores template syntax in files matched in copyWithoutRender', async () => { + await expect( + fs.readFile( + `${workspacePath}/target/.unprocessed/templated-content-\${{ values.name }}.txt`, + 'utf-8', + ), + ).resolves.toEqual('${{ values.count }}'); + }); + + it('processes files not matched in copyWithoutRender', async () => { + await expect( + fs.readFile( + `${workspacePath}/target/processed/templated-content-test-project.txt`, + 'utf-8', + ), + ).resolves.toEqual('1234'); + }); + }); + }); + + describe('cookiecutter compatibility mode', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + cookiecutterCompat: true, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + '{{ cookiecutter.name }}.txt': 'static content', + subdir: { + 'templated-content.txt': + '{{ cookiecutter.name }}: {{ cookiecutter.count }}', + }, + '{{ cookiecutter.name }}.json': + '{{ cookiecutter.itemList | jsonify }}', + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('copies files with cookiecutter-style templated names successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with cookiecutter-style templated content successfully', async () => { + await expect( + fs.readFile( + `${workspacePath}/target/subdir/templated-content.txt`, + 'utf-8', + ), + ).resolves.toEqual('test-project: 1234'); + }); + + it('includes the jsonify filter', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.json`, 'utf-8'), + ).resolves.toEqual('["first","second","third"]'); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts new file mode 100644 index 0000000000..87e67f76be --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts @@ -0,0 +1,239 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common'; +import { InputError } from '@backstage/errors'; +import { ScmIntegrations } from '@backstage/integration'; +import { fetchContents } from './helpers'; +import { createTemplateAction } from '../../createTemplateAction'; +import globby from 'globby'; +import nunjucks from 'nunjucks'; +import fs from 'fs-extra'; +import { isBinaryFile } from 'isbinaryfile'; + +/* + * Maximise compatibility with Jinja (and therefore cookiecutter) + * using nunjucks jinja compat mode. Since this method mutates + * the global nunjucks instance, we can't enable this per-template, + * or only for templates with cookiecutter compat enabled, so the + * next best option is to explicitly enable it globally and allow + * folks to rely on jinja compatibility behaviour in fetch:template + * templates if they wish. + * + * cf. https://mozilla.github.io/nunjucks/api.html#installjinjacompat + */ +nunjucks.installJinjaCompat(); + +export type FetchTemplateInput = { + url: string; + targetPath?: string; + values: any; + copyWithoutRender?: string[]; + cookiecutterCompat?: boolean; +}; + +export function createFetchTemplateAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; +}) { + const { reader, integrations } = options; + + return createTemplateAction({ + id: 'fetch:template', + description: + "Downloads a skeleton, templates variables into file and directory names and content, and places the result in the workspace, or optionally in a subdirectory specified by the 'targetPath' input option.", + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to. Defaults to the working directory root.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to the templating engine', + type: 'object', + }, + copyWithoutRender: { + title: 'Copy Without Render', + description: + 'An array of glob patterns. Any files or directories which match are copied without being processed as templates.', + type: 'array', + items: { + type: 'string', + }, + }, + cookiecutterCompat: { + title: 'Cookiecutter compatibility mode', + description: + 'Enable features to maximise compatibility with templates built for fetch:cookiecutter', + type: 'boolean', + }, + }, + }, + }, + async handler(ctx) { + ctx.logger.info('Fetching template content from remote URL'); + + const workDir = await ctx.createTemporaryDirectory(); + const templateDir = resolvePath(workDir, 'template'); + + const targetPath = ctx.input.targetPath ?? './'; + const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath); + + if ( + ctx.input.copyWithoutRender && + !Array.isArray(ctx.input.copyWithoutRender) + ) { + throw new InputError( + 'Fetch action input copyWithoutRender must be an Array', + ); + } + + await fetchContents({ + reader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.input.url, + outputPath: templateDir, + }); + + ctx.logger.info('Listing files and directories in template'); + const allEntriesInTemplate = await globby(`**/*`, { + cwd: templateDir, + dot: true, + onlyFiles: false, + markDirectories: true, + }); + + const nonTemplatedEntries = new Set( + ( + await Promise.all( + (ctx.input.copyWithoutRender || []).map(pattern => + globby(pattern, { + cwd: templateDir, + dot: true, + onlyFiles: false, + markDirectories: true, + }), + ), + ) + ).flat(), + ); + + // Create a templater + const templater = nunjucks.configure({ + ...(ctx.input.cookiecutterCompat + ? {} + : { + tags: { + // TODO(mtlewis/orkohunter): Document Why we are changing the literals? Not here, but on scaffolder docs. ADR? + variableStart: '${{', + variableEnd: '}}', + }, + }), + // We don't want this builtin auto-escaping, since uses HTML escape sequences + // like `"` - the correct way to escape strings in our case depends on + // the file type. + autoescape: false, + }); + + if (ctx.input.cookiecutterCompat) { + // The "jsonify" filter built into cookiecutter is common + // in fetch:cookiecutter templates, so when compat mode + // is enabled we alias the "dump" filter from nunjucks as + // jsonify. Dump accepts an optional `spaces` parameter + // which enables indented output, but when this parameter + // is not supplied it works identically to jsonify. + // + // cf. https://cookiecutter.readthedocs.io/en/latest/advanced/template_extensions.html?highlight=jsonify#jsonify-extension + // cf. https://mozilla.github.io/nunjucks/templating.html#dump + templater.addFilter('jsonify', templater.getFilter('dump')); + } + + // Cookiecutter prefixes all parameters in templates with + // `cookiecutter.`. To replicate this, we wrap our parameters + // in an object with a `cookiecutter` property when compat + // mode is enabled. + const { cookiecutterCompat, values } = ctx.input; + const context = { + [cookiecutterCompat ? 'cookiecutter' : 'values']: values, + }; + + ctx.logger.info( + `Processing ${allEntriesInTemplate.length} template files/directories with input values`, + ctx.input.values, + ); + + for (const location of allEntriesInTemplate) { + const shouldCopyWithoutRender = nonTemplatedEntries.has(location); + + const outputPath = resolvePath( + outputDir, + shouldCopyWithoutRender + ? location + : templater.renderString(location, context), + ); + + if (shouldCopyWithoutRender) { + ctx.logger.info( + `Copying file/directory ${location} without processing since it matches a pattern in "copyWithoutRender".`, + ); + } + + if (location.endsWith('/')) { + ctx.logger.info( + `Writing directory ${location} to template output path.`, + ); + await fs.ensureDir(outputPath); + } else { + const inputFilePath = resolvePath(templateDir, location); + + if (await isBinaryFile(inputFilePath)) { + ctx.logger.info( + `Copying binary file ${location} to template output path.`, + ); + await fs.copy(inputFilePath, outputPath); + } else { + ctx.logger.info( + `Writing file ${location} to template output path.`, + ); + const inputFileContents = await fs.readFile(inputFilePath, 'utf-8'); + await fs.outputFile( + outputPath, + shouldCopyWithoutRender + ? inputFileContents + : templater.renderString(inputFileContents, context), + ); + } + } + } + + ctx.logger.info(`Template result written to ${outputDir}`); + }, + }); +} diff --git a/yarn.lock b/yarn.lock index 917a6f365c..138b737d0b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6255,6 +6255,11 @@ resolved "https://registry.npmjs.org/@types/npmlog/-/npmlog-4.1.2.tgz#d070fe6a6b78755d1092a3dc492d34c3d8f871c4" integrity sha512-4QQmOF5KlwfxJ5IGXFIudkeLCdMABz03RcUXu+LCb24zmln8QW6aDjuGl4d4XPVLf2j+FnjelHTP7dvceAFbhA== +"@types/nunjucks@^3.1.4": + version "3.1.4" + resolved "https://registry.npmjs.org/@types/nunjucks/-/nunjucks-3.1.4.tgz#2f80e2fa5e7982b2131201d0b4474ab4d03d9de1" + integrity sha512-cR65PLlHKW/qxxj840dbNb3ICO+iAVQzaNKJ8TcKOVKFi+QcAkhw9SCY8VFAyU41SmJMs+2nrIN2JGhX+jYb7A== + "@types/oauth@*": version "0.9.1" resolved "https://registry.npmjs.org/@types/oauth/-/oauth-0.9.1.tgz#e17221e7f7936b0459ae7d006255dff61adca305" @@ -7143,6 +7148,11 @@ JSONStream@^1.0.4: jsonparse "^1.2.0" through ">=2.2.7 <3" +a-sync-waterfall@^1.0.0: + version "1.0.1" + resolved "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz#75b6b6aa72598b497a125e7a2770f14f4c8a1fa7" + integrity sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA== + abab@^2.0.0, abab@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz#623e2075e02eb2d3f2475e49f99c91846467907a" @@ -7863,7 +7873,7 @@ arrify@^2.0.0, arrify@^2.0.1: resolved "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== -asap@^2.0.0, asap@~2.0.3: +asap@^2.0.0, asap@^2.0.3, asap@~2.0.3: version "2.0.6" resolved "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= @@ -16006,6 +16016,11 @@ isarray@^2.0.5: resolved "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== +isbinaryfile@^4.0.8: + version "4.0.8" + resolved "https://registry.npmjs.org/isbinaryfile/-/isbinaryfile-4.0.8.tgz#5d34b94865bd4946633ecc78a026fc76c5b11fcf" + integrity sha512-53h6XFniq77YdW+spoRrebh0mnmTxRPTlcuIArO57lmMdq4uBKFKaeTjnb92oYWrSn/LVL+LT+Hap2tFQj8V+w== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -19552,6 +19567,15 @@ number-is-nan@^1.0.0: resolved "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= +nunjucks@^3.2.3: + version "3.2.3" + resolved "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.3.tgz#1b33615247290e94e28263b5d855ece765648a31" + integrity sha512-psb6xjLj47+fE76JdZwskvwG4MYsQKXUtMsPh6U0YMvmyjRtKRFcxnlXGWglNybtNTNVmGdp94K62/+NjF5FDQ== + dependencies: + a-sync-waterfall "^1.0.0" + asap "^2.0.3" + commander "^5.1.0" + nwsapi@^2.0.7, nwsapi@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7"