diff --git a/.changeset/heavy-monkeys-drum.md b/.changeset/heavy-monkeys-drum.md index 0b03020ca9..24124ad273 100644 --- a/.changeset/heavy-monkeys-drum.md +++ b/.changeset/heavy-monkeys-drum.md @@ -2,8 +2,10 @@ '@backstage/plugin-scaffolder-backend': minor --- -Add `fetch:partial` templating action. +Add partial templating to `fetch:template` action. -- For all files with extension `.njk`, apply templating logic and strip extension. The extension is configurable. -- All other files get copied. -- All output paths are subject to applying templating logic. +If an `extension` input is given, only files with that extension get their content processed. If `extension` is `true`, the `.njk` extension is used. The `extension` input is incompatible with both `cookiecutterCompat` and `copyWithoutRender`. + +All other files get copied. + +All output paths are subject to applying templating logic. diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 0d1b636049..5a16ef9a55 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -24,11 +24,7 @@ import { } from './catalog'; import { createDebugLogAction } from './debug'; -import { - createFetchPartialAction, - createFetchPlainAction, - createFetchTemplateAction, -} from './fetch'; +import { createFetchPlainAction, createFetchTemplateAction } from './fetch'; import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-backend-module-cookiecutter'; import { createFilesystemDeleteAction, @@ -67,10 +63,6 @@ export const createBuiltinActions = (options: { integrations, reader, }), - createFetchPartialAction({ - integrations, - reader, - }), createPublishGithubAction({ integrations, config, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts index 260e85af77..c9a16c274e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -export { createFetchPartialAction } from './partial'; export { createFetchPlainAction } from './plain'; export { createFetchTemplateAction } from './template'; export { fetchContents } from './helpers'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts deleted file mode 100644 index 3b5728111b..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import os from 'os'; -import { join as joinPath, resolve as resolvePath } from 'path'; -import fs from 'fs-extra'; -import mockFs from 'mock-fs'; -import { getVoidLogger, UrlReader } from '@backstage/backend-common'; -import { ScmIntegrations } from '@backstage/integration'; -import { PassThrough } from 'stream'; -import { fetchContents } from './helpers'; -import { ActionContext, TemplateAction } from '../../types'; -import { createFetchPartialAction, FetchPartialInput } from './partial'; - -jest.mock('./helpers', () => ({ - fetchContents: jest.fn(), -})); - -const aBinaryFile = fs.readFileSync( - resolvePath( - 'src', - '../fixtures/test-nested-template/public/react-logo192.png', - ), -); - -const mockFetchContents = fetchContents as jest.MockedFunction< - typeof fetchContents ->; - -describe('fetch:partial', () => { - let action: TemplateAction; - - const workspacePath = os.tmpdir(); - const createTemporaryDirectory: jest.MockedFunction< - ActionContext['createTemporaryDirectory'] - > = jest.fn(() => - Promise.resolve( - joinPath(workspacePath, `${createTemporaryDirectory.mock.calls.length}`), - ), - ); - - const logger = getVoidLogger(); - - const mockContext = (inputPatch: Partial = {}) => ({ - baseUrl: 'base-url', - input: { - url: './skeleton', - targetPath: './target', - values: { - test: 'value', - }, - ...inputPatch, - }, - output: jest.fn(), - logStream: new PassThrough(), - logger, - workspacePath, - createTemporaryDirectory, - }); - - beforeEach(() => { - mockFs(); - - action = createFetchPartialAction({ - reader: Symbol('UrlReader') as unknown as UrlReader, - integrations: Symbol('Integrations') as unknown as ScmIntegrations, - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - - it(`returns a TemplateAction with the id 'fetch:partial'`, () => { - expect(action.id).toEqual('fetch:partial'); - }); - - describe('handler', () => { - it('throws if output directory is outside the workspace', async () => { - await expect(() => - action.handler(mockContext({ targetPath: '../' })), - ).rejects.toThrowError( - /relative path is not allowed to refer to a directory outside its parent/i, - ); - }); - - describe('with valid input', () => { - let context: ActionContext; - - beforeEach(async () => { - context = mockContext({ - values: { - name: 'test-project', - count: 1234, - itemList: ['first', 'second', 'third'], - }, - }); - - mockFetchContents.mockImplementation(({ outputPath }) => { - mockFs({ - [outputPath]: { - 'empty-dir-${{ values.count }}': {}, - 'static.txt': 'static content', - '${{ values.name }}.txt': 'static content', - subdir: { - 'templated-content.txt.njk': - '${{ values.name }}: ${{ values.count }}', - }, - '.${{ values.name }}.njk': '${{ values.itemList | dump }}', - 'a-binary-file.png': aBinaryFile, - }, - }); - - return Promise.resolve(); - }); - - await action.handler(context); - }); - - it('uses fetchContents to retrieve the template content', () => { - expect(mockFetchContents).toHaveBeenCalledWith( - expect.objectContaining({ - baseUrl: context.baseUrl, - fetchUrl: context.input.url, - }), - ); - }); - - it('copies files with no templating in names or content successfully', async () => { - await expect( - fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'), - ).resolves.toEqual('static content'); - }); - - it('copies files with templated names successfully', async () => { - await expect( - fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), - ).resolves.toEqual('static content'); - }); - - it('copies files with templated content successfully', async () => { - await expect( - fs.readFile( - `${workspacePath}/target/subdir/templated-content.txt`, - 'utf-8', - ), - ).resolves.toEqual('test-project: 1234'); - }); - - it('processes dotfiles', async () => { - await expect( - fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'), - ).resolves.toEqual('["first","second","third"]'); - }); - - it('copies empty directories', async () => { - await expect( - fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'), - ).resolves.toEqual([]); - }); - - it('copies binary files as-is without processing them', async () => { - await expect( - fs.readFile(`${workspacePath}/target/a-binary-file.png`), - ).resolves.toEqual(aBinaryFile); - }); - }); - - describe('with specified .jinja2 extension', () => { - let context: ActionContext; - - beforeEach(async () => { - context = mockContext({ - extension: '.jinja2', - values: { - name: 'test-project', - count: 1234, - }, - }); - - mockFetchContents.mockImplementation(({ outputPath }) => { - mockFs({ - [outputPath]: { - '${{ values.name }}.njk': - '${{ values.name }}: ${{ values.count }}', - '${{ values.name }}.txt.jinja2': - '${{ values.name }}: ${{ values.count }}', - }, - }); - - return Promise.resolve(); - }); - - await action.handler(context); - }); - - it('does not process .njk files', async () => { - await expect( - fs.readFile(`${workspacePath}/target/test-project.njk`, 'utf-8'), - ).resolves.toEqual('${{ values.name }}: ${{ values.count }}'); - }); - - it('does process .jinja2 files', async () => { - await expect( - fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), - ).resolves.toEqual('test-project: 1234'); - }); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts deleted file mode 100644 index 2b9aecb6af..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { resolve as resolvePath } from 'path'; -import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common'; -import { ScmIntegrations } from '@backstage/integration'; -import { fetchContents } from './helpers'; -import { createTemplateAction } from '../../createTemplateAction'; -import globby from 'globby'; -import nunjucks from 'nunjucks'; -import fs from 'fs-extra'; -import { isBinaryFile } from 'isbinaryfile'; - -/* - * Maximise compatibility with Jinja (and therefore fetch:template) - * using nunjucks jinja compat mode. Since this method mutates - * the global nunjucks instance, we can't enable this per-template, - * so the next best option is to explicitly enable it globally and allow - * folks to rely on jinja compatibility behaviour in fetch:template - * templates if they wish. - * - * cf. https://mozilla.github.io/nunjucks/api.html#installjinjacompat - */ -nunjucks.installJinjaCompat(); - -export type FetchPartialInput = { - url: string; - targetPath?: string; - values: any; - extension?: string; -}; - -export function createFetchPartialAction(options: { - reader: UrlReader; - integrations: ScmIntegrations; -}) { - const { reader, integrations } = options; - - return createTemplateAction({ - id: 'fetch:partial', - description: - "Downloads a skeleton, templates variables into file and directory names and content that end with the specified extension, and places the result in the workspace, or optionally in a subdirectory specified by the 'targetPath' input option.", - schema: { - input: { - type: 'object', - required: ['url'], - properties: { - url: { - title: 'Fetch URL', - description: - 'Relative path or absolute URL pointing to the directory tree to fetch', - type: 'string', - }, - targetPath: { - title: 'Target Path', - description: - 'Target path within the working directory to download the contents to. Defaults to the working directory root.', - type: 'string', - }, - values: { - title: 'Template Values', - description: 'Values to pass on to the templating engine', - type: 'object', - }, - extension: { - title: 'Extension to Process (.njk)', - description: 'Extension to use for template.', - type: 'string', - }, - }, - }, - }, - async handler(ctx) { - ctx.logger.info('Fetching template content from remote URL'); - - const workDir = await ctx.createTemporaryDirectory(); - const templateDir = resolvePath(workDir, 'template'); - - const targetPath = ctx.input.targetPath ?? './'; - const extension = ctx.input.extension ?? '.njk'; - const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath); - - await fetchContents({ - reader, - integrations, - baseUrl: ctx.baseUrl, - fetchUrl: ctx.input.url, - outputPath: templateDir, - }); - - ctx.logger.info('Listing files and directories in template'); - const allEntriesInTemplate = await globby(`**/*`, { - cwd: templateDir, - dot: true, - onlyFiles: false, - markDirectories: true, - }); - - // Create a templater - const templater = nunjucks.configure({ - tags: { - // TODO(mtlewis/orkohunter): Document Why we are changing the literals? Not here, but on scaffolder docs. ADR? - variableStart: '${{', - variableEnd: '}}', - }, - // We don't want this builtin auto-escaping, since uses HTML escape sequences - // like `"` - the correct way to escape strings in our case depends on - // the file type. - autoescape: false, - }); - - const { values } = ctx.input; - const context = { - values, - }; - - ctx.logger.info( - `Processing ${allEntriesInTemplate.length} template files/directories with input values`, - ctx.input.values, - ); - - for (const location of allEntriesInTemplate) { - let outputPath = resolvePath( - outputDir, - templater.renderString(location, context), - ); - if (outputPath.endsWith(extension)) { - outputPath = outputPath.slice(0, -extension.length); - } - - if (location.endsWith('/')) { - ctx.logger.info( - `Writing directory ${location} to template output path.`, - ); - await fs.ensureDir(outputPath); - } else { - const inputFilePath = resolvePath(templateDir, location); - - if ( - !location.endsWith(extension) || - (await isBinaryFile(inputFilePath)) - ) { - ctx.logger.info( - `Copying file ${location} to template output path.`, - ); - await fs.copy(inputFilePath, outputPath); - } else { - ctx.logger.info( - `Writing file ${location} to template output path.`, - ); - const inputFileContents = await fs.readFile(inputFilePath, 'utf-8'); - await fs.outputFile( - outputPath, - templater.renderString(inputFileContents, context), - ); - } - } - } - - ctx.logger.info(`Template result written to ${outputDir}`); - }, - }); -} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts index aa13141f55..659fe1692a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts @@ -292,4 +292,118 @@ describe('fetch:template', () => { ).resolves.toEqual('["first","second","third"]'); }); }); + + describe('with extension=true', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + extension: true, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + 'empty-dir-${{ values.count }}': {}, + 'static.txt': 'static content', + '${{ values.name }}.txt': 'static content', + subdir: { + 'templated-content.txt.njk': + '${{ values.name }}: ${{ values.count }}', + }, + '.${{ values.name }}.njk': '${{ values.itemList | dump }}', + 'a-binary-file.png': aBinaryFile, + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('copies files with no templating in names or content successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with templated names successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with templated content successfully', async () => { + await expect( + fs.readFile( + `${workspacePath}/target/subdir/templated-content.txt`, + 'utf-8', + ), + ).resolves.toEqual('test-project: 1234'); + }); + + it('processes dotfiles', async () => { + await expect( + fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'), + ).resolves.toEqual('["first","second","third"]'); + }); + + it('copies empty directories', async () => { + await expect( + fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'), + ).resolves.toEqual([]); + }); + + it('copies binary files as-is without processing them', async () => { + await expect( + fs.readFile(`${workspacePath}/target/a-binary-file.png`), + ).resolves.toEqual(aBinaryFile); + }); + }); + + describe('with specified .jinja2 extension', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + extension: '.jinja2', + values: { + name: 'test-project', + count: 1234, + }, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + '${{ values.name }}.njk': '${{ values.name }}: ${{ values.count }}', + '${{ values.name }}.txt.jinja2': + '${{ values.name }}: ${{ values.count }}', + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('does not process .njk files', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.njk`, 'utf-8'), + ).resolves.toEqual('${{ values.name }}: ${{ values.count }}'); + }); + + it('does process .jinja2 files', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual('test-project: 1234'); + }); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts index 87e67f76be..f6afa5e8c6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts @@ -38,13 +38,21 @@ import { isBinaryFile } from 'isbinaryfile'; */ nunjucks.installJinjaCompat(); +type CookieCompatInput = { + copyWithoutRender?: string[]; + cookiecutterCompat?: boolean; +}; + +type ExtensionInput = { + extension?: string | boolean; +}; + export type FetchTemplateInput = { url: string; targetPath?: string; values: any; - copyWithoutRender?: string[]; - cookiecutterCompat?: boolean; -}; +} & CookieCompatInput & + ExtensionInput; export function createFetchTemplateAction(options: { reader: UrlReader; @@ -93,6 +101,11 @@ export function createFetchTemplateAction(options: { 'Enable features to maximise compatibility with templates built for fetch:cookiecutter', type: 'boolean', }, + extension: { + title: 'Extension to Process (.njk)', + description: 'Extension to use for templated files.', + type: ['string', 'boolean'], + }, }, }, }, @@ -114,6 +127,23 @@ export function createFetchTemplateAction(options: { ); } + if ( + ctx.input.extension && + (ctx.input.copyWithoutRender || ctx.input.cookiecutterCompat) + ) { + throw new InputError( + 'Fetch action input extension incompatible with copyWithoutRender and cookiecutterCompat', + ); + } + + let extension: string | false = false; + if (ctx.input.extension) { + extension = + typeof ctx.input.extension === 'boolean' + ? '.njk' + : ctx.input.extension; + } + await fetchContents({ reader, integrations, @@ -190,14 +220,20 @@ export function createFetchTemplateAction(options: { ); for (const location of allEntriesInTemplate) { - const shouldCopyWithoutRender = nonTemplatedEntries.has(location); + let shouldCopyWithoutRender = nonTemplatedEntries.has(location); - const outputPath = resolvePath( - outputDir, - shouldCopyWithoutRender - ? location - : templater.renderString(location, context), - ); + let localOutputPath = location; + if (extension) { + if (localOutputPath.endsWith(extension)) { + localOutputPath = localOutputPath.slice(0, -extension.length); + } else { + shouldCopyWithoutRender = true; + } + localOutputPath = templater.renderString(localOutputPath, context); + } else if (!shouldCopyWithoutRender) { + localOutputPath = templater.renderString(localOutputPath, context); + } + const outputPath = resolvePath(outputDir, localOutputPath); if (shouldCopyWithoutRender) { ctx.logger.info(