diff --git a/.changeset/heavy-monkeys-drum.md b/.changeset/heavy-monkeys-drum.md new file mode 100644 index 0000000000..0b03020ca9 --- /dev/null +++ b/.changeset/heavy-monkeys-drum.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Add `fetch:partial` templating action. + +- For all files with extension `.njk`, apply templating logic and strip extension. The extension is configurable. +- All other files get copied. +- All output paths are subject to applying templating logic. diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index 5a16ef9a55..0d1b636049 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -24,7 +24,11 @@ import { } from './catalog'; import { createDebugLogAction } from './debug'; -import { createFetchPlainAction, createFetchTemplateAction } from './fetch'; +import { + createFetchPartialAction, + createFetchPlainAction, + createFetchTemplateAction, +} from './fetch'; import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-backend-module-cookiecutter'; import { createFilesystemDeleteAction, @@ -63,6 +67,10 @@ export const createBuiltinActions = (options: { integrations, reader, }), + createFetchPartialAction({ + integrations, + reader, + }), createPublishGithubAction({ integrations, config, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts index c9a16c274e..260e85af77 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export { createFetchPartialAction } from './partial'; export { createFetchPlainAction } from './plain'; export { createFetchTemplateAction } from './template'; export { fetchContents } from './helpers'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts new file mode 100644 index 0000000000..3b5728111b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts @@ -0,0 +1,223 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import { join as joinPath, resolve as resolvePath } from 'path'; +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { PassThrough } from 'stream'; +import { fetchContents } from './helpers'; +import { ActionContext, TemplateAction } from '../../types'; +import { createFetchPartialAction, FetchPartialInput } from './partial'; + +jest.mock('./helpers', () => ({ + fetchContents: jest.fn(), +})); + +const aBinaryFile = fs.readFileSync( + resolvePath( + 'src', + '../fixtures/test-nested-template/public/react-logo192.png', + ), +); + +const mockFetchContents = fetchContents as jest.MockedFunction< + typeof fetchContents +>; + +describe('fetch:partial', () => { + let action: TemplateAction; + + const workspacePath = os.tmpdir(); + const createTemporaryDirectory: jest.MockedFunction< + ActionContext['createTemporaryDirectory'] + > = jest.fn(() => + Promise.resolve( + joinPath(workspacePath, `${createTemporaryDirectory.mock.calls.length}`), + ), + ); + + const logger = getVoidLogger(); + + const mockContext = (inputPatch: Partial = {}) => ({ + baseUrl: 'base-url', + input: { + url: './skeleton', + targetPath: './target', + values: { + test: 'value', + }, + ...inputPatch, + }, + output: jest.fn(), + logStream: new PassThrough(), + logger, + workspacePath, + createTemporaryDirectory, + }); + + beforeEach(() => { + mockFs(); + + action = createFetchPartialAction({ + reader: Symbol('UrlReader') as unknown as UrlReader, + integrations: Symbol('Integrations') as unknown as ScmIntegrations, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it(`returns a TemplateAction with the id 'fetch:partial'`, () => { + expect(action.id).toEqual('fetch:partial'); + }); + + describe('handler', () => { + it('throws if output directory is outside the workspace', async () => { + await expect(() => + action.handler(mockContext({ targetPath: '../' })), + ).rejects.toThrowError( + /relative path is not allowed to refer to a directory outside its parent/i, + ); + }); + + describe('with valid input', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + 'empty-dir-${{ values.count }}': {}, + 'static.txt': 'static content', + '${{ values.name }}.txt': 'static content', + subdir: { + 'templated-content.txt.njk': + '${{ values.name }}: ${{ values.count }}', + }, + '.${{ values.name }}.njk': '${{ values.itemList | dump }}', + 'a-binary-file.png': aBinaryFile, + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('uses fetchContents to retrieve the template content', () => { + expect(mockFetchContents).toHaveBeenCalledWith( + expect.objectContaining({ + baseUrl: context.baseUrl, + fetchUrl: context.input.url, + }), + ); + }); + + it('copies files with no templating in names or content successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with templated names successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual('static content'); + }); + + it('copies files with templated content successfully', async () => { + await expect( + fs.readFile( + `${workspacePath}/target/subdir/templated-content.txt`, + 'utf-8', + ), + ).resolves.toEqual('test-project: 1234'); + }); + + it('processes dotfiles', async () => { + await expect( + fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'), + ).resolves.toEqual('["first","second","third"]'); + }); + + it('copies empty directories', async () => { + await expect( + fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'), + ).resolves.toEqual([]); + }); + + it('copies binary files as-is without processing them', async () => { + await expect( + fs.readFile(`${workspacePath}/target/a-binary-file.png`), + ).resolves.toEqual(aBinaryFile); + }); + }); + + describe('with specified .jinja2 extension', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + extension: '.jinja2', + values: { + name: 'test-project', + count: 1234, + }, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + [outputPath]: { + '${{ values.name }}.njk': + '${{ values.name }}: ${{ values.count }}', + '${{ values.name }}.txt.jinja2': + '${{ values.name }}: ${{ values.count }}', + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('does not process .njk files', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.njk`, 'utf-8'), + ).resolves.toEqual('${{ values.name }}: ${{ values.count }}'); + }); + + it('does process .jinja2 files', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual('test-project: 1234'); + }); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts new file mode 100644 index 0000000000..2b9aecb6af --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { fetchContents } from './helpers'; +import { createTemplateAction } from '../../createTemplateAction'; +import globby from 'globby'; +import nunjucks from 'nunjucks'; +import fs from 'fs-extra'; +import { isBinaryFile } from 'isbinaryfile'; + +/* + * Maximise compatibility with Jinja (and therefore fetch:template) + * using nunjucks jinja compat mode. Since this method mutates + * the global nunjucks instance, we can't enable this per-template, + * so the next best option is to explicitly enable it globally and allow + * folks to rely on jinja compatibility behaviour in fetch:template + * templates if they wish. + * + * cf. https://mozilla.github.io/nunjucks/api.html#installjinjacompat + */ +nunjucks.installJinjaCompat(); + +export type FetchPartialInput = { + url: string; + targetPath?: string; + values: any; + extension?: string; +}; + +export function createFetchPartialAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; +}) { + const { reader, integrations } = options; + + return createTemplateAction({ + id: 'fetch:partial', + description: + "Downloads a skeleton, templates variables into file and directory names and content that end with the specified extension, and places the result in the workspace, or optionally in a subdirectory specified by the 'targetPath' input option.", + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to. Defaults to the working directory root.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to the templating engine', + type: 'object', + }, + extension: { + title: 'Extension to Process (.njk)', + description: 'Extension to use for template.', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + ctx.logger.info('Fetching template content from remote URL'); + + const workDir = await ctx.createTemporaryDirectory(); + const templateDir = resolvePath(workDir, 'template'); + + const targetPath = ctx.input.targetPath ?? './'; + const extension = ctx.input.extension ?? '.njk'; + const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath); + + await fetchContents({ + reader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.input.url, + outputPath: templateDir, + }); + + ctx.logger.info('Listing files and directories in template'); + const allEntriesInTemplate = await globby(`**/*`, { + cwd: templateDir, + dot: true, + onlyFiles: false, + markDirectories: true, + }); + + // Create a templater + const templater = nunjucks.configure({ + tags: { + // TODO(mtlewis/orkohunter): Document Why we are changing the literals? Not here, but on scaffolder docs. ADR? + variableStart: '${{', + variableEnd: '}}', + }, + // We don't want this builtin auto-escaping, since uses HTML escape sequences + // like `"` - the correct way to escape strings in our case depends on + // the file type. + autoescape: false, + }); + + const { values } = ctx.input; + const context = { + values, + }; + + ctx.logger.info( + `Processing ${allEntriesInTemplate.length} template files/directories with input values`, + ctx.input.values, + ); + + for (const location of allEntriesInTemplate) { + let outputPath = resolvePath( + outputDir, + templater.renderString(location, context), + ); + if (outputPath.endsWith(extension)) { + outputPath = outputPath.slice(0, -extension.length); + } + + if (location.endsWith('/')) { + ctx.logger.info( + `Writing directory ${location} to template output path.`, + ); + await fs.ensureDir(outputPath); + } else { + const inputFilePath = resolvePath(templateDir, location); + + if ( + !location.endsWith(extension) || + (await isBinaryFile(inputFilePath)) + ) { + ctx.logger.info( + `Copying file ${location} to template output path.`, + ); + await fs.copy(inputFilePath, outputPath); + } else { + ctx.logger.info( + `Writing file ${location} to template output path.`, + ); + const inputFileContents = await fs.readFile(inputFilePath, 'utf-8'); + await fs.outputFile( + outputPath, + templater.renderString(inputFileContents, context), + ); + } + } + } + + ctx.logger.info(`Template result written to ${outputDir}`); + }, + }); +}