Add a partial template action to the builtin scaffolder actions.

Closes #6794

This blends fetch:plain and fetch:template, and triggers the template
engine only for files with a specific extension, .njk by default.
It doesn't contain the cookiecutter-compat logic of fetch:template.

Signed-off-by: Axel Hecht <axel@pike.org>

.changeset/heavy-monkeys-drum.md

@@ -0,0 +1,9 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Add `fetch:partial` templating action.
+
+- For all files with extension `.njk`, apply templating logic and strip extension. The extension is configurable.
+- All other files get copied.
+- All output paths are subject to applying templating logic.

plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts

@@ -24,7 +24,11 @@ import {
 } from './catalog';
 
 import { createDebugLogAction } from './debug';
-import { createFetchPlainAction, createFetchTemplateAction } from './fetch';
+import {
+  createFetchPartialAction,
+  createFetchPlainAction,
+  createFetchTemplateAction,
+} from './fetch';
 import { createFetchCookiecutterAction } from '@backstage/plugin-scaffolder-backend-module-cookiecutter';
 import {
   createFilesystemDeleteAction,
@@ -63,6 +67,10 @@ export const createBuiltinActions = (options: {
       integrations,
       reader,
     }),
+    createFetchPartialAction({
+      integrations,
+      reader,
+    }),
     createPublishGithubAction({
       integrations,
       config,

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+export { createFetchPartialAction } from './partial';
 export { createFetchPlainAction } from './plain';
 export { createFetchTemplateAction } from './template';
 export { fetchContents } from './helpers';

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.test.ts

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import { join as joinPath, resolve as resolvePath } from 'path';
+import fs from 'fs-extra';
+import mockFs from 'mock-fs';
+import { getVoidLogger, UrlReader } from '@backstage/backend-common';
+import { ScmIntegrations } from '@backstage/integration';
+import { PassThrough } from 'stream';
+import { fetchContents } from './helpers';
+import { ActionContext, TemplateAction } from '../../types';
+import { createFetchPartialAction, FetchPartialInput } from './partial';
+
+jest.mock('./helpers', () => ({
+  fetchContents: jest.fn(),
+}));
+
+const aBinaryFile = fs.readFileSync(
+  resolvePath(
+    'src',
+    '../fixtures/test-nested-template/public/react-logo192.png',
+  ),
+);
+
+const mockFetchContents = fetchContents as jest.MockedFunction<
+  typeof fetchContents
+>;
+
+describe('fetch:partial', () => {
+  let action: TemplateAction<any>;
+
+  const workspacePath = os.tmpdir();
+  const createTemporaryDirectory: jest.MockedFunction<
+    ActionContext<FetchPartialInput>['createTemporaryDirectory']
+  > = jest.fn(() =>
+    Promise.resolve(
+      joinPath(workspacePath, `${createTemporaryDirectory.mock.calls.length}`),
+    ),
+  );
+
+  const logger = getVoidLogger();
+
+  const mockContext = (inputPatch: Partial<FetchPartialInput> = {}) => ({
+    baseUrl: 'base-url',
+    input: {
+      url: './skeleton',
+      targetPath: './target',
+      values: {
+        test: 'value',
+      },
+      ...inputPatch,
+    },
+    output: jest.fn(),
+    logStream: new PassThrough(),
+    logger,
+    workspacePath,
+    createTemporaryDirectory,
+  });
+
+  beforeEach(() => {
+    mockFs();
+
+    action = createFetchPartialAction({
+      reader: Symbol('UrlReader') as unknown as UrlReader,
+      integrations: Symbol('Integrations') as unknown as ScmIntegrations,
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it(`returns a TemplateAction with the id 'fetch:partial'`, () => {
+    expect(action.id).toEqual('fetch:partial');
+  });
+
+  describe('handler', () => {
+    it('throws if output directory is outside the workspace', async () => {
+      await expect(() =>
+        action.handler(mockContext({ targetPath: '../' })),
+      ).rejects.toThrowError(
+        /relative path is not allowed to refer to a directory outside its parent/i,
+      );
+    });
+
+    describe('with valid input', () => {
+      let context: ActionContext<FetchPartialInput>;
+
+      beforeEach(async () => {
+        context = mockContext({
+          values: {
+            name: 'test-project',
+            count: 1234,
+            itemList: ['first', 'second', 'third'],
+          },
+        });
+
+        mockFetchContents.mockImplementation(({ outputPath }) => {
+          mockFs({
+            [outputPath]: {
+              'empty-dir-${{ values.count }}': {},
+              'static.txt': 'static content',
+              '${{ values.name }}.txt': 'static content',
+              subdir: {
+                'templated-content.txt.njk':
+                  '${{ values.name }}: ${{ values.count }}',
+              },
+              '.${{ values.name }}.njk': '${{ values.itemList | dump }}',
+              'a-binary-file.png': aBinaryFile,
+            },
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('uses fetchContents to retrieve the template content', () => {
+        expect(mockFetchContents).toHaveBeenCalledWith(
+          expect.objectContaining({
+            baseUrl: context.baseUrl,
+            fetchUrl: context.input.url,
+          }),
+        );
+      });
+
+      it('copies files with no templating in names or content successfully', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'),
+        ).resolves.toEqual('static content');
+      });
+
+      it('copies files with templated names successfully', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
+        ).resolves.toEqual('static content');
+      });
+
+      it('copies files with templated content successfully', async () => {
+        await expect(
+          fs.readFile(
+            `${workspacePath}/target/subdir/templated-content.txt`,
+            'utf-8',
+          ),
+        ).resolves.toEqual('test-project: 1234');
+      });
+
+      it('processes dotfiles', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'),
+        ).resolves.toEqual('["first","second","third"]');
+      });
+
+      it('copies empty directories', async () => {
+        await expect(
+          fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'),
+        ).resolves.toEqual([]);
+      });
+
+      it('copies binary files as-is without processing them', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/a-binary-file.png`),
+        ).resolves.toEqual(aBinaryFile);
+      });
+    });
+
+    describe('with specified .jinja2 extension', () => {
+      let context: ActionContext<FetchPartialInput>;
+
+      beforeEach(async () => {
+        context = mockContext({
+          extension: '.jinja2',
+          values: {
+            name: 'test-project',
+            count: 1234,
+          },
+        });
+
+        mockFetchContents.mockImplementation(({ outputPath }) => {
+          mockFs({
+            [outputPath]: {
+              '${{ values.name }}.njk':
+                '${{ values.name }}: ${{ values.count }}',
+              '${{ values.name }}.txt.jinja2':
+                '${{ values.name }}: ${{ values.count }}',
+            },
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('does not process .njk files', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/test-project.njk`, 'utf-8'),
+        ).resolves.toEqual('${{ values.name }}: ${{ values.count }}');
+      });
+
+      it('does process .jinja2 files', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
+        ).resolves.toEqual('test-project: 1234');
+      });
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/partial.ts

@@ -0,0 +1,176 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { resolve as resolvePath } from 'path';
+import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common';
+import { ScmIntegrations } from '@backstage/integration';
+import { fetchContents } from './helpers';
+import { createTemplateAction } from '../../createTemplateAction';
+import globby from 'globby';
+import nunjucks from 'nunjucks';
+import fs from 'fs-extra';
+import { isBinaryFile } from 'isbinaryfile';
+
+/*
+ * Maximise compatibility with Jinja (and therefore fetch:template)
+ * using nunjucks jinja compat mode. Since this method mutates
+ * the global nunjucks instance, we can't enable this per-template,
+ * so the next best option is to explicitly enable it globally and allow
+ * folks to rely on jinja compatibility behaviour in fetch:template
+ * templates if they wish.
+ *
+ * cf. https://mozilla.github.io/nunjucks/api.html#installjinjacompat
+ */
+nunjucks.installJinjaCompat();
+
+export type FetchPartialInput = {
+  url: string;
+  targetPath?: string;
+  values: any;
+  extension?: string;
+};
+
+export function createFetchPartialAction(options: {
+  reader: UrlReader;
+  integrations: ScmIntegrations;
+}) {
+  const { reader, integrations } = options;
+
+  return createTemplateAction<FetchPartialInput>({
+    id: 'fetch:partial',
+    description:
+      "Downloads a skeleton, templates variables into file and directory names and content that end with the specified extension, and places the result in the workspace, or optionally in a subdirectory specified by the 'targetPath' input option.",
+    schema: {
+      input: {
+        type: 'object',
+        required: ['url'],
+        properties: {
+          url: {
+            title: 'Fetch URL',
+            description:
+              'Relative path or absolute URL pointing to the directory tree to fetch',
+            type: 'string',
+          },
+          targetPath: {
+            title: 'Target Path',
+            description:
+              'Target path within the working directory to download the contents to. Defaults to the working directory root.',
+            type: 'string',
+          },
+          values: {
+            title: 'Template Values',
+            description: 'Values to pass on to the templating engine',
+            type: 'object',
+          },
+          extension: {
+            title: 'Extension to Process (.njk)',
+            description: 'Extension to use for template.',
+            type: 'string',
+          },
+        },
+      },
+    },
+    async handler(ctx) {
+      ctx.logger.info('Fetching template content from remote URL');
+
+      const workDir = await ctx.createTemporaryDirectory();
+      const templateDir = resolvePath(workDir, 'template');
+
+      const targetPath = ctx.input.targetPath ?? './';
+      const extension = ctx.input.extension ?? '.njk';
+      const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath);
+
+      await fetchContents({
+        reader,
+        integrations,
+        baseUrl: ctx.baseUrl,
+        fetchUrl: ctx.input.url,
+        outputPath: templateDir,
+      });
+
+      ctx.logger.info('Listing files and directories in template');
+      const allEntriesInTemplate = await globby(`**/*`, {
+        cwd: templateDir,
+        dot: true,
+        onlyFiles: false,
+        markDirectories: true,
+      });
+
+      // Create a templater
+      const templater = nunjucks.configure({
+        tags: {
+          // TODO(mtlewis/orkohunter): Document Why we are changing the literals? Not here, but on scaffolder docs. ADR?
+          variableStart: '${{',
+          variableEnd: '}}',
+        },
+        // We don't want this builtin auto-escaping, since uses HTML escape sequences
+        // like `&quot;` - the correct way to escape strings in our case depends on
+        // the file type.
+        autoescape: false,
+      });
+
+      const { values } = ctx.input;
+      const context = {
+        values,
+      };
+
+      ctx.logger.info(
+        `Processing ${allEntriesInTemplate.length} template files/directories with input values`,
+        ctx.input.values,
+      );
+
+      for (const location of allEntriesInTemplate) {
+        let outputPath = resolvePath(
+          outputDir,
+          templater.renderString(location, context),
+        );
+        if (outputPath.endsWith(extension)) {
+          outputPath = outputPath.slice(0, -extension.length);
+        }
+
+        if (location.endsWith('/')) {
+          ctx.logger.info(
+            `Writing directory ${location} to template output path.`,
+          );
+          await fs.ensureDir(outputPath);
+        } else {
+          const inputFilePath = resolvePath(templateDir, location);
+
+          if (
+            !location.endsWith(extension) ||
+            (await isBinaryFile(inputFilePath))
+          ) {
+            ctx.logger.info(
+              `Copying file ${location} to template output path.`,
+            );
+            await fs.copy(inputFilePath, outputPath);
+          } else {
+            ctx.logger.info(
+              `Writing file ${location} to template output path.`,
+            );
+            const inputFileContents = await fs.readFile(inputFilePath, 'utf-8');
+            await fs.outputFile(
+              outputPath,
+              templater.renderString(inputFileContents, context),
+            );
+          }
+        }
+      }
+
+      ctx.logger.info(`Template result written to ${outputDir}`);
+    },
+  });
+}