Merge pull request #6800 from Pike/add-partial-template

Add a partial template action to the builtin scaffolder actions.

.changeset/heavy-monkeys-drum.md

@@ -0,0 +1,11 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Add partial templating to `fetch:template` action.
+
+If an `templateFileExtension` input is given, only files with that extension get their content processed. If `templateFileExtension` is `true`, the `.njk` extension is used. The `templateFileExtension` input is incompatible with both `cookiecutterCompat` and `copyWithoutRender`.
+
+All other files get copied.
+
+All output paths are subject to applying templating logic.

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts

@@ -105,6 +105,32 @@ describe('fetch:template', () => {
       ).rejects.toThrowError(/copyWithoutRender must be an array/i);
     });
 
+    it('throws if copyWithoutRender is used with extension', async () => {
+      await expect(() =>
+        action.handler(
+          mockContext({
+            copyWithoutRender: ['abc'],
+            templateFileExtension: true,
+          }),
+        ),
+      ).rejects.toThrowError(
+        /input extension incompatible with copyWithoutRender and cookiecutterCompat/,
+      );
+    });
+
+    it('throws if cookiecutterCompat is used with extension', async () => {
+      await expect(() =>
+        action.handler(
+          mockContext({
+            cookiecutterCompat: true,
+            templateFileExtension: true,
+          }),
+        ),
+      ).rejects.toThrowError(
+        /input extension incompatible with copyWithoutRender and cookiecutterCompat/,
+      );
+    });
+
     describe('with valid input', () => {
       let context: ActionContext<FetchTemplateInput>;
 
@@ -292,4 +318,126 @@ describe('fetch:template', () => {
       ).resolves.toEqual('["first","second","third"]');
     });
   });
+
+  describe('with extension=true', () => {
+    let context: ActionContext<FetchTemplateInput>;
+
+    beforeEach(async () => {
+      context = mockContext({
+        values: {
+          name: 'test-project',
+          count: 1234,
+          itemList: ['first', 'second', 'third'],
+        },
+        templateFileExtension: true,
+      });
+
+      mockFetchContents.mockImplementation(({ outputPath }) => {
+        mockFs({
+          [outputPath]: {
+            'empty-dir-${{ values.count }}': {},
+            'static.txt': 'static content',
+            '${{ values.name }}.txt': 'static content',
+            '${{ values.name }}.txt.jinja2':
+              '${{ values.name }}: ${{ values.count }}',
+            subdir: {
+              'templated-content.txt.njk':
+                '${{ values.name }}: ${{ values.count }}',
+            },
+            '.${{ values.name }}.njk': '${{ values.itemList | dump }}',
+            'a-binary-file.png': aBinaryFile,
+          },
+        });
+
+        return Promise.resolve();
+      });
+
+      await action.handler(context);
+    });
+
+    it('copies files with no templating in names or content successfully', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'),
+      ).resolves.toEqual('static content');
+    });
+
+    it('copies files with templated names successfully', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
+      ).resolves.toEqual('static content');
+    });
+
+    it('copies jinja2 files with templated names successfully', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/test-project.txt.jinja2`, 'utf-8'),
+      ).resolves.toEqual('${{ values.name }}: ${{ values.count }}');
+    });
+
+    it('copies files with templated content successfully', async () => {
+      await expect(
+        fs.readFile(
+          `${workspacePath}/target/subdir/templated-content.txt`,
+          'utf-8',
+        ),
+      ).resolves.toEqual('test-project: 1234');
+    });
+
+    it('processes dotfiles', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'),
+      ).resolves.toEqual('["first","second","third"]');
+    });
+
+    it('copies empty directories', async () => {
+      await expect(
+        fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'),
+      ).resolves.toEqual([]);
+    });
+
+    it('copies binary files as-is without processing them', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/a-binary-file.png`),
+      ).resolves.toEqual(aBinaryFile);
+    });
+  });
+
+  describe('with specified .jinja2 extension', () => {
+    let context: ActionContext<FetchTemplateInput>;
+
+    beforeEach(async () => {
+      context = mockContext({
+        templateFileExtension: '.jinja2',
+        values: {
+          name: 'test-project',
+          count: 1234,
+        },
+      });
+
+      mockFetchContents.mockImplementation(({ outputPath }) => {
+        mockFs({
+          [outputPath]: {
+            '${{ values.name }}.njk': '${{ values.name }}: ${{ values.count }}',
+            '${{ values.name }}.txt.jinja2':
+              '${{ values.name }}: ${{ values.count }}',
+          },
+        });
+
+        return Promise.resolve();
+      });
+
+      await action.handler(context);
+    });
+
+    it('does not process .njk files', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/test-project.njk`, 'utf-8'),
+      ).resolves.toEqual('${{ values.name }}: ${{ values.count }}');
+    });
+
+    it('does process .jinja2 files', async () => {
+      await expect(
+        fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
+      ).resolves.toEqual('test-project: 1234');
+    });
+  });
 });

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { resolve as resolvePath } from 'path';
+import { resolve as resolvePath, extname } from 'path';
 import { resolveSafeChildPath, UrlReader } from '@backstage/backend-common';
 import { InputError } from '@backstage/errors';
 import { ScmIntegrations } from '@backstage/integration';
@@ -38,13 +38,21 @@ import { isBinaryFile } from 'isbinaryfile';
  */
 nunjucks.installJinjaCompat();
 
+type CookieCompatInput = {
+  copyWithoutRender?: string[];
+  cookiecutterCompat?: boolean;
+};
+
+type ExtensionInput = {
+  templateFileExtension?: string | boolean;
+};
+
 export type FetchTemplateInput = {
   url: string;
   targetPath?: string;
   values: any;
-  copyWithoutRender?: string[];
-  cookiecutterCompat?: boolean;
-};
+} & CookieCompatInput &
+  ExtensionInput;
 
 export function createFetchTemplateAction(options: {
   reader: UrlReader;
@@ -93,6 +101,12 @@ export function createFetchTemplateAction(options: {
               'Enable features to maximise compatibility with templates built for fetch:cookiecutter',
             type: 'boolean',
           },
+          templateFileExtension: {
+            title: 'Template File Extension',
+            description:
+              'If set, only files with the given extension will be templated. If set to `true`, the default extension `.njk` is used.',
+            type: ['string', 'boolean'],
+          },
         },
       },
     },
@@ -114,6 +128,26 @@ export function createFetchTemplateAction(options: {
         );
       }
 
+      if (
+        ctx.input.templateFileExtension &&
+        (ctx.input.copyWithoutRender || ctx.input.cookiecutterCompat)
+      ) {
+        throw new InputError(
+          'Fetch action input extension incompatible with copyWithoutRender and cookiecutterCompat',
+        );
+      }
+
+      let extension: string | false = false;
+      if (ctx.input.templateFileExtension) {
+        extension =
+          ctx.input.templateFileExtension === true
+            ? '.njk'
+            : ctx.input.templateFileExtension;
+        if (!extension.startsWith('.')) {
+          extension = `.${extension}`;
+        }
+      }
+
       await fetchContents({
         reader,
         integrations,
@@ -190,18 +224,27 @@ export function createFetchTemplateAction(options: {
       );
 
       for (const location of allEntriesInTemplate) {
-        const shouldCopyWithoutRender = nonTemplatedEntries.has(location);
+        let renderFilename: boolean;
+        let renderContents: boolean;
 
-        const outputPath = resolvePath(
-          outputDir,
-          shouldCopyWithoutRender
-            ? location
-            : templater.renderString(location, context),
-        );
+        let localOutputPath = location;
+        if (extension) {
+          renderFilename = true;
+          renderContents = extname(localOutputPath) === extension;
+          if (renderContents) {
+            localOutputPath = localOutputPath.slice(0, -extension.length);
+          }
+        } else {
+          renderFilename = renderContents = !nonTemplatedEntries.has(location);
+        }
+        if (renderFilename) {
+          localOutputPath = templater.renderString(localOutputPath, context);
+        }
+        const outputPath = resolvePath(outputDir, localOutputPath);
 
-        if (shouldCopyWithoutRender) {
+        if (!renderContents && !extension) {
           ctx.logger.info(
-            `Copying file/directory ${location} without processing since it matches a pattern in "copyWithoutRender".`,
+            `Copying file/directory ${location} without processing.`,
           );
         }
 
@@ -225,9 +268,9 @@ export function createFetchTemplateAction(options: {
             const inputFileContents = await fs.readFile(inputFilePath, 'utf-8');
             await fs.outputFile(
               outputPath,
-              shouldCopyWithoutRender
-                ? inputFileContents
-                : templater.renderString(inputFileContents, context),
+              renderContents
+                ? templater.renderString(inputFileContents, context)
+                : inputFileContents,
             );
           }
         }