add fetch:template:file action

Signed-off-by: Stephen Glass <stephen@stephen.glass>

.changeset/brown-frogs-walk.md

@@ -0,0 +1,17 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Add `fetch:template:file` scaffolder action to download a single file and template the contents. Example usage:
+
+```yaml
+- id: fetch-file
+  name: Fetch File
+  action: fetch:template:file
+  input:
+    url: https://github.com/backstage/software-templates/blob/main/scaffolder-templates/create-react-app/skeleton/catalog-info.yaml
+    targetPath: './target/catalog-info.yaml'
+    values:
+      component_id: My Component
+      owner: Test
+```

plugins/scaffolder-backend/api-report.md

@@ -202,6 +202,26 @@ export function createFetchTemplateAction(options: {
   JsonObject
 >;
 
+// @public
+export function createFetchTemplateFileAction(options: {
+  reader: UrlReaderService;
+  integrations: ScmIntegrations;
+  additionalTemplateFilters?: Record<string, TemplateFilter_2>;
+  additionalTemplateGlobals?: Record<string, TemplateGlobal_2>;
+}): TemplateAction_2<
+  {
+    url: string;
+    targetPath: string;
+    values: any;
+    cookiecutterCompat?: boolean | undefined;
+    replace?: boolean | undefined;
+    trimBlocks?: boolean | undefined;
+    lstripBlocks?: boolean | undefined;
+    token?: string | undefined;
+  },
+  JsonObject
+>;
+
 // @public
 export const createFilesystemDeleteAction: () => TemplateAction_2<
   {

plugins/scaffolder-backend/src/ScaffolderPlugin.ts

@@ -44,6 +44,7 @@ import {
   createFetchPlainAction,
   createFetchPlainFileAction,
   createFetchTemplateAction,
+  createFetchTemplateFileAction,
   createFilesystemDeleteAction,
   createFilesystemRenameAction,
   createWaitAction,
@@ -149,6 +150,12 @@ export const scaffolderPlugin = createBackendPlugin({
             additionalTemplateFilters,
             additionalTemplateGlobals,
           }),
+          createFetchTemplateFileAction({
+            integrations,
+            reader,
+            additionalTemplateFilters,
+            additionalTemplateGlobals,
+          }),
           createDebugLogAction(),
           createWaitAction(),
           // todo(blam): maybe these should be a -catalog module?

plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts

@@ -37,6 +37,7 @@ import {
   createFetchPlainAction,
   createFetchPlainFileAction,
   createFetchTemplateAction,
+  createFetchTemplateFileAction,
 } from './fetch';
 import {
   createFilesystemDeleteAction,
@@ -157,6 +158,12 @@ export const createBuiltinActions = (
       additionalTemplateFilters,
       additionalTemplateGlobals,
     }),
+    createFetchTemplateFileAction({
+      integrations,
+      reader,
+      additionalTemplateFilters,
+      additionalTemplateGlobals,
+    }),
     createPublishGerritAction({
       integrations,
       config,

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts

@@ -17,3 +17,4 @@
 export { createFetchPlainAction } from './plain';
 export { createFetchPlainFileAction } from './plainFile';
 export { createFetchTemplateAction } from './template';
+export { createFetchTemplateFileAction } from './templateFile';

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.test.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { join as joinPath } from 'path';
+import fs from 'fs-extra';
+import { UrlReaderService } from '@backstage/backend-plugin-api';
+import { ScmIntegrations } from '@backstage/integration';
+import { createMockActionContext } from '@backstage/plugin-scaffolder-node-test-utils';
+import { createFetchTemplateFileAction } from './templateFile';
+import {
+  ActionContext,
+  TemplateAction,
+  fetchFile,
+} from '@backstage/plugin-scaffolder-node';
+import { examples } from './templateFile.examples';
+import yaml from 'yaml';
+import { createMockDirectory } from '@backstage/backend-test-utils';
+
+jest.mock('@backstage/plugin-scaffolder-node', () => ({
+  ...jest.requireActual('@backstage/plugin-scaffolder-node'),
+  fetchFile: jest.fn(),
+}));
+
+type FetchTemplateInput = ReturnType<
+  typeof createFetchTemplateFileAction
+> extends TemplateAction<infer U>
+  ? U
+  : never;
+
+const mockFetchFile = fetchFile as jest.MockedFunction<typeof fetchFile>;
+
+describe('fetch:template:file examples', () => {
+  let action: TemplateAction<any>;
+
+  const mockDir = createMockDirectory();
+  const workspacePath = mockDir.resolve('workspace');
+
+  const mockContext = (input: any) =>
+    createMockActionContext({
+      templateInfo: {
+        baseUrl: 'base-url',
+        entityRef: 'template:default/test-template',
+      },
+      input,
+      workspacePath,
+    });
+
+  beforeEach(() => {
+    mockDir.clear();
+    action = createFetchTemplateFileAction({
+      reader: Symbol('UrlReader') as unknown as UrlReaderService,
+      integrations: Symbol('Integrations') as unknown as ScmIntegrations,
+    });
+  });
+
+  describe('handler', () => {
+    describe('with valid input', () => {
+      let context: ActionContext<FetchTemplateInput>;
+
+      beforeEach(async () => {
+        context = mockContext(yaml.parse(examples[0].example).steps[0].input);
+
+        mockFetchFile.mockImplementation(({ outputPath }) => {
+          mockDir.setContent({
+            [outputPath]:
+              '${{ values.name }}: ${{ values.count }} ${{ values.itemList | dump }}',
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('uses fetchFile to retrieve the template content', () => {
+        expect(mockFetchFile).toHaveBeenCalledWith(
+          expect.objectContaining({
+            baseUrl: context.templateInfo?.baseUrl,
+            fetchUrl: context.input.url,
+          }),
+        );
+      });
+
+      it('copies files with no templating in names or content successfully', async () => {
+        await expect(
+          fs.readFile(
+            joinPath(workspacePath, context.input.targetPath),
+            'utf-8',
+          ),
+        ).resolves.toEqual('test-project: 1234 ["first","second","third"]');
+      });
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TemplateExample } from '@backstage/plugin-scaffolder-node';
+import yaml from 'yaml';
+
+export const examples: TemplateExample[] = [
+  {
+    description: 'Downloads a template file and fill it out with values.',
+    example: yaml.stringify({
+      steps: [
+        {
+          action: 'fetch:template:file',
+          id: 'fetch-template-file',
+          name: 'Fetch template file',
+          input: {
+            url: './skeleton.txt',
+            targetPath: './target/skeleton.txt',
+            values: {
+              name: 'test-project',
+              count: 1234,
+              itemList: ['first', 'second', 'third'],
+            },
+          },
+        },
+      ],
+    }),
+  },
+];

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.test.ts

@@ -0,0 +1,251 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('@backstage/plugin-scaffolder-node', () => {
+  const actual = jest.requireActual('@backstage/plugin-scaffolder-node');
+  return { ...actual, fetchFile: jest.fn() };
+});
+
+import { join as joinPath } from 'path';
+import fs from 'fs-extra';
+import { UrlReaderService } from '@backstage/backend-plugin-api';
+import { ScmIntegrations } from '@backstage/integration';
+import { createFetchTemplateFileAction } from './templateFile';
+import {
+  ActionContext,
+  TemplateAction,
+  fetchFile,
+} from '@backstage/plugin-scaffolder-node';
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import { createMockActionContext } from '@backstage/plugin-scaffolder-node-test-utils';
+
+type FetchTemplateInput = ReturnType<
+  typeof createFetchTemplateFileAction
+> extends TemplateAction<infer U>
+  ? U
+  : never;
+
+const mockFetchFile = fetchFile as jest.MockedFunction<typeof fetchFile>;
+
+describe('fetch:template:file', () => {
+  let action: TemplateAction<any>;
+
+  const mockDir = createMockDirectory();
+  const workspacePath = mockDir.resolve('workspace');
+  const mockContext = (inputPatch: Partial<FetchTemplateInput> = {}) =>
+    createMockActionContext({
+      templateInfo: {
+        baseUrl: 'base-url',
+        entityRef: 'template:default/test-template',
+      },
+      input: {
+        url: './skeleton.txt',
+        targetPath: './target/skeleton.txt',
+        values: {
+          test: 'value',
+        },
+        ...inputPatch,
+      },
+      workspacePath,
+    });
+
+  beforeEach(() => {
+    mockDir.setContent({
+      workspace: {},
+    });
+    action = createFetchTemplateFileAction({
+      reader: Symbol('UrlReader') as unknown as UrlReaderService,
+      integrations: Symbol('Integrations') as unknown as ScmIntegrations,
+    });
+  });
+
+  it(`returns a TemplateAction with the id 'fetch:template:file'`, () => {
+    expect(action.id).toEqual('fetch:template:file');
+  });
+
+  describe('handler', () => {
+    it('should disallow a target path outside working directory', async () => {
+      await expect(
+        action.handler(mockContext({ targetPath: '../' })),
+      ).rejects.toThrow(
+        /Relative path is not allowed to refer to a directory outside its parent/,
+      );
+    });
+
+    describe('valid input', () => {
+      let context: ActionContext<FetchTemplateInput>;
+
+      beforeEach(async () => {
+        context = mockContext({
+          values: {
+            name: 'test-project',
+            count: 1234,
+            itemList: ['first', 'second', 'third'],
+          },
+          token: 'mockToken',
+        });
+
+        mockFetchFile.mockImplementation(({ outputPath }) => {
+          mockDir.setContent({
+            [outputPath]:
+              '${{ values.name }}: ${{ values.count }} ${{ values.itemList | dump }}',
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('uses fetchFile to retrieve the template content', () => {
+        expect(mockFetchFile).toHaveBeenCalledWith(
+          expect.objectContaining({
+            baseUrl: context.templateInfo?.baseUrl,
+            fetchUrl: context.input.url,
+          }),
+        );
+      });
+
+      it('passed through the token to fetchFile', () => {
+        expect(mockFetchFile).toHaveBeenCalledWith(
+          expect.objectContaining({
+            token: 'mockToken',
+          }),
+        );
+      });
+
+      it('templates content successfully', async () => {
+        await expect(
+          fs.readFile(
+            joinPath(workspacePath, context.input.targetPath),
+            'utf-8',
+          ),
+        ).resolves.toEqual('test-project: 1234 ["first","second","third"]');
+      });
+    });
+
+    describe('with replacement of existing files', () => {
+      let context: ActionContext<FetchTemplateInput>;
+
+      beforeEach(async () => {
+        context = mockContext({
+          url: './static-content.txt',
+          targetPath: './target/static-content.txt',
+          values: {
+            name: 'test-project',
+            count: 1234,
+          },
+          replace: true,
+        });
+
+        mockDir.setContent({
+          [joinPath(workspacePath, 'target')]: {
+            'static-content.txt': 'static-content',
+          },
+        });
+
+        mockFetchFile.mockImplementation(({ outputPath }) => {
+          mockDir.setContent({
+            [outputPath]: '${{ values.name }}: ${{ values.count }}',
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('overwrites existing file', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/static-content.txt`, 'utf-8'),
+        ).resolves.toEqual('test-project: 1234');
+      });
+    });
+
+    describe('without replacement of existing files', () => {
+      let context: ActionContext<FetchTemplateInput>;
+
+      beforeEach(async () => {
+        context = mockContext({
+          url: './static-content.txt',
+          targetPath: './target/static-content.txt',
+          values: {
+            name: 'test-project',
+            count: 1234,
+          },
+          replace: false,
+        });
+
+        mockDir.setContent({
+          [joinPath(workspacePath, 'target')]: {
+            'static-content.txt': 'static-content',
+          },
+        });
+
+        mockFetchFile.mockImplementation(({ outputPath }) => {
+          mockDir.setContent({
+            [outputPath]: '${{ values.name }}: ${{ values.count }}',
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('keeps existing file', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/static-content.txt`, 'utf-8'),
+        ).resolves.toEqual('static-content');
+      });
+    });
+
+    describe('cookiecutter compatibility mode', () => {
+      let context: ActionContext<FetchTemplateInput>;
+
+      beforeEach(async () => {
+        context = mockContext({
+          targetPath: './target/test-project.txt',
+          values: {
+            name: 'test-project',
+            count: 1234,
+            itemList: ['first', 'second', 'third'],
+          },
+          cookiecutterCompat: true,
+        });
+
+        mockFetchFile.mockImplementation(({ outputPath }) => {
+          mockDir.setContent({
+            [outputPath]:
+              'static:{{ cookiecutter.name }}:{{ cookiecutter.count }}:{{ cookiecutter.itemList | jsonify }}',
+          });
+
+          return Promise.resolve();
+        });
+
+        await action.handler(context);
+      });
+
+      it('copies files with cookiecutter-style templated variables successfully', async () => {
+        await expect(
+          fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
+        ).resolves.toEqual(
+          'static:test-project:1234:["first","second","third"]',
+        );
+      });
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.ts

@@ -0,0 +1,167 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReaderService } from '@backstage/backend-plugin-api';
+import { resolveSafeChildPath } from '@backstage/backend-plugin-api';
+import { ScmIntegrations } from '@backstage/integration';
+import { examples } from './plainFile.examples';
+import {
+  createTemplateAction,
+  fetchFile,
+  TemplateFilter,
+  TemplateGlobal,
+} from '@backstage/plugin-scaffolder-node';
+import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
+import { createDefaultFilters } from '../../../../lib/templating/filters';
+import path from 'path';
+import fs from 'fs-extra';
+
+/**
+ * Downloads a single file and templates variables into file.
+ * Then places the result in the workspace, or optionally in a subdirectory
+ * specified by the 'targetPath' input option.
+ * @public
+ */
+export function createFetchTemplateFileAction(options: {
+  reader: UrlReaderService;
+  integrations: ScmIntegrations;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
+  additionalTemplateGlobals?: Record<string, TemplateGlobal>;
+}) {
+  const {
+    reader,
+    integrations,
+    additionalTemplateFilters,
+    additionalTemplateGlobals,
+  } = options;
+
+  const defaultTemplateFilters = createDefaultFilters({ integrations });
+
+  return createTemplateAction<{
+    url: string;
+    targetPath: string;
+    values: any;
+    cookiecutterCompat?: boolean;
+    replace?: boolean;
+    trimBlocks?: boolean;
+    lstripBlocks?: boolean;
+    token?: string;
+  }>({
+    id: 'fetch:template:file',
+    description: 'Downloads single file and places it in the workspace.',
+    examples,
+    schema: {
+      input: {
+        type: 'object',
+        required: ['url', 'targetPath'],
+        properties: {
+          url: {
+            title: 'Fetch URL',
+            description:
+              'Relative path or absolute URL pointing to the single file to fetch.',
+            type: 'string',
+          },
+          targetPath: {
+            title: 'Target Path',
+            description:
+              'Target path within the working directory to download the file as.',
+            type: 'string',
+          },
+          values: {
+            title: 'Template Values',
+            description: 'Values to pass on to the templating engine',
+            type: 'object',
+          },
+          cookiecutterCompat: {
+            title: 'Cookiecutter compatibility mode',
+            description:
+              'Enable features to maximise compatibility with templates built for fetch:cookiecutter',
+            type: 'boolean',
+          },
+          replace: {
+            title: 'Replace file',
+            description:
+              'If set, replace file in targetPath instead of overwriting existing one.',
+            type: 'boolean',
+          },
+          token: {
+            title: 'Token',
+            description:
+              'An optional token to use for authentication when reading the resources.',
+            type: 'string',
+          },
+        },
+      },
+    },
+    supportsDryRun: true,
+    async handler(ctx) {
+      ctx.logger.info('Fetching template file content from remote URL');
+
+      const workDir = await ctx.createTemporaryDirectory();
+      // Write to a tmp file, render the template, then copy to workspace.
+      const tmpFilePath = path.join(workDir, 'tmp');
+
+      const outputPath = resolveSafeChildPath(
+        ctx.workspacePath,
+        ctx.input.targetPath,
+      );
+
+      if (fs.existsSync(outputPath) && !ctx.input.replace) {
+        ctx.logger.info('File already exists in workspace, not replacing.');
+        return;
+      }
+
+      await fetchFile({
+        reader,
+        integrations,
+        baseUrl: ctx.templateInfo?.baseUrl,
+        fetchUrl: ctx.input.url,
+        outputPath: tmpFilePath,
+        token: ctx.input.token,
+      });
+
+      const { cookiecutterCompat, values } = ctx.input;
+      const context = {
+        [cookiecutterCompat ? 'cookiecutter' : 'values']: values,
+      };
+
+      ctx.logger.info(
+        `Processing template file with input values`,
+        ctx.input.values,
+      );
+
+      const renderTemplate = await SecureTemplater.loadRenderer({
+        cookiecutterCompat,
+        templateFilters: {
+          ...defaultTemplateFilters,
+          ...additionalTemplateFilters,
+        },
+        templateGlobals: additionalTemplateGlobals,
+        nunjucksConfigs: {
+          trimBlocks: ctx.input.trimBlocks,
+          lstripBlocks: ctx.input.lstripBlocks,
+        },
+      });
+
+      const contents = await fs.readFile(tmpFilePath, 'utf-8');
+      const result = renderTemplate(contents, context);
+      await fs.ensureDir(path.dirname(outputPath));
+      await fs.outputFile(outputPath, result);
+
+      ctx.logger.info(`Template result written to ${outputPath}`);
+    },
+  });
+}