diff --git a/.changeset/brown-frogs-walk.md b/.changeset/brown-frogs-walk.md new file mode 100644 index 0000000000..6fae44be73 --- /dev/null +++ b/.changeset/brown-frogs-walk.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Add `fetch:template:file` scaffolder action to download a single file and template the contents. Example usage: + +```yaml +- id: fetch-file + name: Fetch File + action: fetch:template:file + input: + url: https://github.com/backstage/software-templates/blob/main/scaffolder-templates/create-react-app/skeleton/catalog-info.yaml + targetPath: './target/catalog-info.yaml' + values: + component_id: My Component + owner: Test +``` diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 00f690c646..4459fa8918 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -202,6 +202,26 @@ export function createFetchTemplateAction(options: { JsonObject >; +// @public +export function createFetchTemplateFileAction(options: { + reader: UrlReaderService; + integrations: ScmIntegrations; + additionalTemplateFilters?: Record; + additionalTemplateGlobals?: Record; +}): TemplateAction_2< + { + url: string; + targetPath: string; + values: any; + cookiecutterCompat?: boolean | undefined; + replace?: boolean | undefined; + trimBlocks?: boolean | undefined; + lstripBlocks?: boolean | undefined; + token?: string | undefined; + }, + JsonObject +>; + // @public export const createFilesystemDeleteAction: () => TemplateAction_2< { diff --git a/plugins/scaffolder-backend/src/ScaffolderPlugin.ts b/plugins/scaffolder-backend/src/ScaffolderPlugin.ts index 27b4f6b196..cf0f557639 100644 --- a/plugins/scaffolder-backend/src/ScaffolderPlugin.ts +++ b/plugins/scaffolder-backend/src/ScaffolderPlugin.ts @@ -44,6 +44,7 @@ import { createFetchPlainAction, createFetchPlainFileAction, createFetchTemplateAction, + createFetchTemplateFileAction, createFilesystemDeleteAction, createFilesystemRenameAction, createWaitAction, @@ -149,6 +150,12 @@ export const scaffolderPlugin = createBackendPlugin({ additionalTemplateFilters, additionalTemplateGlobals, }), + createFetchTemplateFileAction({ + integrations, + reader, + additionalTemplateFilters, + additionalTemplateGlobals, + }), createDebugLogAction(), createWaitAction(), // todo(blam): maybe these should be a -catalog module? diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index dba867c96b..8a2cfde33e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -37,6 +37,7 @@ import { createFetchPlainAction, createFetchPlainFileAction, createFetchTemplateAction, + createFetchTemplateFileAction, } from './fetch'; import { createFilesystemDeleteAction, @@ -157,6 +158,12 @@ export const createBuiltinActions = ( additionalTemplateFilters, additionalTemplateGlobals, }), + createFetchTemplateFileAction({ + integrations, + reader, + additionalTemplateFilters, + additionalTemplateGlobals, + }), createPublishGerritAction({ integrations, config, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts index 67f1bc360f..dd3942a458 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -17,3 +17,4 @@ export { createFetchPlainAction } from './plain'; export { createFetchPlainFileAction } from './plainFile'; export { createFetchTemplateAction } from './template'; +export { createFetchTemplateFileAction } from './templateFile'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.test.ts new file mode 100644 index 0000000000..c7823ee9d8 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { join as joinPath } from 'path'; +import fs from 'fs-extra'; +import { UrlReaderService } from '@backstage/backend-plugin-api'; +import { ScmIntegrations } from '@backstage/integration'; +import { createMockActionContext } from '@backstage/plugin-scaffolder-node-test-utils'; +import { createFetchTemplateFileAction } from './templateFile'; +import { + ActionContext, + TemplateAction, + fetchFile, +} from '@backstage/plugin-scaffolder-node'; +import { examples } from './templateFile.examples'; +import yaml from 'yaml'; +import { createMockDirectory } from '@backstage/backend-test-utils'; + +jest.mock('@backstage/plugin-scaffolder-node', () => ({ + ...jest.requireActual('@backstage/plugin-scaffolder-node'), + fetchFile: jest.fn(), +})); + +type FetchTemplateInput = ReturnType< + typeof createFetchTemplateFileAction +> extends TemplateAction + ? U + : never; + +const mockFetchFile = fetchFile as jest.MockedFunction; + +describe('fetch:template:file examples', () => { + let action: TemplateAction; + + const mockDir = createMockDirectory(); + const workspacePath = mockDir.resolve('workspace'); + + const mockContext = (input: any) => + createMockActionContext({ + templateInfo: { + baseUrl: 'base-url', + entityRef: 'template:default/test-template', + }, + input, + workspacePath, + }); + + beforeEach(() => { + mockDir.clear(); + action = createFetchTemplateFileAction({ + reader: Symbol('UrlReader') as unknown as UrlReaderService, + integrations: Symbol('Integrations') as unknown as ScmIntegrations, + }); + }); + + describe('handler', () => { + describe('with valid input', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext(yaml.parse(examples[0].example).steps[0].input); + + mockFetchFile.mockImplementation(({ outputPath }) => { + mockDir.setContent({ + [outputPath]: + '${{ values.name }}: ${{ values.count }} ${{ values.itemList | dump }}', + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('uses fetchFile to retrieve the template content', () => { + expect(mockFetchFile).toHaveBeenCalledWith( + expect.objectContaining({ + baseUrl: context.templateInfo?.baseUrl, + fetchUrl: context.input.url, + }), + ); + }); + + it('copies files with no templating in names or content successfully', async () => { + await expect( + fs.readFile( + joinPath(workspacePath, context.input.targetPath), + 'utf-8', + ), + ).resolves.toEqual('test-project: 1234 ["first","second","third"]'); + }); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.ts new file mode 100644 index 0000000000..b2f3947f3a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.examples.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TemplateExample } from '@backstage/plugin-scaffolder-node'; +import yaml from 'yaml'; + +export const examples: TemplateExample[] = [ + { + description: 'Downloads a template file and fill it out with values.', + example: yaml.stringify({ + steps: [ + { + action: 'fetch:template:file', + id: 'fetch-template-file', + name: 'Fetch template file', + input: { + url: './skeleton.txt', + targetPath: './target/skeleton.txt', + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + }, + }, + ], + }), + }, +]; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.test.ts new file mode 100644 index 0000000000..23721e685a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.test.ts @@ -0,0 +1,251 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('@backstage/plugin-scaffolder-node', () => { + const actual = jest.requireActual('@backstage/plugin-scaffolder-node'); + return { ...actual, fetchFile: jest.fn() }; +}); + +import { join as joinPath } from 'path'; +import fs from 'fs-extra'; +import { UrlReaderService } from '@backstage/backend-plugin-api'; +import { ScmIntegrations } from '@backstage/integration'; +import { createFetchTemplateFileAction } from './templateFile'; +import { + ActionContext, + TemplateAction, + fetchFile, +} from '@backstage/plugin-scaffolder-node'; +import { createMockDirectory } from '@backstage/backend-test-utils'; +import { createMockActionContext } from '@backstage/plugin-scaffolder-node-test-utils'; + +type FetchTemplateInput = ReturnType< + typeof createFetchTemplateFileAction +> extends TemplateAction + ? U + : never; + +const mockFetchFile = fetchFile as jest.MockedFunction; + +describe('fetch:template:file', () => { + let action: TemplateAction; + + const mockDir = createMockDirectory(); + const workspacePath = mockDir.resolve('workspace'); + const mockContext = (inputPatch: Partial = {}) => + createMockActionContext({ + templateInfo: { + baseUrl: 'base-url', + entityRef: 'template:default/test-template', + }, + input: { + url: './skeleton.txt', + targetPath: './target/skeleton.txt', + values: { + test: 'value', + }, + ...inputPatch, + }, + workspacePath, + }); + + beforeEach(() => { + mockDir.setContent({ + workspace: {}, + }); + action = createFetchTemplateFileAction({ + reader: Symbol('UrlReader') as unknown as UrlReaderService, + integrations: Symbol('Integrations') as unknown as ScmIntegrations, + }); + }); + + it(`returns a TemplateAction with the id 'fetch:template:file'`, () => { + expect(action.id).toEqual('fetch:template:file'); + }); + + describe('handler', () => { + it('should disallow a target path outside working directory', async () => { + await expect( + action.handler(mockContext({ targetPath: '../' })), + ).rejects.toThrow( + /Relative path is not allowed to refer to a directory outside its parent/, + ); + }); + + describe('valid input', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + token: 'mockToken', + }); + + mockFetchFile.mockImplementation(({ outputPath }) => { + mockDir.setContent({ + [outputPath]: + '${{ values.name }}: ${{ values.count }} ${{ values.itemList | dump }}', + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('uses fetchFile to retrieve the template content', () => { + expect(mockFetchFile).toHaveBeenCalledWith( + expect.objectContaining({ + baseUrl: context.templateInfo?.baseUrl, + fetchUrl: context.input.url, + }), + ); + }); + + it('passed through the token to fetchFile', () => { + expect(mockFetchFile).toHaveBeenCalledWith( + expect.objectContaining({ + token: 'mockToken', + }), + ); + }); + + it('templates content successfully', async () => { + await expect( + fs.readFile( + joinPath(workspacePath, context.input.targetPath), + 'utf-8', + ), + ).resolves.toEqual('test-project: 1234 ["first","second","third"]'); + }); + }); + + describe('with replacement of existing files', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + url: './static-content.txt', + targetPath: './target/static-content.txt', + values: { + name: 'test-project', + count: 1234, + }, + replace: true, + }); + + mockDir.setContent({ + [joinPath(workspacePath, 'target')]: { + 'static-content.txt': 'static-content', + }, + }); + + mockFetchFile.mockImplementation(({ outputPath }) => { + mockDir.setContent({ + [outputPath]: '${{ values.name }}: ${{ values.count }}', + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('overwrites existing file', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static-content.txt`, 'utf-8'), + ).resolves.toEqual('test-project: 1234'); + }); + }); + + describe('without replacement of existing files', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + url: './static-content.txt', + targetPath: './target/static-content.txt', + values: { + name: 'test-project', + count: 1234, + }, + replace: false, + }); + + mockDir.setContent({ + [joinPath(workspacePath, 'target')]: { + 'static-content.txt': 'static-content', + }, + }); + + mockFetchFile.mockImplementation(({ outputPath }) => { + mockDir.setContent({ + [outputPath]: '${{ values.name }}: ${{ values.count }}', + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('keeps existing file', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static-content.txt`, 'utf-8'), + ).resolves.toEqual('static-content'); + }); + }); + + describe('cookiecutter compatibility mode', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + targetPath: './target/test-project.txt', + values: { + name: 'test-project', + count: 1234, + itemList: ['first', 'second', 'third'], + }, + cookiecutterCompat: true, + }); + + mockFetchFile.mockImplementation(({ outputPath }) => { + mockDir.setContent({ + [outputPath]: + 'static:{{ cookiecutter.name }}:{{ cookiecutter.count }}:{{ cookiecutter.itemList | jsonify }}', + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('copies files with cookiecutter-style templated variables successfully', async () => { + await expect( + fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'), + ).resolves.toEqual( + 'static:test-project:1234:["first","second","third"]', + ); + }); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.ts new file mode 100644 index 0000000000..89c0d1c1db --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.ts @@ -0,0 +1,167 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReaderService } from '@backstage/backend-plugin-api'; +import { resolveSafeChildPath } from '@backstage/backend-plugin-api'; +import { ScmIntegrations } from '@backstage/integration'; +import { examples } from './plainFile.examples'; +import { + createTemplateAction, + fetchFile, + TemplateFilter, + TemplateGlobal, +} from '@backstage/plugin-scaffolder-node'; +import { SecureTemplater } from '../../../../lib/templating/SecureTemplater'; +import { createDefaultFilters } from '../../../../lib/templating/filters'; +import path from 'path'; +import fs from 'fs-extra'; + +/** + * Downloads a single file and templates variables into file. + * Then places the result in the workspace, or optionally in a subdirectory + * specified by the 'targetPath' input option. + * @public + */ +export function createFetchTemplateFileAction(options: { + reader: UrlReaderService; + integrations: ScmIntegrations; + additionalTemplateFilters?: Record; + additionalTemplateGlobals?: Record; +}) { + const { + reader, + integrations, + additionalTemplateFilters, + additionalTemplateGlobals, + } = options; + + const defaultTemplateFilters = createDefaultFilters({ integrations }); + + return createTemplateAction<{ + url: string; + targetPath: string; + values: any; + cookiecutterCompat?: boolean; + replace?: boolean; + trimBlocks?: boolean; + lstripBlocks?: boolean; + token?: string; + }>({ + id: 'fetch:template:file', + description: 'Downloads single file and places it in the workspace.', + examples, + schema: { + input: { + type: 'object', + required: ['url', 'targetPath'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the single file to fetch.', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the file as.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to the templating engine', + type: 'object', + }, + cookiecutterCompat: { + title: 'Cookiecutter compatibility mode', + description: + 'Enable features to maximise compatibility with templates built for fetch:cookiecutter', + type: 'boolean', + }, + replace: { + title: 'Replace file', + description: + 'If set, replace file in targetPath instead of overwriting existing one.', + type: 'boolean', + }, + token: { + title: 'Token', + description: + 'An optional token to use for authentication when reading the resources.', + type: 'string', + }, + }, + }, + }, + supportsDryRun: true, + async handler(ctx) { + ctx.logger.info('Fetching template file content from remote URL'); + + const workDir = await ctx.createTemporaryDirectory(); + // Write to a tmp file, render the template, then copy to workspace. + const tmpFilePath = path.join(workDir, 'tmp'); + + const outputPath = resolveSafeChildPath( + ctx.workspacePath, + ctx.input.targetPath, + ); + + if (fs.existsSync(outputPath) && !ctx.input.replace) { + ctx.logger.info('File already exists in workspace, not replacing.'); + return; + } + + await fetchFile({ + reader, + integrations, + baseUrl: ctx.templateInfo?.baseUrl, + fetchUrl: ctx.input.url, + outputPath: tmpFilePath, + token: ctx.input.token, + }); + + const { cookiecutterCompat, values } = ctx.input; + const context = { + [cookiecutterCompat ? 'cookiecutter' : 'values']: values, + }; + + ctx.logger.info( + `Processing template file with input values`, + ctx.input.values, + ); + + const renderTemplate = await SecureTemplater.loadRenderer({ + cookiecutterCompat, + templateFilters: { + ...defaultTemplateFilters, + ...additionalTemplateFilters, + }, + templateGlobals: additionalTemplateGlobals, + nunjucksConfigs: { + trimBlocks: ctx.input.trimBlocks, + lstripBlocks: ctx.input.lstripBlocks, + }, + }); + + const contents = await fs.readFile(tmpFilePath, 'utf-8'); + const result = renderTemplate(contents, context); + await fs.ensureDir(path.dirname(outputPath)); + await fs.outputFile(outputPath, result); + + ctx.logger.info(`Template result written to ${outputPath}`); + }, + }); +}