factor out common parts of fetch:template and fetch:template:file actions to respective create*Handler functions

Signed-off-by: Matt Benson <gudnabrsam@gmail.com>
This commit is contained in:
Matt Benson
2025-03-07 23:01:28 -06:00
parent 6b6c8e3586
commit db8cb11d0f
4 changed files with 412 additions and 287 deletions
@@ -14,10 +14,10 @@
* limitations under the License.
*/
import { extname } from 'path';
import { UrlReaderService } from '@backstage/backend-plugin-api';
import { resolveSafeChildPath } from '@backstage/backend-plugin-api';
import { InputError } from '@backstage/errors';
import {
resolveSafeChildPath,
UrlReaderService,
} from '@backstage/backend-plugin-api';
import { ScmIntegrations } from '@backstage/integration';
import {
createTemplateAction,
@@ -25,13 +25,8 @@ import {
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import globby from 'globby';
import fs from 'fs-extra';
import { isBinaryFile } from 'isbinaryfile';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { createDefaultFilters } from '../../../../lib/templating/filters/createDefaultFilters';
import { examples } from './template.examples';
import { convertFiltersToRecord } from '../../../../util/templating';
import { createTemplateActionHandler } from './templateActionHandler';
/**
* Downloads a skeleton, templates variables into file and directory names and content.
@@ -46,17 +41,6 @@ export function createFetchTemplateAction(options: {
additionalTemplateFilters?: Record<string, TemplateFilter>;
additionalTemplateGlobals?: Record<string, TemplateGlobal>;
}) {
const {
reader,
integrations,
additionalTemplateFilters,
additionalTemplateGlobals,
} = options;
const defaultTemplateFilters = convertFiltersToRecord(
createDefaultFilters({ integrations }),
);
return createTemplateAction<{
url: string;
targetPath?: string;
@@ -147,201 +131,24 @@ export function createFetchTemplateAction(options: {
},
},
supportsDryRun: true,
async handler(ctx) {
ctx.logger.info('Fetching template content from remote URL');
handler: createTemplateActionHandler({
resolveTemplate: async ctx => {
ctx.logger.info('Fetching template content from remote URL');
const workDir = await ctx.createTemporaryDirectory();
const templateDir = resolveSafeChildPath(workDir, 'template');
const workDir = await ctx.createTemporaryDirectory();
const templateDir = resolveSafeChildPath(workDir, 'template');
const targetPath = ctx.input.targetPath ?? './';
const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath);
if (ctx.input.copyWithoutRender && ctx.input.copyWithoutTemplating) {
throw new InputError(
'Fetch action input copyWithoutRender and copyWithoutTemplating can not be used at the same time',
);
}
await fetchContents({
baseUrl: ctx.templateInfo?.baseUrl,
fetchUrl: ctx.input.url,
outputPath: templateDir,
token: ctx.input.token,
...options,
});
let copyOnlyPatterns: string[] | undefined;
let renderFilename: boolean;
if (ctx.input.copyWithoutRender) {
ctx.logger.warn(
'[Deprecated] copyWithoutRender is deprecated Please use copyWithoutTemplating instead.',
);
copyOnlyPatterns = ctx.input.copyWithoutRender;
renderFilename = false;
} else {
copyOnlyPatterns = ctx.input.copyWithoutTemplating;
renderFilename = true;
}
if (copyOnlyPatterns && !Array.isArray(copyOnlyPatterns)) {
throw new InputError(
'Fetch action input copyWithoutRender/copyWithoutTemplating must be an Array',
);
}
if (
ctx.input.templateFileExtension &&
(copyOnlyPatterns || ctx.input.cookiecutterCompat)
) {
throw new InputError(
'Fetch action input extension incompatible with copyWithoutRender/copyWithoutTemplating and cookiecutterCompat',
);
}
let extension: string | false = false;
if (ctx.input.templateFileExtension) {
extension =
ctx.input.templateFileExtension === true
? '.njk'
: ctx.input.templateFileExtension;
if (!extension.startsWith('.')) {
extension = `.${extension}`;
}
}
await fetchContents({
reader,
integrations,
baseUrl: ctx.templateInfo?.baseUrl,
fetchUrl: ctx.input.url,
outputPath: templateDir,
token: ctx.input.token,
});
ctx.logger.info('Listing files and directories in template');
const allEntriesInTemplate = await globby(`**/*`, {
cwd: templateDir,
dot: true,
onlyFiles: false,
markDirectories: true,
followSymbolicLinks: false,
});
const nonTemplatedEntries = new Set(
await globby(copyOnlyPatterns || [], {
cwd: templateDir,
dot: true,
onlyFiles: false,
markDirectories: true,
followSymbolicLinks: false,
}),
);
// Cookiecutter prefixes all parameters in templates with
// `cookiecutter.`. To replicate this, we wrap our parameters
// in an object with a `cookiecutter` property when compat
// mode is enabled.
const { cookiecutterCompat, values } = ctx.input;
const context = {
[cookiecutterCompat ? 'cookiecutter' : 'values']: values,
};
ctx.logger.info(
`Processing ${allEntriesInTemplate.length} template files/directories with input values`,
ctx.input.values,
);
const renderTemplate = await SecureTemplater.loadRenderer({
cookiecutterCompat: ctx.input.cookiecutterCompat,
templateFilters: {
...defaultTemplateFilters,
...(additionalTemplateFilters ?? {}),
},
templateGlobals: additionalTemplateGlobals,
nunjucksConfigs: {
trimBlocks: ctx.input.trimBlocks,
lstripBlocks: ctx.input.lstripBlocks,
},
});
for (const location of allEntriesInTemplate) {
let renderContents: boolean;
let localOutputPath = location;
if (extension) {
renderContents = extname(localOutputPath) === extension;
if (renderContents) {
localOutputPath = localOutputPath.slice(0, -extension.length);
}
// extension is mutual exclusive with copyWithoutRender/copyWithoutTemplating,
// therefore the output path is always rendered.
localOutputPath = renderTemplate(localOutputPath, context);
} else {
renderContents = !nonTemplatedEntries.has(location);
// The logic here is a bit tangled because it depends on two variables.
// If renderFilename is true, which means copyWithoutTemplating is used,
// then the path is always rendered.
// If renderFilename is false, which means copyWithoutRender is used,
// then matched file/directory won't be processed, same as before.
if (renderFilename) {
localOutputPath = renderTemplate(localOutputPath, context);
} else {
localOutputPath = renderContents
? renderTemplate(localOutputPath, context)
: localOutputPath;
}
}
if (containsSkippedContent(localOutputPath)) {
continue;
}
const outputPath = resolveSafeChildPath(outputDir, localOutputPath);
if (fs.existsSync(outputPath) && !ctx.input.replace) {
continue;
}
if (!renderContents && !extension) {
ctx.logger.info(
`Copying file/directory ${location} without processing.`,
);
}
if (location.endsWith('/')) {
ctx.logger.info(
`Writing directory ${location} to template output path.`,
);
await fs.ensureDir(outputPath);
} else {
const inputFilePath = resolveSafeChildPath(templateDir, location);
const stats = await fs.promises.lstat(inputFilePath);
if (stats.isSymbolicLink() || (await isBinaryFile(inputFilePath))) {
ctx.logger.info(
`Copying file binary or symbolic link at ${location}, to template output path.`,
);
await fs.copy(inputFilePath, outputPath);
} else {
const statsObj = await fs.stat(inputFilePath);
ctx.logger.info(
`Writing file ${location} to template output path with mode ${statsObj.mode}.`,
);
const inputFileContents = await fs.readFile(inputFilePath, 'utf-8');
await fs.outputFile(
outputPath,
renderContents
? renderTemplate(inputFileContents, context)
: inputFileContents,
{ mode: statsObj.mode },
);
}
}
}
ctx.logger.info(`Template result written to ${outputDir}`);
},
return templateDir;
},
...options,
}),
});
}
function containsSkippedContent(localOutputPath: string): boolean {
// if the path is empty means that there is a file skipped in the root
// if the path starts with a separator it means that the root directory has been skipped
// if the path includes // means that there is a subdirectory skipped
// All paths returned are considered with / separator because of globby returning the linux separator for all os'.
return (
localOutputPath === '' ||
localOutputPath.startsWith('/') ||
localOutputPath.includes('//')
);
}
@@ -0,0 +1,272 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
isChildPath,
resolveSafeChildPath,
} from '@backstage/backend-plugin-api';
import { InputError } from '@backstage/errors';
import { ScmIntegrations } from '@backstage/integration';
import {
ActionContext,
TemplateActionOptions,
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import fs from 'fs-extra';
import globby from 'globby';
import { isBinaryFile } from 'isbinaryfile';
import { createDefaultFilters } from '../../../../lib/templating/filters/createDefaultFilters';
import { convertFiltersToRecord } from '../../../../util/templating';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { extname } from 'path';
type TemplateActionInput = {
targetPath?: string;
values: any;
templateFileExtension?: string | boolean;
// Cookiecutter compat options
/**
* @deprecated This field is deprecated in favor of copyWithoutTemplating.
*/
copyWithoutRender?: string[];
copyWithoutTemplating?: string[];
cookiecutterCompat?: boolean;
replace?: boolean;
trimBlocks?: boolean;
lstripBlocks?: boolean;
};
export function createTemplateActionHandler<
I extends TemplateActionInput = TemplateActionInput,
>(options: {
resolveTemplate: (ctx: ActionContext<I, any, any>) => Promise<string>;
integrations: ScmIntegrations;
additionalTemplateFilters?: Record<string, TemplateFilter>;
additionalTemplateGlobals?: Record<string, TemplateGlobal>;
}): TemplateActionOptions<I>['handler'] {
const {
resolveTemplate,
integrations,
additionalTemplateFilters,
additionalTemplateGlobals: templateGlobals,
} = options;
const templateFilters = {
...convertFiltersToRecord(createDefaultFilters({ integrations })),
...additionalTemplateFilters,
};
return async (ctx: ActionContext<I, any, any>) => {
const { outputDir, copyOnlyPatterns, renderFilename, extension } =
resolveTemplateActionSettings(ctx);
const templateDir = await resolveTemplate(ctx);
if (isChildPath(templateDir, outputDir)) {
throw new InputError('targetPath must not be within template path');
}
ctx.logger.info('Listing files and directories in template');
const allEntriesInTemplate = await globby(`**/*`, {
cwd: templateDir,
dot: true,
onlyFiles: false,
markDirectories: true,
followSymbolicLinks: false,
});
const nonTemplatedEntries = new Set(
await globby(copyOnlyPatterns || [], {
cwd: templateDir,
dot: true,
onlyFiles: false,
markDirectories: true,
followSymbolicLinks: false,
}),
);
// Cookiecutter prefixes all parameters in templates with
// `cookiecutter.`. To replicate this, we wrap our parameters
// in an object with a `cookiecutter` property when compat
// mode is enabled.
const { cookiecutterCompat, values } = ctx.input;
const context = {
[cookiecutterCompat ? 'cookiecutter' : 'values']: values,
};
ctx.logger.info(
`Processing ${allEntriesInTemplate.length} template files/directories with input values`,
ctx.input.values,
);
const renderTemplate = await SecureTemplater.loadRenderer({
cookiecutterCompat: ctx.input.cookiecutterCompat,
templateFilters,
templateGlobals,
nunjucksConfigs: {
trimBlocks: ctx.input.trimBlocks,
lstripBlocks: ctx.input.lstripBlocks,
},
});
for (const location of allEntriesInTemplate) {
let renderContents: boolean;
let localOutputPath = location;
if (extension) {
renderContents = extname(localOutputPath) === extension;
if (renderContents) {
localOutputPath = localOutputPath.slice(0, -extension.length);
}
// extension is mutual exclusive with copyWithoutRender/copyWithoutTemplating,
// therefore the output path is always rendered.
localOutputPath = renderTemplate(localOutputPath, context);
} else {
renderContents = !nonTemplatedEntries.has(location);
// The logic here is a bit tangled because it depends on two variables.
// If renderFilename is true, which means copyWithoutTemplating is used,
// then the path is always rendered.
// If renderFilename is false, which means copyWithoutRender is used,
// then matched file/directory won't be processed, same as before.
if (renderFilename) {
localOutputPath = renderTemplate(localOutputPath, context);
} else {
localOutputPath = renderContents
? renderTemplate(localOutputPath, context)
: localOutputPath;
}
}
if (containsSkippedContent(localOutputPath)) {
continue;
}
const outputPath = resolveSafeChildPath(outputDir, localOutputPath);
if (fs.existsSync(outputPath) && !ctx.input.replace) {
continue;
}
if (!renderContents && !extension) {
ctx.logger.info(
`Copying file/directory ${location} without processing.`,
);
}
if (location.endsWith('/')) {
ctx.logger.info(
`Writing directory ${location} to template output path.`,
);
await fs.ensureDir(outputPath);
} else {
const inputFilePath = resolveSafeChildPath(templateDir, location);
const stats = await fs.promises.lstat(inputFilePath);
if (stats.isSymbolicLink() || (await isBinaryFile(inputFilePath))) {
ctx.logger.info(
`Copying file binary or symbolic link at ${location}, to template output path.`,
);
await fs.copy(inputFilePath, outputPath);
} else {
const statsObj = await fs.stat(inputFilePath);
ctx.logger.info(
`Writing file ${location} to template output path with mode ${statsObj.mode}.`,
);
const inputFileContents = await fs.readFile(inputFilePath, 'utf-8');
await fs.outputFile(
outputPath,
renderContents
? renderTemplate(inputFileContents, context)
: inputFileContents,
{ mode: statsObj.mode },
);
}
}
}
ctx.logger.info(`Template result written to ${outputDir}`);
};
}
function resolveTemplateActionSettings<I extends TemplateActionInput>(
ctx: ActionContext<I, any, any>,
): {
outputDir: string;
copyOnlyPatterns?: string[];
renderFilename: boolean;
extension: string | false;
} {
const targetPath = ctx.input.targetPath ?? './';
const outputDir = resolveSafeChildPath(ctx.workspacePath, targetPath);
if (ctx.input.copyWithoutRender && ctx.input.copyWithoutTemplating) {
throw new InputError(
'Fetch action input copyWithoutRender and copyWithoutTemplating can not be used at the same time',
);
}
let copyOnlyPatterns: string[] | undefined;
let renderFilename: boolean;
if (ctx.input.copyWithoutRender) {
ctx.logger.warn(
'[Deprecated] copyWithoutRender is deprecated Please use copyWithoutTemplating instead.',
);
copyOnlyPatterns = ctx.input.copyWithoutRender;
renderFilename = false;
} else {
copyOnlyPatterns = ctx.input.copyWithoutTemplating;
renderFilename = true;
}
if (copyOnlyPatterns && !Array.isArray(copyOnlyPatterns)) {
throw new InputError(
'Fetch action input copyWithoutRender/copyWithoutTemplating must be an Array',
);
}
if (
ctx.input.templateFileExtension &&
(copyOnlyPatterns || ctx.input.cookiecutterCompat)
) {
throw new InputError(
'Fetch action input extension incompatible with copyWithoutRender/copyWithoutTemplating and cookiecutterCompat',
);
}
let extension: string | false = false;
if (ctx.input.templateFileExtension) {
extension =
ctx.input.templateFileExtension === true
? '.njk'
: ctx.input.templateFileExtension;
if (!extension.startsWith('.')) {
extension = `.${extension}`;
}
}
return {
outputDir,
copyOnlyPatterns,
renderFilename,
extension,
};
}
function containsSkippedContent(localOutputPath: string): boolean {
// if the path is empty means that there is a file skipped in the root
// if the path starts with a separator it means that the root directory has been skipped
// if the path includes // means that there is a subdirectory skipped
// All paths returned are considered with / separator because of globby returning the linux separator for all os'.
return (
localOutputPath === '' ||
localOutputPath.startsWith('/') ||
localOutputPath.includes('//')
);
}
@@ -15,20 +15,16 @@
*/
import { UrlReaderService } from '@backstage/backend-plugin-api';
import { resolveSafeChildPath } from '@backstage/backend-plugin-api';
import { ScmIntegrations } from '@backstage/integration';
import { examples } from './templateFile.examples';
import {
createTemplateAction,
fetchFile,
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { createDefaultFilters } from '../../../../lib/templating/filters/createDefaultFilters';
import path from 'path';
import fs from 'fs-extra';
import { convertFiltersToRecord } from '../../../../util/templating';
import { examples } from './templateFile.examples';
import { createTemplateFileActionHandler } from './templateFileActionHandler';
/**
* Downloads a single file and templates variables into file.
@@ -42,17 +38,6 @@ export function createFetchTemplateFileAction(options: {
additionalTemplateFilters?: Record<string, TemplateFilter>;
additionalTemplateGlobals?: Record<string, TemplateGlobal>;
}) {
const {
reader,
integrations,
additionalTemplateFilters,
additionalTemplateGlobals,
} = options;
const defaultTemplateFilters = convertFiltersToRecord(
createDefaultFilters({ integrations }),
);
return createTemplateAction<{
url: string;
targetPath: string;
@@ -110,63 +95,24 @@ export function createFetchTemplateFileAction(options: {
},
},
supportsDryRun: true,
async handler(ctx) {
ctx.logger.info('Fetching template file content from remote URL');
handler: createTemplateFileActionHandler({
resolveTemplateFile: async ctx => {
ctx.logger.info('Fetching template file content from remote URL');
const workDir = await ctx.createTemporaryDirectory();
// Write to a tmp file, render the template, then copy to workspace.
const tmpFilePath = path.join(workDir, 'tmp');
const workDir = await ctx.createTemporaryDirectory();
// Write to a tmp file, render the template, then copy to workspace.
const tmpFilePath = path.join(workDir, 'tmp');
const outputPath = resolveSafeChildPath(
ctx.workspacePath,
ctx.input.targetPath,
);
if (fs.existsSync(outputPath) && !ctx.input.replace) {
ctx.logger.info(
`File ${ctx.input.targetPath} already exists in workspace, not replacing.`,
);
return;
}
await fetchFile({
reader,
integrations,
baseUrl: ctx.templateInfo?.baseUrl,
fetchUrl: ctx.input.url,
outputPath: tmpFilePath,
token: ctx.input.token,
});
const { cookiecutterCompat, values } = ctx.input;
const context = {
[cookiecutterCompat ? 'cookiecutter' : 'values']: values,
};
ctx.logger.info(
`Processing template file with input values`,
ctx.input.values,
);
const renderTemplate = await SecureTemplater.loadRenderer({
cookiecutterCompat,
templateFilters: {
...defaultTemplateFilters,
...additionalTemplateFilters,
},
templateGlobals: additionalTemplateGlobals,
nunjucksConfigs: {
trimBlocks: ctx.input.trimBlocks,
lstripBlocks: ctx.input.lstripBlocks,
},
});
const contents = await fs.readFile(tmpFilePath, 'utf-8');
const result = renderTemplate(contents, context);
await fs.ensureDir(path.dirname(outputPath));
await fs.outputFile(outputPath, result);
ctx.logger.info(`Template file has been written to ${outputPath}`);
},
await fetchFile({
baseUrl: ctx.templateInfo?.baseUrl,
fetchUrl: ctx.input.url,
outputPath: tmpFilePath,
token: ctx.input.token,
...options,
});
return tmpFilePath;
},
...options,
}),
});
}
@@ -0,0 +1,100 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ScmIntegrations } from '@backstage/integration';
import {
ActionContext,
TemplateActionOptions,
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import fs from 'fs-extra';
import { createDefaultFilters } from '../../../../lib/templating/filters/createDefaultFilters';
import { convertFiltersToRecord } from '../../../../util/templating';
import { resolveSafeChildPath } from '@backstage/backend-plugin-api';
import path from 'path';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
export type TemplateFileActionInput = {
targetPath: string;
values: any;
cookiecutterCompat?: boolean;
replace?: boolean;
trimBlocks?: boolean;
lstripBlocks?: boolean;
};
export function createTemplateFileActionHandler<
I extends TemplateFileActionInput = TemplateFileActionInput,
>(options: {
resolveTemplateFile: (ctx: ActionContext<I, any, any>) => Promise<string>;
integrations: ScmIntegrations;
additionalTemplateFilters?: Record<string, TemplateFilter>;
additionalTemplateGlobals?: Record<string, TemplateGlobal>;
}): TemplateActionOptions<I>['handler'] {
const {
resolveTemplateFile,
integrations,
additionalTemplateFilters,
additionalTemplateGlobals: templateGlobals,
} = options;
const templateFilters = {
...convertFiltersToRecord(createDefaultFilters({ integrations })),
...additionalTemplateFilters,
};
return async (ctx: ActionContext<I, any, any>) => {
const outputPath = resolveSafeChildPath(
ctx.workspacePath,
ctx.input.targetPath,
);
if (fs.existsSync(outputPath) && !ctx.input.replace) {
ctx.logger.info(
`File ${ctx.input.targetPath} already exists in workspace, not replacing.`,
);
return;
}
const filePath = await resolveTemplateFile(ctx);
const { cookiecutterCompat, values } = ctx.input;
const context = {
[cookiecutterCompat ? 'cookiecutter' : 'values']: values,
};
ctx.logger.info(
`Processing template file with input values`,
ctx.input.values,
);
const renderTemplate = await SecureTemplater.loadRenderer({
cookiecutterCompat,
templateFilters,
templateGlobals,
nunjucksConfigs: {
trimBlocks: ctx.input.trimBlocks,
lstripBlocks: ctx.input.lstripBlocks,
},
});
const contents = await fs.readFile(filePath, 'utf-8');
const result = renderTemplate(contents, context);
await fs.ensureDir(path.dirname(outputPath));
await fs.outputFile(outputPath, result);
ctx.logger.info(`Template file has been written to ${outputPath}`);
};
}