Merge pull request #8235 from backstage/mob/secure-templating

scaffolder-backend: sandbox v1beta3 template execution
This commit is contained in:
Johan Haals
2021-11-25 11:51:22 +01:00
committed by GitHub
11 changed files with 10961 additions and 89 deletions
File diff suppressed because it is too large Load Diff
+7 -3
View File
@@ -27,7 +27,8 @@
"test": "backstage-cli test",
"prepack": "backstage-cli prepack",
"postpack": "backstage-cli postpack",
"clean": "backstage-cli clean"
"clean": "backstage-cli clean",
"build:assets": "node scripts/build-nunjucks.js"
},
"dependencies": {
"@backstage/backend-common": "^0.9.10",
@@ -68,7 +69,8 @@
"octokit-plugin-create-pull-request": "^3.10.0",
"uuid": "^8.2.0",
"winston": "^3.2.1",
"yaml": "^1.10.0"
"yaml": "^1.10.0",
"vm2": "^3.9.5"
},
"devDependencies": {
"@backstage/cli": "^0.9.0",
@@ -79,6 +81,7 @@
"@types/mock-fs": "^4.13.0",
"@types/nunjucks": "^3.1.4",
"@types/supertest": "^2.0.8",
"esbuild": "^0.13.14",
"jest-when": "^3.1.0",
"mock-fs": "^5.1.0",
"msw": "^0.35.0",
@@ -88,7 +91,8 @@
"files": [
"dist",
"migrations",
"config.d.ts"
"config.d.ts",
"assets"
],
"configSchema": "config.d.ts"
}
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env node
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable no-restricted-syntax */
/* eslint-disable import/no-extraneous-dependencies */
const path = require('path');
const NUNJUCKS_LICENSE = `/**
* Copyright (c) 2012-2015, James Long
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
`;
// This script is used to bundle nunjucks into a single script that is
// loaded into a sandbox for executing templates.
require('esbuild')
.build({
entryPoints: [require.resolve('nunjucks')],
bundle: true,
format: 'cjs',
platform: 'node',
target: 'node14',
banner: { js: NUNJUCKS_LICENSE },
external: ['fsevents'],
outfile: path.resolve(__dirname, '../assets/nunjucks.js.txt'),
})
.catch(err => {
console.log(err.stack);
process.exit(1);
});
@@ -0,0 +1,145 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { SecureTemplater } from './SecureTemplater';
describe('SecureTemplater', () => {
it('should render some templates', async () => {
const render = await SecureTemplater.loadRenderer();
expect(render('${{ test }}', { test: 'my-value' })).toBe('my-value');
expect(render('${{ test | dump }}', { test: 'my-value' })).toBe(
'"my-value"',
);
expect(
render('${{ test | replace("my-", "our-") }}', {
test: 'my-value',
}),
).toBe('our-value');
expect(() =>
render('${{ invalid...syntax }}', {
test: 'my-value',
}),
).toThrow(/expected name as lookup value, got ./);
});
it('should make cookiecutter compatibility available when requested', async () => {
const renderWith = await SecureTemplater.loadRenderer({
cookiecutterCompat: true,
});
const renderWithout = await SecureTemplater.loadRenderer();
// Same two tests repeated to make sure switching back and forth works
expect(renderWith('{{ 1 | jsonify }}', {})).toBe('1');
expect(renderWith('{{ 1 | jsonify }}', {})).toBe('1');
expect(() => renderWithout('${{ 1 | jsonify }}', {})).toThrow(
/Error: filter not found: jsonify/,
);
expect(renderWith('{{ 1 | jsonify }}', {})).toBe('1');
expect(() => renderWithout('${{ 1 | jsonify }}', {})).toThrow(
/Error: filter not found: jsonify/,
);
expect(() => renderWithout('${{ 1 | jsonify }}', {})).toThrow(
/Error: filter not found: jsonify/,
);
expect(() => renderWithout('${{ 1 | jsonify }}', {})).toThrow(
/Error: filter not found: jsonify/,
);
expect(renderWith('{{ 1 | jsonify }}', {})).toBe('1');
});
it('should make parseRepoUrl available when requested', async () => {
const parseRepoUrl = jest.fn(() => ({
repo: 'my-repo',
owner: 'my-owner',
host: 'my-host.com',
}));
const renderWith = await SecureTemplater.loadRenderer({ parseRepoUrl });
const renderWithout = await SecureTemplater.loadRenderer();
const ctx = {
repoUrl: 'https://my-host.com/my-owner/my-repo',
};
expect(renderWith('${{ repoUrl | parseRepoUrl | dump }}', ctx)).toBe(
JSON.stringify({
repo: 'my-repo',
owner: 'my-owner',
host: 'my-host.com',
}),
);
expect(renderWith('${{ repoUrl | projectSlug }}', ctx)).toBe(
'my-owner/my-repo',
);
expect(() =>
renderWithout('${{ repoUrl | parseRepoUrl | dump }}', ctx),
).toThrow(/Error: filter not found: parseRepoUrl/);
expect(() => renderWithout('${{ repoUrl | projectSlug }}', ctx)).toThrow(
/Error: filter not found: projectSlug/,
);
expect(parseRepoUrl.mock.calls).toEqual([
['https://my-host.com/my-owner/my-repo'],
['https://my-host.com/my-owner/my-repo'],
]);
});
it('should not allow helpers to be rewritten', async () => {
const render = await SecureTemplater.loadRenderer({
parseRepoUrl: () => ({
repo: 'my-repo',
owner: 'my-owner',
host: 'my-host.com',
}),
});
const ctx = {
repoUrl: 'https://my-host.com/my-owner/my-repo',
};
expect(
render(
'${{ ({}).constructor.constructor("parseRepoUrl = () => JSON.stringify(`inject`)")() }}',
ctx,
),
).toBe('');
expect(render('${{ repoUrl | parseRepoUrl | dump }}', ctx)).toBe(
JSON.stringify({
repo: 'my-repo',
owner: 'my-owner',
host: 'my-host.com',
}),
);
});
it('allows pollution during a single template execution', async () => {
const render = await SecureTemplater.loadRenderer();
const ctx = {
x: 'foo',
};
expect(render('${{ x }}', ctx)).toBe('foo');
expect(
render(
'${{ ({}).constructor.constructor("Array.prototype.forEach = () => {}")() }}',
ctx,
),
).toBe('');
expect(() => render('${{ x }}', ctx)).toThrow();
});
});
@@ -0,0 +1,141 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { VM } from 'vm2';
import { resolvePackagePath } from '@backstage/backend-common';
import fs from 'fs-extra';
import { RepoSpec } from '../../scaffolder/actions/builtin/publish/util';
const mkScript = (nunjucksSource: string) => `
const { render, renderCompat } = (() => {
const module = {};
const process = { env: {} };
const require = (pkg) => { if (pkg === 'events') { return function (){}; }};
${nunjucksSource}
const env = module.exports.configure({
autoescape: false,
tags: {
variableStart: '\${{',
variableEnd: '}}',
},
});
const compatEnv = module.exports.configure({
autoescape: false,
tags: {
variableStart: '{{',
variableEnd: '}}',
},
});
compatEnv.addFilter('jsonify', compatEnv.getFilter('dump'));
if (typeof parseRepoUrl !== 'undefined') {
const safeHelperRef = parseRepoUrl;
env.addFilter('parseRepoUrl', repoUrl => {
return JSON.parse(safeHelperRef(repoUrl))
});
env.addFilter('projectSlug', repoUrl => {
const { owner, repo } = JSON.parse(safeHelperRef(repoUrl));
return owner + '/' + repo;
});
}
let uninstallCompat = undefined;
function render(str, values) {
try {
if (uninstallCompat) {
uninstallCompat();
uninstallCompat = undefined;
}
return env.renderString(str, JSON.parse(values));
} catch (error) {
// Make sure errors don't leak anything
throw new Error(String(error.message));
}
}
function renderCompat(str, values) {
try {
if (!uninstallCompat) {
uninstallCompat = module.exports.installJinjaCompat();
}
return compatEnv.renderString(str, JSON.parse(values));
} catch (error) {
// Make sure errors don't leak anything
throw new Error(String(error.message));
}
}
return { render, renderCompat };
})();
`;
export interface SecureTemplaterOptions {
/* Optional implementation of the parseRepoUrl filter */
parseRepoUrl?(repoUrl: string): RepoSpec;
/* Enables jinja compatibility and the "jsonify" filter */
cookiecutterCompat?: boolean;
}
export type SecureTemplateRenderer = (
template: string,
values: unknown,
) => string;
export class SecureTemplater {
static async loadRenderer(options: SecureTemplaterOptions = {}) {
const { parseRepoUrl, cookiecutterCompat } = options;
let sandbox = undefined;
if (parseRepoUrl) {
sandbox = {
parseRepoUrl: (url: string) => JSON.stringify(parseRepoUrl(url)),
};
}
const vm = new VM({ sandbox });
const nunjucksSource = await fs.readFile(
resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
'assets/nunjucks.js.txt',
),
'utf-8',
);
vm.run(mkScript(nunjucksSource));
const render: SecureTemplateRenderer = (template, values) => {
if (!vm) {
throw new Error('SecureTemplater has not been initialized');
}
vm.setGlobal('templateStr', template);
vm.setGlobal('templateValues', JSON.stringify(values));
if (cookiecutterCompat) {
return vm.run(`renderCompat(templateStr, templateValues)`);
}
return vm.run(`render(templateStr, templateValues)`);
};
return render;
}
}
@@ -33,6 +33,17 @@ jest.mock('./helpers', () => ({
fetchContents: jest.fn(),
}));
const realFiles = Object.fromEntries(
[
require.resolve('vm2/lib/fixasync'),
resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
'assets',
'nunjucks.js.txt',
),
].map(k => [k, mockFs.load(k)]),
);
const aBinaryFile = fs.readFileSync(
resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
@@ -76,7 +87,9 @@ describe('fetch:template', () => {
});
beforeEach(() => {
mockFs();
mockFs({
...realFiles,
});
action = createFetchTemplateAction({
reader: Symbol('UrlReader') as unknown as UrlReader,
@@ -150,6 +163,7 @@ describe('fetch:template', () => {
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
...realFiles,
[outputPath]: {
'an-executable.sh': mockFs.file({
content: '#!/usr/bin/env bash',
@@ -259,6 +273,7 @@ describe('fetch:template', () => {
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
...realFiles,
[outputPath]: {
processed: {
'templated-content-${{ values.name }}.txt':
@@ -312,6 +327,7 @@ describe('fetch:template', () => {
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
...realFiles,
[outputPath]: {
'{{ cookiecutter.name }}.txt': 'static content',
subdir: {
@@ -366,6 +382,7 @@ describe('fetch:template', () => {
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
...realFiles,
[outputPath]: {
'empty-dir-${{ values.count }}': {},
'static.txt': 'static content',
@@ -447,6 +464,7 @@ describe('fetch:template', () => {
mockFetchContents.mockImplementation(({ outputPath }) => {
mockFs({
...realFiles,
[outputPath]: {
'${{ values.name }}.njk': '${{ values.name }}: ${{ values.count }}',
'${{ values.name }}.txt.jinja2':
@@ -21,22 +21,9 @@ import { ScmIntegrations } from '@backstage/integration';
import { fetchContents } from './helpers';
import { createTemplateAction } from '../../createTemplateAction';
import globby from 'globby';
import nunjucks from 'nunjucks';
import fs from 'fs-extra';
import { isBinaryFile } from 'isbinaryfile';
/*
* Maximise compatibility with Jinja (and therefore cookiecutter)
* using nunjucks jinja compat mode. Since this method mutates
* the global nunjucks instance, we can't enable this per-template,
* or only for templates with cookiecutter compat enabled, so the
* next best option is to explicitly enable it globally and allow
* folks to rely on jinja compatibility behaviour in fetch:template
* templates if they wish.
*
* cf. https://mozilla.github.io/nunjucks/api.html#installjinjacompat
*/
nunjucks.installJinjaCompat();
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
type CookieCompatInput = {
copyWithoutRender?: string[];
@@ -179,36 +166,6 @@ export function createFetchTemplateAction(options: {
).flat(),
);
// Create a templater
const templater = nunjucks.configure({
...(ctx.input.cookiecutterCompat
? {}
: {
tags: {
// TODO(mtlewis/orkohunter): Document Why we are changing the literals? Not here, but on scaffolder docs. ADR?
variableStart: '${{',
variableEnd: '}}',
},
}),
// We don't want this builtin auto-escaping, since uses HTML escape sequences
// like `"` - the correct way to escape strings in our case depends on
// the file type.
autoescape: false,
});
if (ctx.input.cookiecutterCompat) {
// The "jsonify" filter built into cookiecutter is common
// in fetch:cookiecutter templates, so when compat mode
// is enabled we alias the "dump" filter from nunjucks as
// jsonify. Dump accepts an optional `spaces` parameter
// which enables indented output, but when this parameter
// is not supplied it works identically to jsonify.
//
// cf. https://cookiecutter.readthedocs.io/en/latest/advanced/template_extensions.html?highlight=jsonify#jsonify-extension
// cf. https://mozilla.github.io/nunjucks/templating.html#dump
templater.addFilter('jsonify', templater.getFilter('dump'));
}
// Cookiecutter prefixes all parameters in templates with
// `cookiecutter.`. To replicate this, we wrap our parameters
// in an object with a `cookiecutter` property when compat
@@ -223,6 +180,10 @@ export function createFetchTemplateAction(options: {
ctx.input.values,
);
const renderTemplate = await SecureTemplater.loadRenderer({
cookiecutterCompat: ctx.input.cookiecutterCompat,
});
for (const location of allEntriesInTemplate) {
let renderFilename: boolean;
let renderContents: boolean;
@@ -238,7 +199,7 @@ export function createFetchTemplateAction(options: {
renderFilename = renderContents = !nonTemplatedEntries.has(location);
}
if (renderFilename) {
localOutputPath = templater.renderString(localOutputPath, context);
localOutputPath = renderTemplate(localOutputPath, context);
}
const outputPath = resolveSafeChildPath(outputDir, localOutputPath);
// variables have been expanded to make an empty file name
@@ -275,7 +236,7 @@ export function createFetchTemplateAction(options: {
await fs.outputFile(
outputPath,
renderContents
? templater.renderString(inputFileContents, context)
? renderTemplate(inputFileContents, context)
: inputFileContents,
{ mode: statsObj.mode },
);
@@ -17,13 +17,24 @@
import mockFs from 'mock-fs';
import * as winston from 'winston';
import { getVoidLogger } from '@backstage/backend-common';
import { getVoidLogger, resolvePackagePath } from '@backstage/backend-common';
import { NunjucksWorkflowRunner } from './NunjucksWorkflowRunner';
import { TemplateActionRegistry } from '../actions';
import { ScmIntegrations } from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
import { TaskContext, TaskSpec } from './types';
const realFiles = Object.fromEntries(
[
require.resolve('vm2/lib/fixasync'),
resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
'assets',
'nunjucks.js.txt',
),
].map(k => [k, mockFs.load(k)]),
);
describe('DefaultWorkflowRunner', () => {
const logger = getVoidLogger();
let actionRegistry = new TemplateActionRegistry();
@@ -50,6 +61,7 @@ describe('DefaultWorkflowRunner', () => {
winston.format.simple(); // put logform the require cache before mocking fs
mockFs({
'/tmp': mockFs.directory(),
...realFiles,
});
jest.resetAllMocks();
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ScmIntegrations } from '@backstage/integration';
import {
TaskContext,
@@ -23,9 +24,9 @@ import {
WorkflowRunner,
} from './types';
import * as winston from 'winston';
import nunjucks from 'nunjucks';
import fs from 'fs-extra';
import path from 'path';
import nunjucks from 'nunjucks';
import { JsonObject, JsonValue } from '@backstage/types';
import { InputError } from '@backstage/errors';
import { PassThrough } from 'stream';
@@ -33,6 +34,10 @@ import { isTruthy } from './helper';
import { validate as validateJsonSchema } from 'jsonschema';
import { parseRepoUrl } from '../actions/builtin/publish/util';
import { TemplateActionRegistry } from '../actions';
import {
SecureTemplater,
SecureTemplateRenderer,
} from '../../lib/templating/SecureTemplater';
type NunjucksWorkflowRunnerOptions = {
workingDirectory: string;
@@ -84,36 +89,31 @@ const createStepLogger = ({
};
export class NunjucksWorkflowRunner implements WorkflowRunner {
private readonly nunjucks: nunjucks.Environment;
private readonly nunjucksOptions: nunjucks.ConfigureOptions = {
autoescape: false,
tags: {
variableStart: '${{',
variableEnd: '}}',
},
};
constructor(private readonly options: NunjucksWorkflowRunnerOptions) {
this.nunjucks = nunjucks.configure(this.nunjucksOptions);
// TODO(blam): let's work out how we can deprecate these.
// We shouldn't really need to be exposing these now we can deal with
// objects in the params block.
// Maybe we can expose a new RepoUrlPicker with secrets for V3 that provides an object already.
this.nunjucks.addFilter('parseRepoUrl', repoUrl => {
return parseRepoUrl(repoUrl, this.options.integrations);
});
this.nunjucks.addFilter('projectSlug', repoUrl => {
const { owner, repo } = parseRepoUrl(repoUrl, this.options.integrations);
return `${owner}/${repo}`;
});
}
constructor(private readonly options: NunjucksWorkflowRunnerOptions) {}
private isSingleTemplateString(input: string) {
const { parser, nodes } = require('nunjucks');
const parsed = parser.parse(input, {}, this.nunjucksOptions);
const { parser, nodes } = nunjucks as unknown as {
parser: {
parse(
template: string,
ctx: object,
options: nunjucks.ConfigureOptions,
): { children: { children?: unknown[] }[] };
};
nodes: { TemplateData: Function };
};
const parsed = parser.parse(
input,
{},
{
autoescape: false,
tags: {
variableStart: '${{',
variableEnd: '}}',
},
},
);
return (
parsed.children.length === 1 &&
@@ -121,7 +121,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
);
}
private render<T>(input: T, context: TemplateContext): T {
private render<T>(
input: T,
context: TemplateContext,
renderTemplate: SecureTemplateRenderer,
): T {
return JSON.parse(JSON.stringify(input), (_key, value) => {
try {
if (typeof value === 'string') {
@@ -134,10 +138,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
);
// Run the templating
const templated = this.nunjucks.renderString(
wrappedDumped,
context,
);
const templated = renderTemplate(wrappedDumped, context);
// If there's an empty string returned, then it's undefined
if (templated === '') {
@@ -154,7 +155,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
}
// Fallback to default behaviour
const templated = this.nunjucks.renderString(value, context);
const templated = renderTemplate(value, context);
if (templated === '') {
return undefined;
@@ -179,6 +180,18 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
this.options.workingDirectory,
await task.getWorkspaceName(),
);
const { integrations } = this.options;
const renderTemplate = await SecureTemplater.loadRenderer({
// TODO(blam): let's work out how we can deprecate this.
// We shouldn't really need to be exposing these now we can deal with
// objects in the params block.
// Maybe we can expose a new RepoUrlPicker with secrets for V3 that provides an object already.
parseRepoUrl(url: string) {
return parseRepoUrl(url, integrations);
},
});
try {
await fs.ensureDir(workspacePath);
await task.emitLog(
@@ -193,7 +206,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
for (const step of task.spec.steps) {
try {
if (step.if) {
const ifResult = await this.render(step.if, context);
const ifResult = await this.render(
step.if,
context,
renderTemplate,
);
if (!isTruthy(ifResult)) {
await task.emitLog(
`Skipping step ${step.id} because it's if condition was false`,
@@ -211,7 +228,9 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
const action = this.options.actionRegistry.get(step.action);
const { taskLogger, streamLogger } = createStepLogger({ task, step });
const input = (step.input && this.render(step.input, context)) ?? {};
const input =
(step.input && this.render(step.input, context, renderTemplate)) ??
{};
if (action.schema?.input) {
const validateResult = validateJsonSchema(
@@ -274,7 +293,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
}
}
const output = this.render(task.spec.output, context);
const output = this.render(task.spec.output, context, renderTemplate);
return { output };
} finally {