Merge pull request #9065 from DavidZemon/feature/8920-custom-nunjucks-filters

feat: [#8920] Add `nunjucksFilters` option for user-provided filters

.changeset/rude-clouds-chew.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Expose a new option to provide additional template filters via `@backstage/scaffolder-backend`'s `createRouter()` function.

plugins/scaffolder-backend/api-report.md

@@ -76,6 +76,7 @@ export const createBuiltinActions: (options: {
   catalogClient: CatalogApi;
   containerRunner?: ContainerRunner;
   config: Config;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 }) => TemplateAction<any>[];
 
 // Warning: (ae-missing-release-tag) "createCatalogRegisterAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -112,6 +113,7 @@ export function createFetchPlainAction(options: {
 export function createFetchTemplateAction(options: {
   reader: UrlReader;
   integrations: ScmIntegrations;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 }): TemplateAction<any>;
 
 // Warning: (ae-missing-release-tag) "createFilesystemDeleteAction" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -219,6 +221,7 @@ export type CreateWorkerOptions = {
   integrations: ScmIntegrations;
   workingDirectory: string;
   logger: Logger_2;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 };
 
 // @public
@@ -304,6 +307,8 @@ export interface RouterOptions {
   // (undocumented)
   actions?: TemplateAction<any>[];
   // (undocumented)
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
+  // (undocumented)
   catalogClient: CatalogApi;
   // (undocumented)
   config: Config;
@@ -545,5 +550,10 @@ export class TemplateActionRegistry {
   ): void;
 }
 
+// Warning: (ae-missing-release-tag) "TemplateFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export type TemplateFilter = (...args: JsonValue[]) => JsonValue | undefined;
+
 export { TemplateMetadata };
 ```

plugins/scaffolder-backend/src/index.ts

@@ -22,5 +22,5 @@
 
 export * from './scaffolder';
 export * from './service/router';
-export * from './lib/catalog';
+export * from './lib';
 export * from './processor';

plugins/scaffolder-backend/src/lib/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './catalog';
+export * from './templating';

plugins/scaffolder-backend/src/lib/templating/SecureTemplater.test.ts

@@ -99,6 +99,52 @@ describe('SecureTemplater', () => {
     ]);
   });
 
+  it('should make additional filters available when requested', async () => {
+    const mockFilter1 = jest.fn(() => 'filtered text');
+    const mockFilter2 = jest.fn((var1, var2) => `${var1} ${var2}`);
+    const mockFilter3 = jest.fn((var1, var2) => ({ var1, var2 }));
+    const renderWith = await SecureTemplater.loadRenderer({
+      additionalTemplateFilters: { mockFilter1, mockFilter2, mockFilter3 },
+    });
+    const renderWithout = await SecureTemplater.loadRenderer();
+
+    const ctx = { inputValue: 'the input value' };
+
+    expect(renderWith('${{ inputValue | mockFilter1 }}', ctx)).toBe(
+      'filtered text',
+    );
+    expect(
+      renderWith('${{ inputValue | mockFilter2("extra arg") }}', ctx),
+    ).toBe('the input value extra arg');
+    expect(
+      renderWith(
+        '${{ inputValue | mockFilter3("another extra arg") | dump }}',
+        ctx,
+      ),
+    ).toBe(
+      JSON.stringify({
+        var1: 'the input value',
+        var2: 'another extra arg',
+      }),
+    );
+
+    expect(() => renderWithout('${{ inputValue | mockFilter1 }}', ctx)).toThrow(
+      /Error: filter not found: mockFilter1/,
+    );
+    expect(() =>
+      renderWithout('${{ inputValue | mockFilter2("extra arg") }}', ctx),
+    ).toThrow(/Error: filter not found: mockFilter2/);
+    expect(() =>
+      renderWithout('${{ inputValue | mockFilter3("extra arg") }}', ctx),
+    ).toThrow(/Error: filter not found: mockFilter3/);
+
+    expect(mockFilter1.mock.calls).toEqual([['the input value']]);
+    expect(mockFilter2.mock.calls).toEqual([['the input value', 'extra arg']]);
+    expect(mockFilter3.mock.calls).toEqual([
+      ['the input value', 'another extra arg'],
+    ]);
+  });
+
   it('should not allow helpers to be rewritten', async () => {
     const render = await SecureTemplater.loadRenderer({
       parseRepoUrl: () => ({

plugins/scaffolder-backend/src/lib/templating/SecureTemplater.ts

@@ -17,8 +17,10 @@
 import { VM } from 'vm2';
 import { resolvePackagePath } from '@backstage/backend-common';
 import fs from 'fs-extra';
+import { JsonValue } from '@backstage/types';
 import { RepoSpec } from '../../scaffolder/actions/builtin/publish/util';
 
+// language=JavaScript
 const mkScript = (nunjucksSource: string) => `
 const { render, renderCompat } = (() => {
   const module = {};
@@ -56,6 +58,12 @@ const { render, renderCompat } = (() => {
     });
   }
 
+  if (typeof additionalTemplateFilters !== 'undefined') {
+    for (const [filterName, filterFn] of Object.entries(additionalTemplateFilters)) {
+      env.addFilter(filterName, (...args) => JSON.parse(filterFn(...args)));
+    }
+  }
+
   let uninstallCompat = undefined;
 
   function render(str, values) {
@@ -87,12 +95,17 @@ const { render, renderCompat } = (() => {
 })();
 `;
 
+export type TemplateFilter = (...args: JsonValue[]) => JsonValue | undefined;
+
 export interface SecureTemplaterOptions {
   /* Optional implementation of the parseRepoUrl filter */
   parseRepoUrl?(repoUrl: string): RepoSpec;
 
   /* Enables jinja compatibility and the "jsonify" filter */
   cookiecutterCompat?: boolean;
+
+  /* Extra user-provided nunjucks filters */
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 }
 
 export type SecureTemplateRenderer = (
@@ -102,13 +115,23 @@ export type SecureTemplateRenderer = (
 
 export class SecureTemplater {
   static async loadRenderer(options: SecureTemplaterOptions = {}) {
-    const { parseRepoUrl, cookiecutterCompat } = options;
-    let sandbox = undefined;
+    const { parseRepoUrl, cookiecutterCompat, additionalTemplateFilters } =
+      options;
+    const sandbox: Record<string, any> = {};
 
     if (parseRepoUrl) {
-      sandbox = {
-        parseRepoUrl: (url: string) => JSON.stringify(parseRepoUrl(url)),
-      };
+      sandbox.parseRepoUrl = (url: string) => JSON.stringify(parseRepoUrl(url));
+    }
+
+    if (additionalTemplateFilters) {
+      sandbox.additionalTemplateFilters = Object.fromEntries(
+        Object.entries(additionalTemplateFilters)
+          .filter(([_, filterFunction]) => !!filterFunction)
+          .map(([filterName, filterFunction]) => [
+            filterName,
+            (...args: JsonValue[]) => JSON.stringify(filterFunction(...args)),
+          ]),
+      );
     }
 
     const vm = new VM({ sandbox });

plugins/scaffolder-backend/src/lib/templating/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type { TemplateFilter } from './SecureTemplater';

plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts

@@ -46,6 +46,7 @@ import {
   createGithubActionsDispatchAction,
   createGithubWebhookAction,
 } from './github';
+import { TemplateFilter } from '../../../lib';
 
 export const createBuiltinActions = (options: {
   reader: UrlReader;
@@ -53,9 +54,16 @@ export const createBuiltinActions = (options: {
   catalogClient: CatalogApi;
   containerRunner?: ContainerRunner;
   config: Config;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 }) => {
-  const { reader, integrations, containerRunner, catalogClient, config } =
-    options;
+  const {
+    reader,
+    integrations,
+    containerRunner,
+    catalogClient,
+    config,
+    additionalTemplateFilters,
+  } = options;
   const githubCredentialsProvider: GithubCredentialsProvider =
     DefaultGithubCredentialsProvider.fromIntegrations(integrations);
 
@@ -67,6 +75,7 @@ export const createBuiltinActions = (options: {
     createFetchTemplateAction({
       integrations,
       reader,
+      additionalTemplateFilters,
     }),
     createPublishGithubAction({
       integrations,

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts

@@ -23,7 +23,10 @@ import { createTemplateAction } from '../../createTemplateAction';
 import globby from 'globby';
 import fs from 'fs-extra';
 import { isBinaryFile } from 'isbinaryfile';
-import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
+import {
+  TemplateFilter,
+  SecureTemplater,
+} from '../../../../lib/templating/SecureTemplater';
 
 type CookieCompatInput = {
   copyWithoutRender?: string[];
@@ -44,8 +47,9 @@ export type FetchTemplateInput = {
 export function createFetchTemplateAction(options: {
   reader: UrlReader;
   integrations: ScmIntegrations;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 }) {
-  const { reader, integrations } = options;
+  const { reader, integrations, additionalTemplateFilters } = options;
 
   return createTemplateAction<FetchTemplateInput>({
     id: 'fetch:template',
@@ -182,6 +186,7 @@ export function createFetchTemplateAction(options: {
 
       const renderTemplate = await SecureTemplater.loadRenderer({
         cookiecutterCompat: ctx.input.cookiecutterCompat,
+        additionalTemplateFilters,
       });
 
       for (const location of allEntriesInTemplate) {

plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts

@@ -35,6 +35,7 @@ import { validate as validateJsonSchema } from 'jsonschema';
 import { parseRepoUrl } from '../actions/builtin/publish/util';
 import { TemplateActionRegistry } from '../actions';
 import {
+  TemplateFilter,
   SecureTemplater,
   SecureTemplateRenderer,
 } from '../../lib/templating/SecureTemplater';
@@ -44,6 +45,7 @@ type NunjucksWorkflowRunnerOptions = {
   actionRegistry: TemplateActionRegistry;
   integrations: ScmIntegrations;
   logger: winston.Logger;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 };
 
 type TemplateContext = {
@@ -190,6 +192,7 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
       parseRepoUrl(url: string) {
         return parseRepoUrl(url, integrations);
       },
+      additionalTemplateFilters: this.options.additionalTemplateFilters,
     });
 
     try {

plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts

@@ -21,6 +21,7 @@ import { Logger } from 'winston';
 import { TemplateActionRegistry } from '../actions';
 import { ScmIntegrations } from '@backstage/integration';
 import { assertError } from '@backstage/errors';
+import { TemplateFilter } from '../../lib/templating/SecureTemplater';
 
 /**
  * TaskWorkerOptions
@@ -46,6 +47,7 @@ export type CreateWorkerOptions = {
   integrations: ScmIntegrations;
   workingDirectory: string;
   logger: Logger;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 };
 
 /**
@@ -63,6 +65,7 @@ export class TaskWorker {
       actionRegistry,
       integrations,
       workingDirectory,
+      additionalTemplateFilters,
     } = options;
 
     const legacyWorkflowRunner = new HandlebarsWorkflowRunner({
@@ -77,6 +80,7 @@ export class TaskWorker {
       integrations,
       logger,
       workingDirectory,
+      additionalTemplateFilters,
     });
 
     return new TaskWorker({

plugins/scaffolder-backend/src/service/router.ts

@@ -29,7 +29,7 @@ import express from 'express';
 import Router from 'express-promise-router';
 import { validate } from 'jsonschema';
 import { Logger } from 'winston';
-import { CatalogEntityClient } from '../lib/catalog';
+import { CatalogEntityClient, TemplateFilter } from '../lib';
 import {
   createBuiltinActions,
   DatabaseTaskStore,
@@ -57,6 +57,7 @@ export interface RouterOptions {
   taskWorkers?: number;
   containerRunner?: ContainerRunner;
   taskBroker?: TaskBroker;
+  additionalTemplateFilters?: Record<string, TemplateFilter>;
 }
 
 function isSupportedTemplate(
@@ -83,6 +84,7 @@ export async function createRouter(
     actions,
     containerRunner,
     taskWorkers,
+    additionalTemplateFilters,
   } = options;
 
   const logger = parentLogger.child({ plugin: 'scaffolder' });
@@ -110,6 +112,7 @@ export async function createRouter(
       integrations,
       logger,
       workingDirectory,
+      additionalTemplateFilters,
     });
     workers.push(worker);
   }
@@ -122,6 +125,7 @@ export async function createRouter(
         containerRunner,
         reader,
         config,
+        additionalTemplateFilters,
       });
 
   actionsToRegister.forEach(action => actionRegistry.register(action));