paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 42 Packages Projects Releases Wiki Activity

use CreatedTemplate[Filter|Global*] as canonical template extensions in scaffolder plugin

Browse Source 
Signed-off-by: Matt Benson <gudnabrsam@gmail.com>
This commit is contained in:
Matt Benson
2025-02-07 13:14:55 -06:00
parent 2c16c79856
commit 2b1e50db33
11 changed files with 463 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/shiny-ways-develop.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-scaffolder-backend': minor
---
use CreatedTemplate[Filter|Global*] as canonical template extensions in scaffolder plugin
plugins/scaffolder-backend/package.json
+2 -1
View File
@@ -113,7 +113,8 @@
"winston-transport": "^4.7.0",
"yaml": "^2.0.0",
"zen-observable": "^0.10.0",
"zod": "^3.22.4"
"zod": "^3.22.4",
"zod-to-json-schema": "^3.20.4"
},
"devDependencies": {
"@backstage/backend-app-api": "workspace:^",
plugins/scaffolder-backend/src/ScaffolderPlugin.ts
+35 -29
View File
@@ -21,14 +21,14 @@ import {
import { ScmIntegrations } from '@backstage/integration';
import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
import { eventsServiceRef } from '@backstage/plugin-events-node';
import {
TaskBroker,
TemplateAction,
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { TaskBroker, TemplateAction } from '@backstage/plugin-scaffolder-node';
import {
AutocompleteHandler,
CreatedTemplateFilter,
CreatedTemplateGlobal,
createTemplateFilter,
createTemplateGlobalFunction,
createTemplateGlobalValue,
scaffolderActionsExtensionPoint,
scaffolderAutocompleteExtensionPoint,
scaffolderTaskBrokerExtensionPoint,
@@ -52,6 +52,7 @@ import {
} from './scaffolder';
import { createRouter } from './service/router';
import { loggerToWinstonLogger } from './util/loggerToWinstonLogger';
import { templateFilterImpls, templateGlobals } from './util/templating';
/**
* Scaffolder plugin
@@ -78,30 +79,31 @@ export const scaffolderPlugin = createBackendPlugin({
},
});
const additionalTemplateFilters: Record<string, TemplateFilter> = {};
const additionalTemplateGlobals: Record<string, TemplateGlobal> = {};
const additionalTemplateFilters: CreatedTemplateFilter[] = [];
const additionalTemplateGlobals: CreatedTemplateGlobal[] = [];
env.registerExtensionPoint(scaffolderTemplatingExtensionPoint, {
addTemplateFilters(newFilters) {
Object.assign(
additionalTemplateFilters,
Array.isArray(newFilters)
? Object.fromEntries(
newFilters.map(tf => [tf.id, tf.filter as TemplateFilter]),
)
: newFilters,
additionalTemplateFilters.push(
...(Array.isArray(newFilters)
? newFilters
: Object.entries(newFilters).map(([id, filter]) =>
createTemplateFilter({
id,
filter,
}),
)),
);
},
addTemplateGlobals(newGlobals) {
Object.assign(
additionalTemplateGlobals,
Array.isArray(newGlobals)
? Object.fromEntries(
newGlobals.map(g => [
g.id,
('value' in g ? g.value : g.fn) as TemplateGlobal,
]),
)
: newGlobals,
additionalTemplateGlobals.push(
...(Array.isArray(newGlobals)
? newGlobals
: Object.entries(newGlobals).map(([id, global]) =>
typeof global === 'function'
? createTemplateGlobalFunction({ id, fn: global })
: createTemplateGlobalValue({ id, value: global }),
)),
);
},
});
@@ -154,6 +156,12 @@ export const scaffolderPlugin = createBackendPlugin({
const log = loggerToWinstonLogger(logger);
const integrations = ScmIntegrations.fromConfig(config);
const templateExtensions = {
additionalTemplateFilters: templateFilterImpls(
additionalTemplateFilters,
),
additionalTemplateGlobals: templateGlobals(additionalTemplateGlobals),
};
const actions = [
// actions provided from other modules
...addedActions,
@@ -170,14 +178,12 @@ export const scaffolderPlugin = createBackendPlugin({
createFetchTemplateAction({
integrations,
reader,
additionalTemplateFilters,
additionalTemplateGlobals,
...templateExtensions,
}),
createFetchTemplateFileAction({
integrations,
reader,
additionalTemplateFilters,
additionalTemplateGlobals,
...templateExtensions,
}),
createDebugLogAction(),
createWaitAction(),
plugins/scaffolder-backend/src/lib/templating/filters.ts
+1 -1
View File
@@ -22,7 +22,7 @@ import {
} from '@backstage/plugin-scaffolder-node';
import get from 'lodash/get';
export const createDefaultFilters = ({
export default ({
integrations,
}: {
integrations: ScmIntegrations;
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts
+6 -3
View File
@@ -29,8 +29,9 @@ import globby from 'globby';
import fs from 'fs-extra';
import { isBinaryFile } from 'isbinaryfile';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { createDefaultFilters } from '../../../../lib/templating/filters';
import createDefaultFilters from '../../../../lib/templating/filters';
import { examples } from './template.examples';
import { templateFilterImpls } from '../../../../util/templating';
/**
* Downloads a skeleton, templates variables into file and directory names and content.
@@ -52,7 +53,9 @@ export function createFetchTemplateAction(options: {
additionalTemplateGlobals,
} = options;
const defaultTemplateFilters = createDefaultFilters({ integrations });
const defaultTemplateFilters = templateFilterImpls(
createDefaultFilters({ integrations }),
);
return createTemplateAction<{
url: string;
@@ -243,7 +246,7 @@ export function createFetchTemplateAction(options: {
cookiecutterCompat: ctx.input.cookiecutterCompat,
templateFilters: {
...defaultTemplateFilters,
...additionalTemplateFilters,
...(additionalTemplateFilters ?? {}),
},
templateGlobals: additionalTemplateGlobals,
nunjucksConfigs: {
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/templateFile.ts
+5 -2
View File
@@ -25,9 +25,10 @@ import {
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { createDefaultFilters } from '../../../../lib/templating/filters';
import createDefaultFilters from '../../../../lib/templating/filters';
import path from 'path';
import fs from 'fs-extra';
import { templateFilterImpls } from '../../../../util/templating';
/**
* Downloads a single file and templates variables into file.
@@ -48,7 +49,9 @@ export function createFetchTemplateFileAction(options: {
additionalTemplateGlobals,
} = options;
const defaultTemplateFilters = createDefaultFilters({ integrations });
const defaultTemplateFilters = templateFilterImpls(
createDefaultFilters({ integrations }),
);
return createTemplateAction<{
url: string;
plugins/scaffolder-backend/src/scaffolder/tasks/NunjucksWorkflowRunner.ts
+7 -4
View File
@@ -55,11 +55,12 @@ import {
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { createDefaultFilters } from '../../lib/templating/filters';
import createDefaultFilters from '../../lib/templating/filters';
import { scaffolderActionRules } from '../../service/rules';
import { createCounterMetric, createHistogramMetric } from '../../util/metrics';
import { BackstageLoggerTransport, WinstonLogger } from './logger';
import { loggerToWinstonLogger } from '../../util/loggerToWinstonLogger';
import { templateFilterImpls } from '../../util/templating';
type NunjucksWorkflowRunnerOptions = {
workingDirectory: string;
@@ -151,9 +152,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
private readonly defaultTemplateFilters: Record<string, TemplateFilter>;
constructor(private readonly options: NunjucksWorkflowRunnerOptions) {
this.defaultTemplateFilters = createDefaultFilters({
integrations: this.options.integrations,
});
this.defaultTemplateFilters = templateFilterImpls(
createDefaultFilters({
integrations: this.options.integrations,
}),
);
}
private readonly tracker = scaffoldingTracker();
plugins/scaffolder-backend/src/service/router.ts
+3 -16
View File
@@ -110,6 +110,7 @@ import {
} from './helpers';
import { scaffolderActionRules, scaffolderTemplateRules } from './rules';
import { HostDiscovery } from '@backstage/backend-defaults/discovery';
import { templateFilterImpls, templateGlobals } from '../util/templating';
/**
*
@@ -368,22 +369,8 @@ export async function createRouter(
const actionRegistry = new TemplateActionRegistry();
const templateExtensions = {
additionalTemplateFilters: Array.isArray(additionalTemplateFilters)
? Object.fromEntries(
additionalTemplateFilters.map(f => [
f.id,
f.filter as TemplateFilter,
]),
)
: additionalTemplateFilters,
additionalTemplateGlobals: Array.isArray(additionalTemplateGlobals)
? Object.fromEntries(
additionalTemplateGlobals.map(g => [
g.id,
('value' in g ? g.value : g.fn) as TemplateGlobal,
]),
)
: additionalTemplateGlobals,
additionalTemplateFilters: templateFilterImpls(additionalTemplateFilters),
additionalTemplateGlobals: templateGlobals(additionalTemplateGlobals),
};
const workers: TaskWorker[] = [];
plugins/scaffolder-backend/src/util/templating.test.ts
+194
View File
@@ -0,0 +1,194 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
TemplateFilter,
CreatedTemplateGlobal,
createTemplateFilter,
createTemplateGlobalFunction,
createTemplateGlobalValue,
} from '@backstage/plugin-scaffolder-node/alpha';
import {
templateFilterImpls,
templateFilterMetadata,
templateGlobalFunctionMetadata,
templateGlobals,
templateGlobalValueMetadata,
} from './templating';
import { JsonValue } from '@backstage/types';
import builtInFilters from '../lib/templating/filters';
import { ScmIntegrations } from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
describe('templating utilities', () => {
describe('built-in filters', () => {
const integrations = ScmIntegrations.fromConfig(new ConfigReader({}));
const filters = builtInFilters({ integrations });
it('generates equivalent filter metadata', () => {
const metadata = templateFilterMetadata(filters);
expect(metadata).toMatchObject(templateFilterMetadata(filters));
});
});
describe('api filters', () => {
const filters = [
createTemplateFilter({
id: 'nop',
description: 'nop',
schema: z => z.function().args(z.any()).returns(z.any()),
filter: (x: JsonValue) => x,
}),
createTemplateFilter({
id: 'nul',
filter: (_: JsonValue) => null,
}),
createTemplateFilter({
id: 'repeat',
schema: z =>
z
.function()
.args(
z.string().describe('input'),
z.number().describe('times'),
z.string().describe('separator').optional(),
)
.returns(z.string()),
filter: (input: string, times: number, separator?: string) =>
Array(times)
.fill(input)
.join(separator ?? ''),
}),
];
it('extracts filter implementations', () => {
const impls = templateFilterImpls(filters);
expect(impls).toHaveProperty('nop');
expect(impls.nop(42)).toBe(42);
expect(impls).toHaveProperty('nul');
expect(impls.nul('anything')).toBeNull();
expect(impls).toHaveProperty('repeat');
expect(impls.repeat('foo', 3)).toBe('foofoofoo');
expect(impls.repeat('foo', 3, '-')).toBe('foo-foo-foo');
});
it('extracts filter metadata', () => {
const metadata = templateFilterMetadata(filters);
expect(metadata).toHaveProperty('nop');
expect(metadata.nop).toHaveProperty('description', 'nop');
expect(metadata.nop).toHaveProperty(
'schema',
expect.objectContaining({
input: expect.not.objectContaining({ type: expect.anything() }),
output: expect.not.objectContaining({ type: expect.anything() }),
}),
);
expect(metadata.nop.schema).not.toHaveProperty('arguments');
expect(metadata).toHaveProperty('nul', {});
expect(metadata).toHaveProperty('repeat');
expect(metadata.repeat).not.toHaveProperty('description');
expect(metadata.repeat).toHaveProperty(
'schema',
expect.objectContaining({
input: expect.objectContaining({ type: 'string' }),
arguments: expect.arrayContaining([
expect.objectContaining({
type: 'number',
}),
expect.objectContaining({
anyOf: expect.arrayContaining([
expect.objectContaining({ not: {} }),
expect.objectContaining({ type: 'string' }),
]),
}),
]),
output: expect.objectContaining({ type: 'string' }),
}),
);
});
});
describe('legacy filters', () => {
const filters = {
nop: x => x,
nul: _ => null,
} as Record<string, TemplateFilter>;
it('extracts filter implementations', () => {
const impls = templateFilterImpls(filters);
expect(impls).toHaveProperty('nop');
expect(impls.nop(42)).toBe(42);
expect(impls).toHaveProperty('nul');
expect(impls.nul('anything')).toBeNull();
});
it('extracts filter metadata', () => {
const metadata = templateFilterMetadata(filters);
expect(metadata).toHaveProperty('nop', {});
expect(metadata).toHaveProperty('nul', {});
});
});
const documentedGlobals: CreatedTemplateGlobal[] = [
createTemplateGlobalFunction({
id: 'foo',
description: 'foo something',
fn: (x: any) => `${x}_FOO`,
}),
createTemplateGlobalValue({
id: 'bar',
description: 'bar value',
value: 'bar',
}),
createTemplateGlobalFunction({
id: 'respond',
schema: z =>
z.function().args(z.string().describe('prompt')).returns(z.string()),
fn: (prompt: string) =>
prompt === 'knock knock' ? "who's there?" : "nobody's home",
}),
];
it('extracts global function metadata', () => {
const metadata = templateGlobalFunctionMetadata(documentedGlobals);
expect(metadata).toHaveProperty('foo');
expect(metadata.foo).toHaveProperty('description', 'foo something');
expect(metadata).not.toHaveProperty('bar');
expect(metadata).toHaveProperty(
'respond',
expect.objectContaining({
schema: expect.objectContaining({
arguments: expect.arrayContaining([
expect.objectContaining({ type: 'string' }),
]),
output: expect.objectContaining({ type: 'string' }),
}),
}),
);
});
it('extracts global value metadata', () => {
const metadata = templateGlobalValueMetadata(documentedGlobals);
expect(metadata).not.toHaveProperty('foo');
expect(metadata).toHaveProperty('bar');
expect(metadata.bar).toHaveProperty('description', 'bar value');
expect(metadata).not.toHaveProperty('respond');
});
const globals = templateGlobals(documentedGlobals);
it('extracts documented globals', () => {
expect(globals).toHaveProperty('foo');
expect((globals.foo as (x: any) => string)('something')).toBe(
'something_FOO',
);
expect(globals).toHaveProperty('bar', 'bar');
expect(globals).toHaveProperty('respond');
expect(typeof globals.respond).toBe('function');
expect((globals.respond as Function)('knock knock')).toBe("who's there?");
});
it('extracts backward-compatible/undocumented globals', () => {
expect(templateGlobals(globals)).toBe(globals);
});
});
plugins/scaffolder-backend/src/util/templating.ts
+204
View File
@@ -0,0 +1,204 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import {
CreatedTemplateFilter,
CreatedTemplateGlobal,
CreatedTemplateGlobalFunction,
CreatedTemplateGlobalValue,
TemplateGlobalFunctionSchema,
} from '@backstage/plugin-scaffolder-node/alpha';
import { JsonValue } from '@backstage/types';
import { Schema } from 'jsonschema';
import {
filter,
fromPairs,
isEmpty,
keyBy,
mapValues,
negate,
pick,
pickBy,
toPairs,
wrap,
} from 'lodash';
import { z, ZodType } from 'zod';
import zodToJsonSchema from 'zod-to-json-schema';
export function templateFilterImpls(
filters?: Record<string, TemplateFilter> | CreatedTemplateFilter<any, any>[],
): Record<string, TemplateFilter> {
if (!filters) {
return {};
}
if (Array.isArray(filters)) {
return mapValues(keyBy(filters, 'id'), f => f.filter as TemplateFilter);
}
return filters;
}
type ExportFunctionSchema = {
arguments?: Schema[];
output?: Schema;
};
type ExportFilterSchema = {
input?: Schema;
} & ExportFunctionSchema;
function zodFunctionToJsonSchema(
t: ReturnType<TemplateGlobalFunctionSchema<any, any>>,
): ExportFunctionSchema {
const args = (t.parameters().items as ZodType[]).map(
zt => zodToJsonSchema(zt) as Schema,
);
const output = wrap(t.returnType(), rt =>
rt._unknown ? undefined : (zodToJsonSchema(rt) as Schema),
)();
return pickBy({ output, arguments: args }, negate(isEmpty));
}
function toExportFilterSchema(
fnSchema: ExportFunctionSchema,
): ExportFilterSchema {
if (fnSchema.arguments?.length) {
const [input, ...rest] = fnSchema.arguments;
const { output } = fnSchema;
return {
input,
...(rest.length ? { arguments: rest } : {}),
...(output ? { output } : {}),
};
}
return fnSchema;
}
type ExportFilter = Pick<
CreatedTemplateFilter<any, any>,
'description' | 'examples'
> & {
schema?: ExportFilterSchema;
};
export function templateFilterMetadata(
filters?: Record<string, TemplateFilter> | CreatedTemplateFilter<any, any>[],
): Record<string, ExportFilter> {
if (!filters) {
return {};
}
if (Array.isArray(filters)) {
return mapValues(
keyBy(filters, 'id'),
<F extends CreatedTemplateFilter<any, any>>(f: F): ExportFilter => {
const schema = f.schema
? toExportFilterSchema(zodFunctionToJsonSchema(f.schema(z)))
: undefined;
return {
...pick(f, 'description', 'examples'),
...pickBy({ schema }),
};
},
);
}
return mapValues(filters, _ => ({}));
}
function isGlobalFunctionInfo(
global: CreatedTemplateGlobal,
): global is CreatedTemplateGlobalFunction<
TemplateGlobalFunctionSchema<any, any> | undefined,
any
> {
return 'fn' in global;
}
type GlobalRecordRow = [string, TemplateGlobal];
export function templateGlobalFunctionMetadata(
globals?: Record<string, TemplateGlobal> | CreatedTemplateGlobal[],
): Record<
string,
Pick<CreatedTemplateGlobalFunction<any, any>, 'description' | 'examples'> & {
schema?: ExportFunctionSchema;
}
> {
if (!globals) {
return {};
}
if (Array.isArray(globals)) {
return mapValues(keyBy(filter(globals, isGlobalFunctionInfo), 'id'), v => {
const schema = v.schema
? zodFunctionToJsonSchema(v.schema(z))
: undefined;
return {
...pick(v, 'description', 'examples'),
...pickBy({ schema }),
};
});
}
const rows = toPairs(globals) as GlobalRecordRow[];
const fns = rows.filter(([_, g]) => typeof g === 'function') as [
string,
Exclude<TemplateGlobal, JsonValue>,
][];
return fromPairs(fns.map(([k, _]) => [k, {}]));
}
export function templateGlobalValueMetadata(
globals?: Record<string, TemplateGlobal> | CreatedTemplateGlobal[],
): Record<string, Omit<CreatedTemplateGlobalValue, 'id'>> {
if (!globals) {
return {};
}
if (Array.isArray(globals)) {
return mapValues(
keyBy(
filter(
globals,
negate(isGlobalFunctionInfo),
) as CreatedTemplateGlobalValue[],
'id',
),
v => pick(v, 'value', 'description'),
);
}
const rows = toPairs(globals) as GlobalRecordRow[];
const vals = rows.filter(([_, g]) => typeof g !== 'function') as [
string,
JsonValue,
][];
return fromPairs(vals.map(([k, value]) => [k, { value }]));
}
export function templateGlobals(
globals?: Record<string, TemplateGlobal> | CreatedTemplateGlobal[],
): Record<string, TemplateGlobal> {
if (!globals) {
return {};
}
if (!Array.isArray(globals)) {
return globals;
}
return mapValues(keyBy(globals, 'id'), v =>
isGlobalFunctionInfo(v)
? (v.fn as TemplateGlobal)
: (v as CreatedTemplateGlobalValue).value,
);
}
yarn.lock
+1
View File
@@ -7738,6 +7738,7 @@ __metadata:
yaml: ^2.0.0
zen-observable: ^0.10.0
zod: ^3.22.4
zod-to-json-schema: ^3.20.4
languageName: unknown
linkType: soft