Merge pull request #28813 from mbenson/scaffolder-template-extensions-stage2

use CreatedTemplate[Filter|Global*] as canonical template extensions …
This commit is contained in:
Ben Lambert
2025-03-07 14:57:55 +01:00
committed by GitHub
11 changed files with 550 additions and 56 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-scaffolder-backend': minor
---
use CreatedTemplate[Filter|Global*] as canonical template extensions in scaffolder plugin
+2 -1
View File
@@ -113,7 +113,8 @@
"winston-transport": "^4.7.0",
"yaml": "^2.0.0",
"zen-observable": "^0.10.0",
"zod": "^3.22.4"
"zod": "^3.22.4",
"zod-to-json-schema": "^3.20.4"
},
"devDependencies": {
"@backstage/backend-app-api": "workspace:^",
@@ -21,14 +21,14 @@ import {
import { ScmIntegrations } from '@backstage/integration';
import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha';
import { eventsServiceRef } from '@backstage/plugin-events-node';
import {
TaskBroker,
TemplateAction,
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { TaskBroker, TemplateAction } from '@backstage/plugin-scaffolder-node';
import {
AutocompleteHandler,
CreatedTemplateFilter,
CreatedTemplateGlobal,
createTemplateFilter,
createTemplateGlobalFunction,
createTemplateGlobalValue,
scaffolderActionsExtensionPoint,
scaffolderAutocompleteExtensionPoint,
scaffolderTaskBrokerExtensionPoint,
@@ -52,6 +52,10 @@ import {
} from './scaffolder';
import { createRouter } from './service/router';
import { loggerToWinstonLogger } from './util/loggerToWinstonLogger';
import {
convertFiltersToRecord,
convertGlobalsToRecord,
} from './util/templating';
/**
* Scaffolder plugin
@@ -78,30 +82,31 @@ export const scaffolderPlugin = createBackendPlugin({
},
});
const additionalTemplateFilters: Record<string, TemplateFilter> = {};
const additionalTemplateGlobals: Record<string, TemplateGlobal> = {};
const additionalTemplateFilters: CreatedTemplateFilter[] = [];
const additionalTemplateGlobals: CreatedTemplateGlobal[] = [];
env.registerExtensionPoint(scaffolderTemplatingExtensionPoint, {
addTemplateFilters(newFilters) {
Object.assign(
additionalTemplateFilters,
Array.isArray(newFilters)
? Object.fromEntries(
newFilters.map(tf => [tf.id, tf.filter as TemplateFilter]),
)
: newFilters,
additionalTemplateFilters.push(
...(Array.isArray(newFilters)
? newFilters
: Object.entries(newFilters).map(([id, filter]) =>
createTemplateFilter({
id,
filter,
}),
)),
);
},
addTemplateGlobals(newGlobals) {
Object.assign(
additionalTemplateGlobals,
Array.isArray(newGlobals)
? Object.fromEntries(
newGlobals.map(g => [
g.id,
('value' in g ? g.value : g.fn) as TemplateGlobal,
]),
)
: newGlobals,
additionalTemplateGlobals.push(
...(Array.isArray(newGlobals)
? newGlobals
: Object.entries(newGlobals).map(([id, global]) =>
typeof global === 'function'
? createTemplateGlobalFunction({ id, fn: global })
: createTemplateGlobalValue({ id, value: global }),
)),
);
},
});
@@ -154,6 +159,14 @@ export const scaffolderPlugin = createBackendPlugin({
const log = loggerToWinstonLogger(logger);
const integrations = ScmIntegrations.fromConfig(config);
const templateExtensions = {
additionalTemplateFilters: convertFiltersToRecord(
additionalTemplateFilters,
),
additionalTemplateGlobals: convertGlobalsToRecord(
additionalTemplateGlobals,
),
};
const actions = [
// actions provided from other modules
...addedActions,
@@ -170,14 +183,12 @@ export const scaffolderPlugin = createBackendPlugin({
createFetchTemplateAction({
integrations,
reader,
additionalTemplateFilters,
additionalTemplateGlobals,
...templateExtensions,
}),
createFetchTemplateFileAction({
integrations,
reader,
additionalTemplateFilters,
additionalTemplateGlobals,
...templateExtensions,
}),
createDebugLogAction(),
createWaitAction(),
@@ -22,7 +22,7 @@ import {
} from '@backstage/plugin-scaffolder-node';
import get from 'lodash/get';
export const createDefaultFilters = ({
export default ({
integrations,
}: {
integrations: ScmIntegrations;
@@ -29,8 +29,9 @@ import globby from 'globby';
import fs from 'fs-extra';
import { isBinaryFile } from 'isbinaryfile';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { createDefaultFilters } from '../../../../lib/templating/filters';
import createDefaultFilters from '../../../../lib/templating/filters';
import { examples } from './template.examples';
import { convertFiltersToRecord } from '../../../../util/templating';
/**
* Downloads a skeleton, templates variables into file and directory names and content.
@@ -52,7 +53,9 @@ export function createFetchTemplateAction(options: {
additionalTemplateGlobals,
} = options;
const defaultTemplateFilters = createDefaultFilters({ integrations });
const defaultTemplateFilters = convertFiltersToRecord(
createDefaultFilters({ integrations }),
);
return createTemplateAction<{
url: string;
@@ -243,7 +246,7 @@ export function createFetchTemplateAction(options: {
cookiecutterCompat: ctx.input.cookiecutterCompat,
templateFilters: {
...defaultTemplateFilters,
...additionalTemplateFilters,
...(additionalTemplateFilters ?? {}),
},
templateGlobals: additionalTemplateGlobals,
nunjucksConfigs: {
@@ -25,9 +25,10 @@ import {
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { SecureTemplater } from '../../../../lib/templating/SecureTemplater';
import { createDefaultFilters } from '../../../../lib/templating/filters';
import createDefaultFilters from '../../../../lib/templating/filters';
import path from 'path';
import fs from 'fs-extra';
import { convertFiltersToRecord } from '../../../../util/templating';
/**
* Downloads a single file and templates variables into file.
@@ -48,7 +49,9 @@ export function createFetchTemplateFileAction(options: {
additionalTemplateGlobals,
} = options;
const defaultTemplateFilters = createDefaultFilters({ integrations });
const defaultTemplateFilters = convertFiltersToRecord(
createDefaultFilters({ integrations }),
);
return createTemplateAction<{
url: string;
@@ -55,11 +55,12 @@ import {
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import { createDefaultFilters } from '../../lib/templating/filters';
import createDefaultFilters from '../../lib/templating/filters';
import { scaffolderActionRules } from '../../service/rules';
import { createCounterMetric, createHistogramMetric } from '../../util/metrics';
import { BackstageLoggerTransport, WinstonLogger } from './logger';
import { loggerToWinstonLogger } from '../../util/loggerToWinstonLogger';
import { convertFiltersToRecord } from '../../util/templating';
type NunjucksWorkflowRunnerOptions = {
workingDirectory: string;
@@ -151,9 +152,11 @@ export class NunjucksWorkflowRunner implements WorkflowRunner {
private readonly defaultTemplateFilters: Record<string, TemplateFilter>;
constructor(private readonly options: NunjucksWorkflowRunnerOptions) {
this.defaultTemplateFilters = createDefaultFilters({
integrations: this.options.integrations,
});
this.defaultTemplateFilters = convertFiltersToRecord(
createDefaultFilters({
integrations: this.options.integrations,
}),
);
}
private readonly tracker = scaffoldingTracker();
@@ -110,6 +110,10 @@ import {
} from './helpers';
import { scaffolderActionRules, scaffolderTemplateRules } from './rules';
import { HostDiscovery } from '@backstage/backend-defaults/discovery';
import {
convertFiltersToRecord,
convertGlobalsToRecord,
} from '../util/templating';
/**
*
@@ -368,22 +372,12 @@ export async function createRouter(
const actionRegistry = new TemplateActionRegistry();
const templateExtensions = {
additionalTemplateFilters: Array.isArray(additionalTemplateFilters)
? Object.fromEntries(
additionalTemplateFilters.map(f => [
f.id,
f.filter as TemplateFilter,
]),
)
: additionalTemplateFilters,
additionalTemplateGlobals: Array.isArray(additionalTemplateGlobals)
? Object.fromEntries(
additionalTemplateGlobals.map(g => [
g.id,
('value' in g ? g.value : g.fn) as TemplateGlobal,
]),
)
: additionalTemplateGlobals,
additionalTemplateFilters: convertFiltersToRecord(
additionalTemplateFilters,
),
additionalTemplateGlobals: convertGlobalsToRecord(
additionalTemplateGlobals,
),
};
const workers: TaskWorker[] = [];
@@ -0,0 +1,194 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
TemplateFilter,
CreatedTemplateGlobal,
createTemplateFilter,
createTemplateGlobalFunction,
createTemplateGlobalValue,
} from '@backstage/plugin-scaffolder-node/alpha';
import {
convertFiltersToRecord,
extractFilterMetadata,
extractGlobalFunctionMetadata,
convertGlobalsToRecord,
extractGlobalValueMetadata,
} from './templating';
import { JsonValue } from '@backstage/types';
import builtInFilters from '../lib/templating/filters';
import { ScmIntegrations } from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
describe('templating utilities', () => {
describe('built-in filters', () => {
const integrations = ScmIntegrations.fromConfig(new ConfigReader({}));
const filters = builtInFilters({ integrations });
it('generates equivalent filter metadata', () => {
const metadata = extractFilterMetadata(filters);
expect(metadata).toMatchObject(extractFilterMetadata(filters));
});
});
describe('api filters', () => {
const filters = [
createTemplateFilter({
id: 'nop',
description: 'nop',
schema: z => z.function().args(z.any()).returns(z.any()),
filter: (x: JsonValue) => x,
}),
createTemplateFilter({
id: 'nul',
filter: (_: JsonValue) => null,
}),
createTemplateFilter({
id: 'repeat',
schema: z =>
z
.function()
.args(
z.string().describe('input'),
z.number().describe('times'),
z.string().describe('separator').optional(),
)
.returns(z.string()),
filter: (input: string, times: number, separator?: string) =>
Array(times)
.fill(input)
.join(separator ?? ''),
}),
];
it('extracts filter implementations', () => {
const impls = convertFiltersToRecord(filters);
expect(impls).toHaveProperty('nop');
expect(impls.nop(42)).toBe(42);
expect(impls).toHaveProperty('nul');
expect(impls.nul('anything')).toBeNull();
expect(impls).toHaveProperty('repeat');
expect(impls.repeat('foo', 3)).toBe('foofoofoo');
expect(impls.repeat('foo', 3, '-')).toBe('foo-foo-foo');
});
it('extracts filter metadata', () => {
const metadata = extractFilterMetadata(filters);
expect(metadata).toHaveProperty('nop');
expect(metadata.nop).toHaveProperty('description', 'nop');
expect(metadata.nop).toHaveProperty(
'schema',
expect.objectContaining({
input: expect.not.objectContaining({ type: expect.anything() }),
output: expect.not.objectContaining({ type: expect.anything() }),
}),
);
expect(metadata.nop.schema).not.toHaveProperty('arguments');
expect(metadata).toHaveProperty('nul', {});
expect(metadata).toHaveProperty('repeat');
expect(metadata.repeat).not.toHaveProperty('description');
expect(metadata.repeat).toHaveProperty(
'schema',
expect.objectContaining({
input: expect.objectContaining({ type: 'string' }),
arguments: expect.arrayContaining([
expect.objectContaining({
type: 'number',
}),
expect.objectContaining({
anyOf: expect.arrayContaining([
expect.objectContaining({ not: {} }),
expect.objectContaining({ type: 'string' }),
]),
}),
]),
output: expect.objectContaining({ type: 'string' }),
}),
);
});
});
describe('legacy filters', () => {
const filters = {
nop: x => x,
nul: _ => null,
} as Record<string, TemplateFilter>;
it('extracts filter implementations', () => {
const impls = convertFiltersToRecord(filters);
expect(impls).toHaveProperty('nop');
expect(impls.nop(42)).toBe(42);
expect(impls).toHaveProperty('nul');
expect(impls.nul('anything')).toBeNull();
});
it('extracts filter metadata', () => {
const metadata = extractFilterMetadata(filters);
expect(metadata).toHaveProperty('nop', {});
expect(metadata).toHaveProperty('nul', {});
});
});
const documentedGlobals: CreatedTemplateGlobal[] = [
createTemplateGlobalFunction({
id: 'foo',
description: 'foo something',
fn: (x: any) => `${x}_FOO`,
}),
createTemplateGlobalValue({
id: 'bar',
description: 'bar value',
value: 'bar',
}),
createTemplateGlobalFunction({
id: 'respond',
schema: z =>
z.function().args(z.string().describe('prompt')).returns(z.string()),
fn: (prompt: string) =>
prompt === 'knock knock' ? "who's there?" : "nobody's home",
}),
];
it('extracts global function metadata', () => {
const metadata = extractGlobalFunctionMetadata(documentedGlobals);
expect(metadata).toHaveProperty('foo');
expect(metadata.foo).toHaveProperty('description', 'foo something');
expect(metadata).not.toHaveProperty('bar');
expect(metadata).toHaveProperty(
'respond',
expect.objectContaining({
schema: expect.objectContaining({
arguments: expect.arrayContaining([
expect.objectContaining({ type: 'string' }),
]),
output: expect.objectContaining({ type: 'string' }),
}),
}),
);
});
it('extracts global value metadata', () => {
const metadata = extractGlobalValueMetadata(documentedGlobals);
expect(metadata).not.toHaveProperty('foo');
expect(metadata).toHaveProperty('bar');
expect(metadata.bar).toHaveProperty('description', 'bar value');
expect(metadata).not.toHaveProperty('respond');
});
const globals = convertGlobalsToRecord(documentedGlobals);
it('extracts documented globals', () => {
expect(globals).toHaveProperty('foo');
expect((globals.foo as (x: any) => string)('something')).toBe(
'something_FOO',
);
expect(globals).toHaveProperty('bar', 'bar');
expect(globals).toHaveProperty('respond');
expect(typeof globals.respond).toBe('function');
expect((globals.respond as Function)('knock knock')).toBe("who's there?");
});
it('extracts backward-compatible/undocumented globals', () => {
expect(convertGlobalsToRecord(globals)).toBe(globals);
});
});
@@ -0,0 +1,279 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
TemplateFilter,
TemplateGlobal,
} from '@backstage/plugin-scaffolder-node';
import {
CreatedTemplateFilter,
CreatedTemplateGlobal,
CreatedTemplateGlobalFunction,
CreatedTemplateGlobalValue,
TemplateGlobalFunctionSchema,
} from '@backstage/plugin-scaffolder-node/alpha';
import { JsonValue } from '@backstage/types';
import { Schema } from 'jsonschema';
import { ZodType, z } from 'zod';
import zodToJsonSchema from 'zod-to-json-schema';
/**
* Converts template filters to a record of filter functions
*/
export function convertFiltersToRecord(
filters?: Record<string, TemplateFilter> | CreatedTemplateFilter<any, any>[],
): Record<string, TemplateFilter> {
if (!filters) {
return {};
}
if (Array.isArray(filters)) {
const result: Record<string, TemplateFilter> = {};
for (const filter of filters) {
result[filter.id] = filter.filter as TemplateFilter;
}
return result;
}
return filters;
}
type ExportFunctionSchema = {
arguments?: Schema[];
output?: Schema;
};
type ExportFilterSchema = {
input?: Schema;
} & ExportFunctionSchema;
/**
* Converts a Zod function schema to JSON schema
*/
function convertZodFunctionToJsonSchema(
t: ReturnType<TemplateGlobalFunctionSchema<any, any>>,
): ExportFunctionSchema {
const args = (t.parameters().items as ZodType[]).map(
zt => zodToJsonSchema(zt) as Schema,
);
let output: Schema | undefined = undefined;
const returnType = t.returnType();
if (!returnType._unknown) {
output = zodToJsonSchema(returnType) as Schema;
}
const result: ExportFunctionSchema = {};
if (args.length > 0) {
result.arguments = args;
}
if (output) {
result.output = output;
}
return result;
}
/**
* Converts a function schema to a filter schema
*/
function convertToFilterSchema(
fnSchema: ExportFunctionSchema,
): ExportFilterSchema {
if (fnSchema.arguments?.length) {
const [input, ...rest] = fnSchema.arguments;
const result: ExportFilterSchema = { input };
if (rest.length > 0) {
result.arguments = rest;
}
if (fnSchema.output) {
result.output = fnSchema.output;
}
return result;
}
return fnSchema;
}
type ExportFilter = Pick<
CreatedTemplateFilter<any, any>,
'description' | 'examples'
> & {
schema?: ExportFilterSchema;
};
/**
* Extracts metadata from template filters
*/
export function extractFilterMetadata(
filters?: Record<string, TemplateFilter> | CreatedTemplateFilter<any, any>[],
): Record<string, ExportFilter> {
if (!filters) {
return {};
}
if (Array.isArray(filters)) {
const result: Record<string, ExportFilter> = {};
for (const filter of filters) {
const metadata: ExportFilter = {};
if (filter.description) {
metadata.description = filter.description;
}
if (filter.examples) {
metadata.examples = filter.examples;
}
if (filter.schema) {
metadata.schema = convertToFilterSchema(
convertZodFunctionToJsonSchema(filter.schema(z)),
);
}
result[filter.id] = metadata;
}
return result;
}
// For non-array filters, return empty metadata
const result: Record<string, ExportFilter> = {};
for (const key in filters) {
if (filters.hasOwnProperty(key)) {
result[key] = {};
}
}
return result;
}
/**
* Checks if a global is a function
*/
function isGlobalFunction(
global: CreatedTemplateGlobal,
): global is CreatedTemplateGlobalFunction<
TemplateGlobalFunctionSchema<any, any> | undefined,
any
> {
return 'fn' in global;
}
/**
* Extracts metadata from template global functions
*/
export function extractGlobalFunctionMetadata(
globals?: Record<string, TemplateGlobal> | CreatedTemplateGlobal[],
): Record<
string,
Pick<CreatedTemplateGlobalFunction<any, any>, 'description' | 'examples'> & {
schema?: ExportFunctionSchema;
}
> {
if (!globals) {
return {};
}
if (Array.isArray(globals)) {
const result: Record<string, any> = {};
for (const global of globals) {
if (isGlobalFunction(global)) {
const metadata: any = {
description: global.description,
examples: global.examples,
};
if (global.schema) {
metadata.schema = convertZodFunctionToJsonSchema(global.schema(z));
}
result[global.id] = metadata;
}
}
return result;
}
// For non-array globals, extract function metadata
const result: Record<string, any> = {};
for (const key in globals) {
if (typeof globals[key] === 'function') {
result[key] = {};
}
}
return result;
}
/**
* Extracts metadata from template global values
*/
export function extractGlobalValueMetadata(
globals?: Record<string, TemplateGlobal> | CreatedTemplateGlobal[],
): Record<string, Omit<CreatedTemplateGlobalValue, 'id'>> {
if (!globals) {
return {};
}
if (Array.isArray(globals)) {
const result: Record<string, Omit<CreatedTemplateGlobalValue, 'id'>> = {};
for (const global of globals) {
if (!isGlobalFunction(global)) {
result[global.id] = {
value: (global as CreatedTemplateGlobalValue).value,
description: global.description,
};
}
}
return result;
}
// For non-array globals, extract value metadata
const result: Record<string, Omit<CreatedTemplateGlobalValue, 'id'>> = {};
for (const key in globals) {
if (typeof globals[key] !== 'function') {
result[key] = { value: globals[key] as JsonValue };
}
}
return result;
}
/**
* Converts template globals to a record of global values and functions
*/
export function convertGlobalsToRecord(
globals?: Record<string, TemplateGlobal> | CreatedTemplateGlobal[],
): Record<string, TemplateGlobal> {
if (!globals) {
return {};
}
if (!Array.isArray(globals)) {
return globals;
}
const result: Record<string, TemplateGlobal> = {};
for (const global of globals) {
result[global.id] = isGlobalFunction(global)
? (global.fn as TemplateGlobal)
: (global as CreatedTemplateGlobalValue).value;
}
return result;
}
+1
View File
@@ -7739,6 +7739,7 @@ __metadata:
yaml: ^2.0.0
zen-observable: ^0.10.0
zod: ^3.22.4
zod-to-json-schema: ^3.20.4
languageName: unknown
linkType: soft