diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts index 092dfd7c7b..cc275435e8 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1beta3.test.ts @@ -30,6 +30,7 @@ describe('templateEntityV1beta3Validator', () => { name: 'test', }, spec: { + type: 'website', parameters: { required: ['storePath', 'owner'], properties: { @@ -38,11 +39,6 @@ describe('templateEntityV1beta3Validator', () => { title: 'Owner', description: 'Who is going to own this component', }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, }, }, steps: [ diff --git a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json index 236944c9a1..e87a436d34 100644 --- a/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json +++ b/packages/catalog-model/src/schema/kinds/Template.v1beta3.schema.json @@ -14,9 +14,8 @@ }, "spec": { "owner": "artist-relations-team", - "type": "website", "parameters": { - "required": ["name", "description"], + "required": ["name", "description", "repoUrl"], "properties": { "name": { "title": "Name", @@ -27,6 +26,11 @@ "title": "Description", "type": "string", "description": "Description of the component" + }, + "repoUrl": { + "title": "Pick a repository", + "type": "string", + "ui:field": "RepoUrlPicker" } } }, @@ -44,13 +48,13 @@ "name": "Publish to GitHub", "action": "publish:github", "parameters": { - "repoUrl": "{{ parameters.repoUrl }}" + "repoUrl": "${{ parameters.repoUrl }}" }, - "if": "{{ parameters.repoUrl }}" + "if": "${{ parameters.repoUrl }}" } ], "output": { - "catalogInfoUrl": "{{ steps.publish.output.catalogInfoUrl }}" + "catalogInfoUrl": "${{ steps.publish.output.catalogInfoUrl }}" } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts new file mode 100644 index 0000000000..813fa1b363 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DefaultWorkflowRunner.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateActionRegistry } from '..'; +import { Task, WorkflowResponse, WorkflowRunner } from './types'; + +type Options = { + workingDirectory: string; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; +}; + +export class DefaultWorkflowRunner implements WorkflowRunner { + constructor(private readonly options: Options) {} + async execute(task: Task): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts new file mode 100644 index 0000000000..a5bf740c07 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/LegacyWorkflowRunner.ts @@ -0,0 +1,297 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Task, WorkflowRunner, WorkflowResponse } from './types'; +import * as Handlebars from 'handlebars'; +import { TemplateActionRegistry } from '..'; +import { ScmIntegrations } from '@backstage/integration'; +import { parseRepoUrl } from '../actions/builtin/publish/util'; +import { isTruthy } from './helper'; +import { PassThrough } from 'stream'; +import * as winston from 'winston'; +import { Logger } from 'winston'; +import path from 'path'; +import fs from 'fs-extra'; +import { validate as validateJsonSchema } from 'jsonschema'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { InputError } from '@backstage/errors'; + +type Options = { + workingDirectory: string; + actionRegistry: TemplateActionRegistry; + integrations: ScmIntegrations; + logger: Logger; +}; + +/** + * This is the legacy workflow runner, which supports handlebars. This entire implementation will be replaced + * with the default workflow runner interface in the future so this entire thing can go bye bye. + */ +export class LegacyWorkflowRunner implements WorkflowRunner { + private readonly handlebars: typeof Handlebars; + + constructor(private readonly options: Options) { + this.handlebars = Handlebars.create(); + + // TODO(blam): this should be a public facing API but it's a little + // scary right now, so we're going to lock it off like the component API is + // in the frontend until we can work out a nice way to do it. + this.handlebars.registerHelper('parseRepoUrl', repoUrl => { + return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations)); + }); + + this.handlebars.registerHelper('projectSlug', repoUrl => { + const { owner, repo } = parseRepoUrl(repoUrl, this.options.integrations); + return `${owner}/${repo}`; + }); + + this.handlebars.registerHelper('json', obj => JSON.stringify(obj)); + + this.handlebars.registerHelper('not', value => !isTruthy(value)); + + this.handlebars.registerHelper('eq', (a, b) => a === b); + } + + async execute(task: Task): Promise { + const { actionRegistry } = this.options; + + const workspacePath = path.join( + this.options.workingDirectory, + await task.getWorkspaceName(), + ); + try { + await fs.ensureDir(workspacePath); + await task.emitLog( + `Starting up task with ${task.spec.steps.length} steps`, + ); + + const templateCtx: { + parameters: JsonObject; + steps: { + [stepName: string]: { output: { [outputName: string]: JsonValue } }; + }; + } = { parameters: task.spec.values, steps: {} }; + + for (const step of task.spec.steps) { + const metadata = { stepId: step.id }; + try { + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const stream = new PassThrough(); + stream.on('data', async data => { + const message = data.toString().trim(); + if (message?.length > 1) { + await task.emitLog(message, metadata); + } + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + + if (step.if !== undefined) { + // Support passing values like false to disable steps + let skip = !step.if; + + // Evaluate strings as handlebar templates + if (typeof step.if === 'string') { + const condition = JSON.parse( + JSON.stringify(step.if), + (_key, value) => { + if (typeof value === 'string') { + const templated = this.handlebars.compile(value, { + noEscape: true, + data: false, + preventIndent: true, + })(templateCtx); + + // If it's just an empty string, treat it as undefined + if (templated === '') { + return undefined; + } + + try { + return JSON.parse(templated); + } catch { + return templated; + } + } + + return value; + }, + ); + + skip = !isTruthy(condition); + } + + if (skip) { + await task.emitLog(`Skipped step ${step.name}`, { + ...metadata, + status: 'skipped', + }); + continue; + } + } + + await task.emitLog(`Beginning step ${step.name}`, { + ...metadata, + status: 'processing', + }); + + const action = actionRegistry.get(step.action); + if (!action) { + throw new Error(`Action '${step.action}' does not exist`); + } + + const input = + step.input && + JSON.parse(JSON.stringify(step.input), (_key, value) => { + if (typeof value === 'string') { + const templated = this.handlebars.compile(value, { + noEscape: true, + data: false, + preventIndent: true, + })(templateCtx); + + // If it smells like a JSON object then give it a parse as an object and if it fails return the string + if ( + (templated.startsWith('"') && templated.endsWith('"')) || + (templated.startsWith('{') && templated.endsWith('}')) || + (templated.startsWith('[') && templated.endsWith(']')) + ) { + try { + // Don't recursively JSON parse the values of this string. + // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else + return JSON.parse(templated); + } catch { + return templated; + } + } + return templated; + } + + return value; + }); + + if (action.schema?.input) { + const validateResult = validateJsonSchema( + input, + action.schema.input, + ); + if (!validateResult.valid) { + const errors = validateResult.errors.join(', '); + throw new InputError( + `Invalid input passed to action ${action.id}, ${errors}`, + ); + } + } + + const stepOutputs: { [name: string]: JsonValue } = {}; + + // Keep track of all tmp dirs that are created by the action so we can remove them after + const tmpDirs = new Array(); + + this.options.logger.debug(`Running ${action.id} with input`, { + input: JSON.stringify(input, null, 2), + }); + + await action.handler({ + baseUrl: task.spec.baseUrl, + logger: taskLogger, + logStream: stream, + input, + token: task.secrets?.token, + workspacePath, + async createTemporaryDirectory() { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, + output(name: string, value: JsonValue) { + stepOutputs[name] = value; + }, + }); + + // Remove all temporary directories that were created when executing the action + for (const tmpDir of tmpDirs) { + await fs.remove(tmpDir); + } + + templateCtx.steps[step.id] = { output: stepOutputs }; + + await task.emitLog(`Finished step ${step.name}`, { + ...metadata, + status: 'completed', + }); + } catch (error) { + await task.emitLog(String(error.stack), { + ...metadata, + status: 'failed', + }); + throw error; + } + } + + const output = JSON.parse( + JSON.stringify(task.spec.output), + (_key, value) => { + if (typeof value === 'string') { + const templated = this.handlebars.compile(value, { + noEscape: true, + data: false, + preventIndent: true, + })(templateCtx); + + // If it's just an empty string, treat it as undefined + if (templated === '') { + return undefined; + } + + // If it smells like a JSON object then give it a parse as an object and if it fails return the string + if ( + (templated.startsWith('"') && templated.endsWith('"')) || + (templated.startsWith('{') && templated.endsWith('}')) || + (templated.startsWith('[') && templated.endsWith(']')) + ) { + try { + // Don't recursively JSON parse the values of this string. + // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else + return JSON.parse(templated); + } catch { + return templated; + } + } + return templated; + } + return value; + }, + ); + + return { output }; + } finally { + if (workspacePath) { + await fs.remove(workspacePath); + } + } + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 7577a62d1f..a1af3df2b0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -17,17 +17,17 @@ import { JsonObject, JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import fs from 'fs-extra'; -import * as Handlebars from 'handlebars'; -import { validate as validateJsonSchema } from 'jsonschema'; + import path from 'path'; -import { PassThrough } from 'stream'; -import * as winston from 'winston'; + import { Logger } from 'winston'; import { parseRepoUrl } from '../actions/builtin/publish/util'; import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import { isTruthy } from './helper'; -import { Task, TaskBroker } from './types'; +import { Task, TaskBroker, WorkflowRunner } from './types'; import { ScmIntegrations } from '@backstage/integration'; +import { LegacyWorkflowRunner } from './LegacyWorkflowRunner'; +import { DefaultWorkflowRunner } from './DefaultWorkflowRunner'; type Options = { logger: Logger; @@ -38,28 +38,12 @@ type Options = { }; export class TaskWorker { - private readonly handlebars: typeof Handlebars; + private readonly legacyWorkflowRunner: LegacyWorkflowRunner; + private readonly workflowRunner: WorkflowRunner; constructor(private readonly options: Options) { - this.handlebars = Handlebars.create(); - - // TODO(blam): this should be a public facing API but it's a little - // scary right now, so we're going to lock it off like the component API is - // in the frontend until we can work out a nice way to do it. - this.handlebars.registerHelper('parseRepoUrl', repoUrl => { - return JSON.stringify(parseRepoUrl(repoUrl, options.integrations)); - }); - - this.handlebars.registerHelper('projectSlug', repoUrl => { - const { owner, repo } = parseRepoUrl(repoUrl, options.integrations); - return `${owner}/${repo}`; - }); - - this.handlebars.registerHelper('json', obj => JSON.stringify(obj)); - - this.handlebars.registerHelper('not', value => !isTruthy(value)); - - this.handlebars.registerHelper('eq', (a, b) => a === b); + this.legacyWorkflowRunner = new LegacyWorkflowRunner(options); + this.workflowRunner = new DefaultWorkflowRunner(options); } start() { @@ -72,247 +56,17 @@ export class TaskWorker { } async runOneTask(task: Task) { - let workspacePath: string | undefined = undefined; try { - const { actionRegistry } = this.options; - - workspacePath = path.join( - this.options.workingDirectory, - await task.getWorkspaceName(), - ); - await fs.ensureDir(workspacePath); - await task.emitLog( - `Starting up task with ${task.spec.steps.length} steps`, - ); - - const templateCtx: { - parameters: JsonObject; - steps: { - [stepName: string]: { output: { [outputName: string]: JsonValue } }; - }; - } = { parameters: task.spec.values, steps: {} }; - const { output } = task.spec.apiVersion === 'backstage.io/v1beta3' - ? await TemplateWorkflowRunner.execute(task) - : await LegacyWorkflowRunner.execute(task); - if (task.spec.apiVersion === 'backstage.io/v1beta3') { - const { output } = await TemplateWorkflowRunnger.execute(task); - } else { - } - - for (const step of task.spec.steps) { - const metadata = { stepId: step.id }; - try { - const taskLogger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: {}, - }); - - const stream = new PassThrough(); - stream.on('data', async data => { - const message = data.toString().trim(); - if (message?.length > 1) { - await task.emitLog(message, metadata); - } - }); - - taskLogger.add(new winston.transports.Stream({ stream })); - - if (step.if !== undefined) { - // Support passing values like false to disable steps - let skip = !step.if; - - // Evaluate strings as handlebar templates - if (typeof step.if === 'string') { - const condition = JSON.parse( - JSON.stringify(step.if), - (_key, value) => { - if (typeof value === 'string') { - const templated = this.handlebars.compile(value, { - noEscape: true, - data: false, - preventIndent: true, - })(templateCtx); - - // If it's just an empty string, treat it as undefined - if (templated === '') { - return undefined; - } - - try { - return JSON.parse(templated); - } catch { - return templated; - } - } - - return value; - }, - ); - - skip = !isTruthy(condition); - } - - if (skip) { - await task.emitLog(`Skipped step ${step.name}`, { - ...metadata, - status: 'skipped', - }); - continue; - } - } - - await task.emitLog(`Beginning step ${step.name}`, { - ...metadata, - status: 'processing', - }); - - const action = actionRegistry.get(step.action); - if (!action) { - throw new Error(`Action '${step.action}' does not exist`); - } - - const input = - step.input && - JSON.parse(JSON.stringify(step.input), (_key, value) => { - if (typeof value === 'string') { - const templated = this.handlebars.compile(value, { - noEscape: true, - data: false, - preventIndent: true, - })(templateCtx); - - // If it smells like a JSON object then give it a parse as an object and if it fails return the string - if ( - (templated.startsWith('"') && templated.endsWith('"')) || - (templated.startsWith('{') && templated.endsWith('}')) || - (templated.startsWith('[') && templated.endsWith(']')) - ) { - try { - // Don't recursively JSON parse the values of this string. - // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else - return JSON.parse(templated); - } catch { - return templated; - } - } - return templated; - } - - return value; - }); - - if (action.schema?.input) { - const validateResult = validateJsonSchema( - input, - action.schema.input, - ); - if (!validateResult.valid) { - const errors = validateResult.errors.join(', '); - throw new InputError( - `Invalid input passed to action ${action.id}, ${errors}`, - ); - } - } - - const stepOutputs: { [name: string]: JsonValue } = {}; - - // Keep track of all tmp dirs that are created by the action so we can remove them after - const tmpDirs = new Array(); - - this.options.logger.debug(`Running ${action.id} with input`, { - input: JSON.stringify(input, null, 2), - }); - - await action.handler({ - baseUrl: task.spec.baseUrl, - logger: taskLogger, - logStream: stream, - input, - token: task.secrets?.token, - workspacePath, - async createTemporaryDirectory() { - const tmpDir = await fs.mkdtemp( - `${workspacePath}_step-${step.id}-`, - ); - tmpDirs.push(tmpDir); - return tmpDir; - }, - output(name: string, value: JsonValue) { - stepOutputs[name] = value; - }, - }); - - // Remove all temporary directories that were created when executing the action - for (const tmpDir of tmpDirs) { - await fs.remove(tmpDir); - } - - templateCtx.steps[step.id] = { output: stepOutputs }; - - await task.emitLog(`Finished step ${step.name}`, { - ...metadata, - status: 'completed', - }); - } catch (error) { - await task.emitLog(String(error.stack), { - ...metadata, - status: 'failed', - }); - throw error; - } - } - - const output = JSON.parse( - JSON.stringify(task.spec.output), - (_key, value) => { - if (typeof value === 'string') { - const templated = this.handlebars.compile(value, { - noEscape: true, - data: false, - preventIndent: true, - })(templateCtx); - - // If it's just an empty string, treat it as undefined - if (templated === '') { - return undefined; - } - - // If it smells like a JSON object then give it a parse as an object and if it fails return the string - if ( - (templated.startsWith('"') && templated.endsWith('"')) || - (templated.startsWith('{') && templated.endsWith('}')) || - (templated.startsWith('[') && templated.endsWith(']')) - ) { - try { - // Don't recursively JSON parse the values of this string. - // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else - return JSON.parse(templated); - } catch { - return templated; - } - } - return templated; - } - return value; - }, - ); + ? await this.workflowRunner.execute(task) + : await this.legacyWorkflowRunner.execute(task); await task.complete('completed', { output }); } catch (error) { await task.complete('failed', { error: { name: error.name, message: error.message }, }); - } finally { - if (workspacePath) { - await fs.remove(workspacePath); - } } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 3b1805f117..5ff40b9459 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -44,6 +44,7 @@ export type DbTaskEventRow = { }; export type TaskSpec = { + apiVersion: 'backstage.io/v1beta2' | 'backstage.io/v1beta3'; baseUrl?: string; values: JsonObject; steps: Array<{ @@ -122,3 +123,8 @@ export interface TaskStore { after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; } + +export type WorkflowResponse = { output: { [name: string]: JsonValue } }; +export interface WorkflowRunner { + execute(task: Task): Promise; +}