From cf896972ef873b4a94f669d10b9a8f5a3e186e00 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 29 Jan 2021 13:16:50 +0100 Subject: [PATCH] Wire up task registry with legacy tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: blam Co-authored-by: Patrik Oldsberg --- .../src/scaffolder/stages/legacy.ts | 105 ++++++------ .../src/scaffolder/tasks/MemoryTaskBroker.ts | 8 + .../src/scaffolder/tasks/TaskWorker.ts | 155 ++++++++++-------- .../src/scaffolder/tasks/TemplateConverter.ts | 31 ++-- .../src/scaffolder/tasks/types.ts | 6 +- .../scaffolder-backend/src/service/helpers.ts | 44 +++++ .../scaffolder-backend/src/service/router.ts | 23 ++- 7 files changed, 242 insertions(+), 130 deletions(-) create mode 100644 plugins/scaffolder-backend/src/service/helpers.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index c9bd89c3dd..6244e40d68 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -14,21 +14,25 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import { TemplateActionRegistry } from '../TemplateConverter'; -import { FilePreparer } from './prepare'; +import { TemplateActionRegistry } from '../tasks/TemplateConverter'; +import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; +import { TemplaterBuilder, TemplaterValues } from './templater'; +import { PublisherBuilder } from './publish'; type Options = { - logger: Logger; - config: Config; dockerClient: Docker; + preparers: PreparerBuilder; + templaters: TemplaterBuilder; + publishers: PublisherBuilder; }; export function registerLegacyActions( registry: TemplateActionRegistry, options: Options, ) { + const { dockerClient, preparers, templaters, publishers } = options; + registry.register({ id: 'legacy:prepare', async handler(ctx) { @@ -41,36 +45,18 @@ export function registerLegacyActions( logger.info('Prepare the skeleton'); const { protocol, pullPath } = ctx.parameters; - + const url = pullPath as string; const preparer = - protocol === 'file' - ? new FilePreparer() - : preparers.get(pullPath as string); + protocol === 'file' ? new FilePreparer() : preparers.get(url); - await preparer.prepare(task.spec.template, { - logger, - ctx.workspaceDir, + await preparer.prepare({ + url, + logger: ctx.logger, + workspacePath: ctx.workspacePath, }); - ctx.output('catalogInfoUrl', 'httpderp://asdasd'); }, }); - // try { - // const { values, template } = task.spec; - // task.emitLog('Prepare the skeleton'); - // const { protocol, location: pullPath } = parseLocationAnnotation( - // task.spec.template, - // ); - // const preparer = - // protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); - // const templater = templaters.get(template); - // const publisher = publishers.get(values.storePath); - - // const skeletonDir = await preparer.prepare(task.spec.template, { - // logger: taskLogger, - // workingDirectory: workingDirectory, - // }); - registry.register({ id: 'legacy:template', async handler(ctx) { @@ -79,28 +65,55 @@ export function registerLegacyActions( const templater = templaters.get(ctx.parameters.templater as string); logger.info('Run the templater'); - const { resultDir } = await templater.run({ - directory: ctx.workspaceDir, + await templater.run({ + workspacePath: ctx.workspacePath, dockerClient, logStream: ctx.logStream, values: ctx.parameters.values as TemplaterValues, }); }, }); - // task.emitLog('Publish template'); - // logger.info('Will now store the template'); - // logger.info('Totally storing the template now'); - // await new Promise(resolve => setTimeout(resolve, 5000)); - // // const result = await publisher.publish({ - // // values: values, - // // directory: resultDir, - // // logger, - // // }); - // // task.emitLog(`Result: ${JSON.stringify(result)}`); - - // await task.complete('completed'); - // } catch (error) { - // await task.complete('failed'); - // } + registry.register({ + id: 'legacy:publish', + async handler(ctx) { + const { values } = ctx.parameters; + if ( + typeof values !== 'object' || + values === null || + Array.isArray(values) + ) { + throw new Error( + `Invalid values passed to publish, got ${typeof values}`, + ); + } + const storePath = values.storePath as unknown; + if (typeof storePath !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof storePath}`, + ); + } + const owner = values.owner as unknown; + if (typeof owner !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof owner}`, + ); + } + const publisher = publishers.get(storePath); + ctx.logger.info('Will now store the template'); + const { remoteUrl, catalogInfoUrl } = await publisher.publish({ + values: { + ...values, + owner, + storePath, + }, + workspacePath: ctx.workspacePath, + logger: ctx.logger, + }); + ctx.output('remoteUrl', remoteUrl); + if (catalogInfoUrl) { + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts index c22bb0bddb..c5920fe836 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts @@ -43,6 +43,14 @@ export class TaskAgent implements Task { return this.state.spec; } + get runId() { + return this.state.runId; + } + + get taskId() { + return this.state.taskId; + } + async emitLog(message: string): Promise { await this.storage.emit({ taskId: this.state.taskId, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index d500fc8e5c..6eb1c4763c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -17,26 +17,17 @@ import { PassThrough } from 'stream'; import { Logger } from 'winston'; import * as winston from 'winston'; -import Docker from 'dockerode'; +import { JsonValue } from '@backstage/config'; import { TaskBroker, Task } from './types'; -import { CatalogEntityClient } from '../../lib/catalog'; -import { - FilePreparer, - parseLocationAnnotation, - PreparerBuilder, - TemplaterBuilder, - PublisherBuilder, -} from '../stages'; +import { TemplateActionRegistry } from './TemplateConverter'; +import fs from 'fs-extra'; +import path from 'path'; type Options = { logger: Logger; taskBroker: TaskBroker; workingDirectory: string; - dockerClient: Docker; - entityClient: CatalogEntityClient; - preparers: PreparerBuilder; - templaters: TemplaterBuilder; - publishers: PublisherBuilder; + actionRegistry: TemplateActionRegistry; }; export class TaskWorker { @@ -52,66 +43,90 @@ export class TaskWorker { } async runOneTask(task: Task) { - const { - dockerClient, - preparers, - templaters, - publishers, - workingDirectory, - logger, - } = this.options; - - const taskLogger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: {}, - }); - - const stream = new PassThrough(); - stream.on('data', data => { - const message = data.toString().trim(); - if (message?.length > 1) task.emitLog(message); - }); - - taskLogger.add(new winston.transports.Stream({ stream })); - try { - task.emitLog('Task claimed, waiting ...'); + const { actionRegistry, logger } = this.options; + + // bbl LUUUNCH + // taskID and runId not part of task? O_o + const workspacePath = await this.createWorkPath(task.taskId, task.runId); + + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); + + const stream = new PassThrough(); + stream.on('data', data => { + const message = data.toString().trim(); + if (message?.length > 1) task.emitLog(message); + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + // Give us some time to curl observe + task.emitLog('Task claimed, waiting ...'); await new Promise(resolve => setTimeout(resolve, 5000)); + task.emitLog(`Starting up work with ${task.spec.steps.length} steps`); - const { values, template } = task.spec; - task.emitLog('Prepare the skeleton'); - const { protocol, location: pullPath } = parseLocationAnnotation( - task.spec.template, - ); + const outputs: { [name: string]: JsonValue } = {}; - const preparer = - protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); - const templater = templaters.get(template); - const publisher = publishers.get(values.storePath); + for (const step of task.spec.steps) { + task.emitLog(`Beginning step ${step.name}`); - const skeletonDir = await preparer.prepare(task.spec.template, { - logger: taskLogger, - workingDirectory: workingDirectory, - }); + const action = actionRegistry.get(step.action); + if (!action) { + throw new Error(`Action '${step.action}' does not exist`); + } - task.emitLog('Run the templater'); - const { resultDir } = await templater.run({ - directory: skeletonDir, - dockerClient, - logStream: stream, - values: values, - }); + // TODO: substitute any placeholders with output from previous steps + const parameters = step.parameters!; - task.emitLog('Publish template'); - logger.info('Will now store the template'); + await action.handler({ + logger, + logStream: stream, + parameters, + workspacePath, + output(name: string, value: JsonValue) { + outputs[name] = value; + }, + }); - logger.info('Totally storing the template now'); + task.emitLog(`Finished step ${step.name}`); + } + + // const { values, template } = task.spec; + // task.emitLog('Prepare the skeleton'); + // const { protocol, location: pullPath } = parseLocationAnnotation( + // task.spec.template, + // ); + + // const preparer = + // protocol === 'file' ? new FilePreparer() : preparers.get(pullPath); + // const templater = templaters.get(template); + // const publisher = publishers.get(values.storePath); + + // const skeletonDir = await preparer.prepare(task.spec.template, { + // logger: taskLogger, + // workingDirectory: workingDirectory, + // }); + + // task.emitLog('Run the templater'); + // const { resultDir } = await templater.run({ + // directory: skeletonDir, + // dockerClient, + // logStream: stream, + // values: values, + // }); + + // task.emitLog('Publish template'); + // logger.info('Will now store the template'); + + logger.info('So done right now'); await new Promise(resolve => setTimeout(resolve, 5000)); // const result = await publisher.publish({ // values: values, @@ -125,4 +140,14 @@ export class TaskWorker { await task.complete('failed'); } } + + async createWorkPath(taskId: string, runId: string): Promise { + const workspacePath = path.join( + this.options.workingDirectory, + taskId, + runId, + ); + fs.ensureDir(workspacePath); + return workspacePath; + } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index e435812fe5..bd06dbaa91 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -14,25 +14,37 @@ * limitations under the License. */ +import { resolve as resolvePath } from 'path'; import { JsonValue } from '@backstage/config'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { Logger } from 'winston'; import type { Writable } from 'stream'; -import { - getTemplaterKey, - parseLocationAnnotation, - TemplaterValues, -} from '../jobs/actions'; + import { TaskSpec } from './types'; import { ConflictError, NotFoundError } from '@backstage/backend-common'; +import { + getTemplaterKey, + joinGitUrlPath, + parseLocationAnnotation, + TemplaterValues, +} from '../stages'; -function templateEntityToSpec( +export function templateEntityToSpec( template: TemplateEntityV1alpha1, values: TemplaterValues, ): TaskSpec { const steps: TaskSpec['steps'] = []; - const { protocol, location: pullPath } = parseLocationAnnotation(template); + const { protocol, location } = parseLocationAnnotation(template); + + let url: string; + if (protocol === 'file') { + const path = resolvePath(location, template.spec.path || '.'); + + url = `file://${path}`; + } else { + url = joinGitUrlPath(location, template.spec.path); + } const templater = getTemplaterKey(template); steps.push({ @@ -41,7 +53,7 @@ function templateEntityToSpec( action: 'legacy:prepare', parameters: { protocol, - pullPath, + url, }, }); @@ -61,7 +73,6 @@ function templateEntityToSpec( action: 'publish', parameters: { values, - directory, }, }); @@ -72,7 +83,7 @@ type ActionContext = { logger: Logger; logStream: Writable; - workspaceDir: string; + workspacePath: string; parameters: { [name: string]: JsonValue }; output(name: string, value: JsonValue): void; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 620b6e7243..496385d4ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { JsonObject } from '@backstage/config'; +import { JsonValue } from '@backstage/config'; export type Status = | 'open' @@ -50,7 +50,7 @@ export type TaskSpec = { id: string; name: string; action: string; - parameters?: JsonObject; + parameters?: { [name: string]: JsonValue }; }>; }; @@ -60,6 +60,8 @@ export type DispatchResult = { export interface Task { spec: TaskSpec; + taskId: string; + runId: string; emitLog(message: string): Promise; complete(result: CompletedTaskState): Promise; } diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts new file mode 100644 index 0000000000..dd3d43c6d1 --- /dev/null +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import fs from 'fs-extra'; +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; + +export async function getWorkingDirectory( + config: Config, + logger: Logger, +): Promise { + if (!config.has('backend.workingDirectory')) { + return os.tmpdir(); + } + + const workingDirectory = config.getString('backend.workingDirectory'); + try { + // Check if working directory exists and is writable + await fs.access(workingDirectory, fs.constants.F_OK | fs.constants.W_OK); + logger.info(`using working directory: ${workingDirectory}`); + } catch (err) { + logger.error( + `working directory ${workingDirectory} ${ + err.code === 'ENOENT' ? 'does not exist' : 'is not writable' + }`, + ); + throw err; + } + return workingDirectory; +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fc40d45ce2..a72a5c5d48 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -38,8 +38,13 @@ import { MemoryDatabase, TaskWorker, } from '../scaffolder/tasks'; -import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateConverter'; +import { + TemplateActionRegistry, + templateEntityToSpec, +} from '../scaffolder/tasks/TemplateConverter'; import { LOCATION_ANNOTATION } from '@backstage/catalog-model'; +import { registerLegacyActions } from '../scaffolder/stages/legacy'; +import { getWorkingDirectory } from './helpers'; export interface RouterOptions { preparers: PreparerBuilder; @@ -69,18 +74,24 @@ export async function createRouter( } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); + const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); const taskBroker = new MemoryTaskBroker(new MemoryDatabase()); + const actionRegistry = new TemplateActionRegistry(); const worker = new TaskWorker({ logger, taskBroker, - workingDirectory: 'todo', + actionRegistry, + workingDirectory, + }); + + registerLegacyActions(actionRegistry, { dockerClient, - entityClient, preparers, publishers, templaters, }); + worker.start(); router @@ -127,10 +138,8 @@ export async function createRouter( res.status(400).json({ errors: validationResult.errors }); return; } - const result = await taskBroker.dispatch({ - template, - values, - }); + const taskSpec = templateEntityToSpec(template, values); + const result = await taskBroker.dispatch(taskSpec); res.status(201).json({ id: result.taskId }); })