Wire up task registry with legacy tasks

Co-authored-by: Fredrik Adelöw <freben@gmail.com>
Co-authored-by: blam<ben@blam.sh>
Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>

plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts

@@ -14,21 +14,25 @@
  * limitations under the License.
  */
 
-import { Config } from '@backstage/config';
-import { TemplateActionRegistry } from '../TemplateConverter';
-import { FilePreparer } from './prepare';
+import { TemplateActionRegistry } from '../tasks/TemplateConverter';
+import { FilePreparer, PreparerBuilder } from './prepare';
 import Docker from 'dockerode';
+import { TemplaterBuilder, TemplaterValues } from './templater';
+import { PublisherBuilder } from './publish';
 
 type Options = {
-  logger: Logger;
-  config: Config;
   dockerClient: Docker;
+  preparers: PreparerBuilder;
+  templaters: TemplaterBuilder;
+  publishers: PublisherBuilder;
 };
 
 export function registerLegacyActions(
   registry: TemplateActionRegistry,
   options: Options,
 ) {
+  const { dockerClient, preparers, templaters, publishers } = options;
+
   registry.register({
     id: 'legacy:prepare',
     async handler(ctx) {
@@ -41,36 +45,18 @@ export function registerLegacyActions(
       logger.info('Prepare the skeleton');
 
       const { protocol, pullPath } = ctx.parameters;
-
+      const url = pullPath as string;
       const preparer =
-        protocol === 'file'
-          ? new FilePreparer()
-          : preparers.get(pullPath as string);
+        protocol === 'file' ? new FilePreparer() : preparers.get(url);
 
-      await preparer.prepare(task.spec.template, {
-        logger,
-        ctx.workspaceDir,
+      await preparer.prepare({
+        url,
+        logger: ctx.logger,
+        workspacePath: ctx.workspacePath,
       });
-      ctx.output('catalogInfoUrl', 'httpderp://asdasd');
     },
   });
 
-  // try {
-  //   const { values, template } = task.spec;
-  //   task.emitLog('Prepare the skeleton');
-  //   const { protocol, location: pullPath } = parseLocationAnnotation(
-  //     task.spec.template,
-  //   );
-  //   const preparer =
-  //     protocol === 'file' ? new FilePreparer() : preparers.get(pullPath);
-  //   const templater = templaters.get(template);
-  //   const publisher = publishers.get(values.storePath);
-
-  //   const skeletonDir = await preparer.prepare(task.spec.template, {
-  //     logger: taskLogger,
-  //     workingDirectory: workingDirectory,
-  //   });
-
   registry.register({
     id: 'legacy:template',
     async handler(ctx) {
@@ -79,28 +65,55 @@ export function registerLegacyActions(
       const templater = templaters.get(ctx.parameters.templater as string);
 
       logger.info('Run the templater');
-      const { resultDir } = await templater.run({
-        directory: ctx.workspaceDir,
+      await templater.run({
+        workspacePath: ctx.workspacePath,
         dockerClient,
         logStream: ctx.logStream,
         values: ctx.parameters.values as TemplaterValues,
       });
     },
   });
-  //   task.emitLog('Publish template');
-  //   logger.info('Will now store the template');
 
-  //   logger.info('Totally storing the template now');
-  //   await new Promise(resolve => setTimeout(resolve, 5000));
-  //   // const result = await publisher.publish({
-  //   //   values: values,
-  //   //   directory: resultDir,
-  //   //   logger,
-  //   // });
-  //   // task.emitLog(`Result: ${JSON.stringify(result)}`);
-
-  //   await task.complete('completed');
-  // } catch (error) {
-  //   await task.complete('failed');
-  // }
+  registry.register({
+    id: 'legacy:publish',
+    async handler(ctx) {
+      const { values } = ctx.parameters;
+      if (
+        typeof values !== 'object' ||
+        values === null ||
+        Array.isArray(values)
+      ) {
+        throw new Error(
+          `Invalid values passed to publish, got ${typeof values}`,
+        );
+      }
+      const storePath = values.storePath as unknown;
+      if (typeof storePath !== 'string') {
+        throw new Error(
+          `Invalid store path passed to publish, got ${typeof storePath}`,
+        );
+      }
+      const owner = values.owner as unknown;
+      if (typeof owner !== 'string') {
+        throw new Error(
+          `Invalid store path passed to publish, got ${typeof owner}`,
+        );
+      }
+      const publisher = publishers.get(storePath);
+      ctx.logger.info('Will now store the template');
+      const { remoteUrl, catalogInfoUrl } = await publisher.publish({
+        values: {
+          ...values,
+          owner,
+          storePath,
+        },
+        workspacePath: ctx.workspacePath,
+        logger: ctx.logger,
+      });
+      ctx.output('remoteUrl', remoteUrl);
+      if (catalogInfoUrl) {
+        ctx.output('catalogInfoUrl', catalogInfoUrl);
+      }
+    },
+  });
 }

plugins/scaffolder-backend/src/scaffolder/tasks/MemoryTaskBroker.ts

@@ -43,6 +43,14 @@ export class TaskAgent implements Task {
     return this.state.spec;
   }
 
+  get runId() {
+    return this.state.runId;
+  }
+
+  get taskId() {
+    return this.state.taskId;
+  }
+
   async emitLog(message: string): Promise<void> {
     await this.storage.emit({
       taskId: this.state.taskId,

plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts

@@ -17,26 +17,17 @@
 import { PassThrough } from 'stream';
 import { Logger } from 'winston';
 import * as winston from 'winston';
-import Docker from 'dockerode';
+import { JsonValue } from '@backstage/config';
 import { TaskBroker, Task } from './types';
-import { CatalogEntityClient } from '../../lib/catalog';
-import {
-  FilePreparer,
-  parseLocationAnnotation,
-  PreparerBuilder,
-  TemplaterBuilder,
-  PublisherBuilder,
-} from '../stages';
+import { TemplateActionRegistry } from './TemplateConverter';
+import fs from 'fs-extra';
+import path from 'path';
 
 type Options = {
   logger: Logger;
   taskBroker: TaskBroker;
   workingDirectory: string;
-  dockerClient: Docker;
-  entityClient: CatalogEntityClient;
-  preparers: PreparerBuilder;
-  templaters: TemplaterBuilder;
-  publishers: PublisherBuilder;
+  actionRegistry: TemplateActionRegistry;
 };
 
 export class TaskWorker {
@@ -52,66 +43,90 @@ export class TaskWorker {
   }
 
   async runOneTask(task: Task) {
-    const {
-      dockerClient,
-      preparers,
-      templaters,
-      publishers,
-      workingDirectory,
-      logger,
-    } = this.options;
-
-    const taskLogger = winston.createLogger({
-      level: process.env.LOG_LEVEL || 'info',
-      format: winston.format.combine(
-        winston.format.colorize(),
-        winston.format.timestamp(),
-        winston.format.simple(),
-      ),
-      defaultMeta: {},
-    });
-
-    const stream = new PassThrough();
-    stream.on('data', data => {
-      const message = data.toString().trim();
-      if (message?.length > 1) task.emitLog(message);
-    });
-
-    taskLogger.add(new winston.transports.Stream({ stream }));
-
     try {
-      task.emitLog('Task claimed, waiting ...');
+      const { actionRegistry, logger } = this.options;
+
+      // bbl LUUUNCH
+      // taskID and runId not part of task? O_o
+      const workspacePath = await this.createWorkPath(task.taskId, task.runId);
+
+      const taskLogger = winston.createLogger({
+        level: process.env.LOG_LEVEL || 'info',
+        format: winston.format.combine(
+          winston.format.colorize(),
+          winston.format.timestamp(),
+          winston.format.simple(),
+        ),
+        defaultMeta: {},
+      });
+
+      const stream = new PassThrough();
+      stream.on('data', data => {
+        const message = data.toString().trim();
+        if (message?.length > 1) task.emitLog(message);
+      });
+
+      taskLogger.add(new winston.transports.Stream({ stream }));
+
       // Give us some time to curl observe
+      task.emitLog('Task claimed, waiting ...');
       await new Promise(resolve => setTimeout(resolve, 5000));
+      task.emitLog(`Starting up work with ${task.spec.steps.length} steps`);
 
-      const { values, template } = task.spec;
-      task.emitLog('Prepare the skeleton');
-      const { protocol, location: pullPath } = parseLocationAnnotation(
-        task.spec.template,
-      );
+      const outputs: { [name: string]: JsonValue } = {};
 
-      const preparer =
-        protocol === 'file' ? new FilePreparer() : preparers.get(pullPath);
-      const templater = templaters.get(template);
-      const publisher = publishers.get(values.storePath);
+      for (const step of task.spec.steps) {
+        task.emitLog(`Beginning step ${step.name}`);
 
-      const skeletonDir = await preparer.prepare(task.spec.template, {
-        logger: taskLogger,
-        workingDirectory: workingDirectory,
-      });
+        const action = actionRegistry.get(step.action);
+        if (!action) {
+          throw new Error(`Action '${step.action}' does not exist`);
+        }
 
-      task.emitLog('Run the templater');
-      const { resultDir } = await templater.run({
-        directory: skeletonDir,
-        dockerClient,
-        logStream: stream,
-        values: values,
-      });
+        // TODO: substitute any placeholders with output from previous steps
+        const parameters = step.parameters!;
 
-      task.emitLog('Publish template');
-      logger.info('Will now store the template');
+        await action.handler({
+          logger,
+          logStream: stream,
+          parameters,
+          workspacePath,
+          output(name: string, value: JsonValue) {
+            outputs[name] = value;
+          },
+        });
 
-      logger.info('Totally storing the template now');
+        task.emitLog(`Finished step ${step.name}`);
+      }
+
+      // const { values, template } = task.spec;
+      // task.emitLog('Prepare the skeleton');
+      // const { protocol, location: pullPath } = parseLocationAnnotation(
+      //   task.spec.template,
+      // );
+
+      // const preparer =
+      //   protocol === 'file' ? new FilePreparer() : preparers.get(pullPath);
+      // const templater = templaters.get(template);
+      // const publisher = publishers.get(values.storePath);
+
+      // const skeletonDir = await preparer.prepare(task.spec.template, {
+      //   logger: taskLogger,
+      //   workingDirectory: workingDirectory,
+      // });
+
+      // task.emitLog('Run the templater');
+      // const { resultDir } = await templater.run({
+      //   directory: skeletonDir,
+      //   dockerClient,
+      //   logStream: stream,
+      //   values: values,
+      // });
+
+      // task.emitLog('Publish template');
+      // logger.info('Will now store the template');
+
+      logger.info('So done right now');
       await new Promise(resolve => setTimeout(resolve, 5000));
       // const result = await publisher.publish({
       //   values: values,
@@ -125,4 +140,14 @@ export class TaskWorker {
       await task.complete('failed');
     }
   }
+
+  async createWorkPath(taskId: string, runId: string): Promise<string> {
+    const workspacePath = path.join(
+      this.options.workingDirectory,
+      taskId,
+      runId,
+    );
+    fs.ensureDir(workspacePath);
+    return workspacePath;
+  }
 }

plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts

@@ -14,25 +14,37 @@
  * limitations under the License.
  */
 
+import { resolve as resolvePath } from 'path';
 import { JsonValue } from '@backstage/config';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { Logger } from 'winston';
 import type { Writable } from 'stream';
-import {
-  getTemplaterKey,
-  parseLocationAnnotation,
-  TemplaterValues,
-} from '../jobs/actions';
+
 import { TaskSpec } from './types';
 import { ConflictError, NotFoundError } from '@backstage/backend-common';
+import {
+  getTemplaterKey,
+  joinGitUrlPath,
+  parseLocationAnnotation,
+  TemplaterValues,
+} from '../stages';
 
-function templateEntityToSpec(
+export function templateEntityToSpec(
   template: TemplateEntityV1alpha1,
   values: TemplaterValues,
 ): TaskSpec {
   const steps: TaskSpec['steps'] = [];
 
-  const { protocol, location: pullPath } = parseLocationAnnotation(template);
+  const { protocol, location } = parseLocationAnnotation(template);
+
+  let url: string;
+  if (protocol === 'file') {
+    const path = resolvePath(location, template.spec.path || '.');
+
+    url = `file://${path}`;
+  } else {
+    url = joinGitUrlPath(location, template.spec.path);
+  }
   const templater = getTemplaterKey(template);
 
   steps.push({
@@ -41,7 +53,7 @@ function templateEntityToSpec(
     action: 'legacy:prepare',
     parameters: {
       protocol,
-      pullPath,
+      url,
     },
   });
 
@@ -61,7 +73,6 @@ function templateEntityToSpec(
     action: 'publish',
     parameters: {
       values,
-      directory,
     },
   });
 
@@ -72,7 +83,7 @@ type ActionContext = {
   logger: Logger;
   logStream: Writable;
 
-  workspaceDir: string;
+  workspacePath: string;
   parameters: { [name: string]: JsonValue };
   output(name: string, value: JsonValue): void;
 };

plugins/scaffolder-backend/src/scaffolder/tasks/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { JsonObject } from '@backstage/config';
+import { JsonValue } from '@backstage/config';
 
 export type Status =
   | 'open'
@@ -50,7 +50,7 @@ export type TaskSpec = {
     id: string;
     name: string;
     action: string;
-    parameters?: JsonObject;
+    parameters?: { [name: string]: JsonValue };
   }>;
 };
 
@@ -60,6 +60,8 @@ export type DispatchResult = {
 
 export interface Task {
   spec: TaskSpec;
+  taskId: string;
+  runId: string;
   emitLog(message: string): Promise<void>;
   complete(result: CompletedTaskState): Promise<void>;
 }

plugins/scaffolder-backend/src/service/helpers.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import fs from 'fs-extra';
+import { Logger } from 'winston';
+import { Config } from '@backstage/config';
+
+export async function getWorkingDirectory(
+  config: Config,
+  logger: Logger,
+): Promise<string> {
+  if (!config.has('backend.workingDirectory')) {
+    return os.tmpdir();
+  }
+
+  const workingDirectory = config.getString('backend.workingDirectory');
+  try {
+    // Check if working directory exists and is writable
+    await fs.access(workingDirectory, fs.constants.F_OK | fs.constants.W_OK);
+    logger.info(`using working directory: ${workingDirectory}`);
+  } catch (err) {
+    logger.error(
+      `working directory ${workingDirectory} ${
+        err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
+      }`,
+    );
+    throw err;
+  }
+  return workingDirectory;
+}

plugins/scaffolder-backend/src/service/router.ts

@@ -38,8 +38,13 @@ import {
   MemoryDatabase,
   TaskWorker,
 } from '../scaffolder/tasks';
-import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateConverter';
+import {
+  TemplateActionRegistry,
+  templateEntityToSpec,
+} from '../scaffolder/tasks/TemplateConverter';
 import { LOCATION_ANNOTATION } from '@backstage/catalog-model';
+import { registerLegacyActions } from '../scaffolder/stages/legacy';
+import { getWorkingDirectory } from './helpers';
 
 export interface RouterOptions {
   preparers: PreparerBuilder;
@@ -69,18 +74,24 @@ export async function createRouter(
   } = options;
 
   const logger = parentLogger.child({ plugin: 'scaffolder' });
+  const workingDirectory = await getWorkingDirectory(config, logger);
   const jobProcessor = await JobProcessor.fromConfig({ config, logger });
   const taskBroker = new MemoryTaskBroker(new MemoryDatabase());
+  const actionRegistry = new TemplateActionRegistry();
   const worker = new TaskWorker({
     logger,
     taskBroker,
-    workingDirectory: 'todo',
+    actionRegistry,
+    workingDirectory,
+  });
+
+  registerLegacyActions(actionRegistry, {
     dockerClient,
-    entityClient,
     preparers,
     publishers,
     templaters,
   });
+
   worker.start();
 
   router
@@ -127,10 +138,8 @@ export async function createRouter(
         res.status(400).json({ errors: validationResult.errors });
         return;
       }
-      const result = await taskBroker.dispatch({
-        template,
-        values,
-      });
+      const taskSpec = templateEntityToSpec(template, values);
+      const result = await taskBroker.dispatch(taskSpec);
 
       res.status(201).json({ id: result.taskId });
     })