feat(scaffolder): reworking how the loading of stages goes and made it super flexible

plugins/scaffolder-backend/src/scaffolder/index.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export * from './templater';
-export * from './prepare';
-export * from './templater/cookiecutter';
+export * from './stages/templater';
+export * from './stages/prepare';
+export * from './stages/templater/cookiecutter';
 export * from './jobs';

plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts

@@ -30,6 +30,7 @@ export const useLogStream = (meta: Record<string, JsonValue>) => {
     format: winston.format.combine(
       winston.format.colorize(),
       winston.format.timestamp(),
+      winston.format.simple(),
     ),
     defaultMeta: meta,
   });

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -52,13 +52,19 @@ export class JobProcessor implements Processor {
       entity,
       values,
       logger,
+      logStream: stream,
     };
 
     const job: Job = {
       id,
       logStream: stream,
       context,
-      stages,
+      stages: stages.map(stage => ({
+        handler: stage.handler,
+        log: [],
+        name: stage.name,
+        status: 'PENDING',
+      })),
       status: 'PENDING',
     };
 
@@ -71,45 +77,49 @@ export class JobProcessor implements Processor {
     return this.jobs.get(id);
   }
 
-  private async prepare(job: Job): Promise<string> {
-    job.status = 'PREPARING';
-    const entity = job.metadata.entity;
-    const preparer = this.preparers.get(entity);
-    return await preparer.prepare(entity);
-  }
-
-  private async run(job: Job, directory: string): Promise<string> {
-    job.status = 'TEMPLATING';
-    return await this.templater.run({
-      directory,
-      values: job.metadata.values,
-      dockerClient: this.dockerClient,
-      logStream: job.logStream,
-    });
-  }
-
-  private async store(job: Job): Promise<void> {
-    job.status = 'STORING';
-  }
-
-  private async complete(job: Job): Promise<void> {
-    job.status = 'COMPLETE';
-  }
-
-  async process(job: Job) {
+  async run(job: Job): Promise<void> {
     if (job.status !== 'PENDING') {
       throw new Error("Job is not in a 'PENDING' state");
     }
 
+    job.status = 'STARTED';
+
     try {
-      const skeletonPath = await this.prepare(job);
-      await this.run(job, skeletonPath);
-      await this.store(job);
-      await this.complete(job);
+      for (const entry of job.stages) {
+        const { logger, log, stream } = useLogStream({
+          id: job.id,
+          stage: entry.name,
+        });
+        try {
+          entry.startedAt = Date.now();
+
+          entry.log = log;
+
+          const handler = await entry.handler({
+            ...job.context,
+            logger,
+            logStream: stream,
+          });
+
+          job.context = {
+            ...job.context,
+            ...handler,
+          };
+
+          entry.status = 'COMPLETED';
+        } catch (error) {
+          logger.error(error);
+          entry.status = 'FAILED';
+          throw error;
+        } finally {
+          entry.endedAt = Date.now();
+        }
+      }
+      job.status = 'COMPLETED';
     } catch (error) {
-      job.error = error;
+      job.error = { name: error.name, message: error.message };
       job.status = 'FAILED';
-      job.logger.error(`Job failed with error ${error.message}`);
+      job.context.logger.error(`Job failed with error ${error.message}`);
     }
   }
 }

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -64,4 +64,6 @@ export type Processor = {
   }): Job;
 
   get(id: string): Job | undefined;
+
+  run(job: Job): Promise<void>;
 };

plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { Logger } from 'winston';
 
 export type PreparerBase = {
   /**
@@ -21,7 +22,10 @@ export type PreparerBase = {
    * with contents from the remote location in temporary storage and return the path
    * @param template The template entity from the Service Catalog
    */
-  prepare(template: TemplateEntityV1alpha1): Promise<string>;
+  prepare(
+    template: TemplateEntityV1alpha1,
+    opts: { logger: Logger },
+  ): Promise<string>;
 };
 
 export type PreparerBuilder = {

plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts

@@ -51,8 +51,15 @@ export class CookieCutter implements TemplaterBase {
     const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
 
     await runDockerContainer({
-      imageName: 'backstage/cookiecutter',
-      args: ['cookiecutter', '--no-input', '-o', '/result', '/template'],
+      imageName: 'spotify/backstage-cookiecutter',
+      args: [
+        'cookiecutter',
+        '--no-input',
+        '-o',
+        '/result',
+        '/template',
+        '--verbose',
+      ],
       templateDir,
       resultDir,
       logStream: options.logStream,

plugins/scaffolder-backend/src/service/router.ts

@@ -49,9 +49,16 @@ export async function createRouter(
 
       res.send({
         id: job.id,
-        metadata: job.metadata,
+        metadata: {
+          ...job.context,
+          logger: undefined,
+          logStream: undefined,
+        },
         status: job.status,
-        log: job.log,
+        stages: job.stages.map(stage => ({
+          ...stage,
+          handler: undefined,
+        })),
         error: job.error,
       });
     })
@@ -66,7 +73,7 @@ export async function createRouter(
         metadata: {
           annotations: {
             'backstage.io/managed-by-location':
-              'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+              'github:https://github.com/benjdlambert/backstage-graphqsl-template/blob/master/template.yaml',
           },
           name: 'graphql-starter',
           title: 'GraphQL Service',
@@ -91,7 +98,9 @@ export async function createRouter(
             name: 'Prepare the skeleton',
             handler: async ctx => {
               const preparer = preparers.get(ctx.entity);
-              const skeletonDir = await preparer.prepare(ctx.entity);
+              const skeletonDir = await preparer.prepare(ctx.entity, {
+                logger: ctx.logger,
+              });
               return { skeletonDir };
             },
           },
@@ -122,9 +131,10 @@ export async function createRouter(
           },
         ],
       });
+
       res.status(201).json({ jobId: job.id });
 
-      jobProcessor.process(job);
+      jobProcessor.run(job);
     });
 
   const app = express();