chore(scaffolder): Reworking how the processor works. It's starting to look a lot cleaner now

plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts

@@ -13,4 +13,36 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-describe('JobProcessor', () => {});
+import { JobProcessor } from './processor';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+describe('JobProcessor', () => {
+  describe('create', () => {
+    const mockEntity: TemplateEntityV1alpha1 = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Template',
+      metadata: {
+        annotations: {
+          'backstage.io/managed-by-location':
+            'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+        },
+        name: 'graphql-starter',
+        title: 'GraphQL Service',
+        description:
+          'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+        uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+        etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
+
+        generation: 1,
+      },
+      spec: {
+        type: 'cookiecutter',
+        path: './template',
+      },
+    };
+    const processor = new JobProcessor();
+
+    it('should create a unique id for the job', async () => {
+      const job = processor.create();
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -13,30 +13,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Processor, Job, ProcessorContstructorArgs } from './types';
+import { Processor, Job } from './types';
 import { JsonValue } from '@backstage/config';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { PassThrough } from 'stream';
 import uuid from 'uuid';
+import Docker from 'dockerode';
 import winston from 'winston';
-import { RequiredTemplateValues } from '../templater';
+import { RequiredTemplateValues, TemplaterBase } from '../templater';
 import { createNewRootLogger } from '@backstage/backend-common';
+import { PreparerBuilder } from '../prepare';
+
+export type JobProcessorArguments = {
+  preparers: PreparerBuilder;
+  templater: TemplaterBase;
+  dockerClient: Docker;
+};
+
+export type JobAndDirectoryTuple = {
+  job: Job;
+  directory: string;
+};
 
 export class JobProcessor implements Processor {
-  private preparers: ProcessorContstructorArgs['preparers'];
-  private templater: ProcessorContstructorArgs['templater'];
-  private dockerClient: ProcessorContstructorArgs['dockerClient'];
+  private preparers: PreparerBuilder;
+  private templater: TemplaterBase;
+  private dockerClient: Docker;
   private jobs = new Map<string, Job>();
 
-  constructor({
-    preparers,
-    templater,
-    dockerClient,
-  }: ProcessorContstructorArgs) {
+  constructor({ preparers, templater, dockerClient }: JobProcessorArguments) {
     this.preparers = preparers;
     this.templater = templater;
     this.dockerClient = dockerClient;
-    return this;
   }
 
   create(
@@ -74,47 +82,50 @@ export class JobProcessor implements Processor {
 
     return job;
   }
+
   get(id: string): Job | undefined {
     return this.jobs.get(id);
   }
-  async run(job: Job) {
+
+  private async prepare(job: Job): Promise<string> {
+    job.status = 'PREPARING';
+    const entity = job.metadata.entity;
+    const preparer = this.preparers.get(entity);
+    return await preparer.prepare(entity);
+  }
+
+  private async run(job: Job, directory: string): Promise<string> {
+    job.status = 'TEMPLATING';
+    return await this.templater.run({
+      directory,
+      values: job.metadata.values,
+      dockerClient: this.dockerClient,
+      logStream: job.logStream,
+    });
+  }
+
+  private async store(job: Job): Promise<void> {
+    job.status = 'STORING';
+  }
+
+  private async complete(job: Job): Promise<void> {
+    job.status = 'COMPLETE';
+  }
+
+  async process(job: Job) {
     if (job.status !== 'PENDING') {
       throw new Error('Job is not in pending state');
     }
 
-    const { logger, logStream } = job;
-
     try {
-      // Prepare a folder for the templater to run in
-      logger.debug('Prepare started');
-      job.status = 'PREPARING';
-      const entity = job.metadata.entity;
-      const preparer = this.preparers.get(entity);
-      const skeletonPath = await preparer.prepare(entity);
-      logger.debug('Prepare finished', {
-        skeletonPath,
-      });
-
-      // Run the templater on the directory with values passed in
-      logger.debug('Templating started');
-      job.status = 'TEMPLATING';
-      const templatedPath = await this.templater.run({
-        directory: skeletonPath,
-        values: job.metadata.values,
-        dockerClient: this.dockerClient,
-        logStream,
-      });
-      logger.debug('Template finished', { templatedPath });
-
-      // Store the template somewhere when finished
-      job.status = 'STORING';
-      // TODO(blam): Implement VCS Push here
-
-      job.status = 'COMPLETE';
+      const skeletonPath = await this.prepare(job);
+      await this.run(job, skeletonPath);
+      await this.store(job);
+      await this.complete(job);
     } catch (error) {
       job.error = error;
       job.status = 'FAILED';
-      logger.error(`Job failed with error ${error.message}`);
+      job.logger.error(`Job failed with error ${error.message}`);
     }
   }
 }

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -16,9 +16,7 @@
 import type { Writable } from 'stream';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { JsonValue } from '@backstage/config';
-import { PreparerBuilder } from '../prepare';
-import Docker from 'dockerode';
-import { TemplaterBase, RequiredTemplateValues } from '../templater';
+import { RequiredTemplateValues } from '../templater';
 import { Logger } from 'winston';
 
 export type Job = {
@@ -40,13 +38,6 @@ export type Job = {
   error?: Error;
 };
 
-export type ProcessorContstructorArgs = {
-  preparers: PreparerBuilder;
-  templater: TemplaterBase;
-  logger: Logger;
-  dockerClient: Docker;
-};
-
 export type Processor = {
   create(
     entity: TemplateEntityV1alpha1,

plugins/scaffolder-backend/src/service/router.ts

@@ -42,57 +42,57 @@ export async function createRouter(
     dockerClient,
   });
 
-  router.get('/v1/job/:jobId', ({ params }, res) => {
-    const job = jobProcessor.get(params.jobId);
+  router
+    .get('/v1/job/:jobId', ({ params }, res) => {
+      const job = jobProcessor.get(params.jobId);
 
-    if (!job) {
-      return res.status(404).send({ error: 'job not found' });
-    }
+      if (!job) {
+        return res.status(404).send({ error: 'job not found' });
+      }
 
-    res.send({
-      id: job.id,
-      metadata: job.metadata,
-      status: job.status,
-      log: job.log,
-      error: job.error,
-    });
-  });
+      res.send({
+        id: job.id,
+        metadata: job.metadata,
+        status: job.status,
+        log: job.log,
+        error: job.error,
+      });
+    })
+    .post('/v1/jobs', async (_, res) => {
+      // TODO(blam): Create a unique job here and return the ID so that
+      // The end user can poll for updates on the current job
 
-  router.post('/v1/jobs', async (_, res) => {
-    // TODO(blam): Create a unique job here and return the ID so that
-    // The end user can poll for updates on the current job
+      // TODO(blam): Take this entity from the post body sent from the frontend
+      const mockEntity: TemplateEntityV1alpha1 = {
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Template',
+        metadata: {
+          annotations: {
+            'backstage.io/managed-by-location':
+              'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+          },
+          name: 'graphql-starter',
+          title: 'GraphQL Service',
+          description:
+            'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+          uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+          etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
 
-    // TODO(blam): Take this entity from the post body sent from the frontend
-    const mockEntity: TemplateEntityV1alpha1 = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Template',
-      metadata: {
-        annotations: {
-          'backstage.io/managed-by-location':
-            'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
+          generation: 1,
         },
-        name: 'graphql-starter',
-        title: 'GraphQL Service',
-        description:
-          'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
-        uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
-        etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
+        spec: {
+          type: 'cookiecutter',
+          path: './template',
+        },
+      };
 
-        generation: 1,
-      },
-      spec: {
-        type: 'cookiecutter',
-        path: './template',
-      },
-    };
+      const job = jobProcessor.create(mockEntity, { component_id: 'test' });
+      res.status(201).json({ jobId: job.id });
 
-    const job = jobProcessor.create(mockEntity, { component_id: 'test' });
-    res.status(201).json({ jobId: job.id });
+      jobProcessor.run(job);
 
-    jobProcessor.run(job);
-
-    // console.warn(templatedPath);
-  });
+      // console.warn(templatedPath);
+    });
 
   const app = express();
   app.set('logger', logger);