feat(scaffolder): added an endpoint for getting each stage logs as plaintext in case that's what is needed.

plugins/scaffolder-backend/src/scaffolder/index.ts

@@ -14,6 +14,6 @@
  * limitations under the License.
  */
 export * from './stages/templater';
-export * from './stages/prepare';
 export * from './stages/templater/cookiecutter';
+export * from './stages/prepare';
 export * from './jobs';

plugins/scaffolder-backend/src/scaffolder/jobs/index.ts

@@ -14,3 +14,4 @@
  * limitations under the License.
  */
 export * from './processor';
+export * from './types';

plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts

@@ -57,7 +57,6 @@ export class JobProcessor implements Processor {
 
     const job: Job = {
       id,
-      logStream: stream,
       context,
       stages: stages.map(stage => ({
         handler: stage.handler,
@@ -85,41 +84,56 @@ export class JobProcessor implements Processor {
     job.status = 'STARTED';
 
     try {
-      for (const entry of job.stages) {
+      for (const stage of job.stages) {
+        // Create a logger for each stage so we can create seperate
+        // Streams for each step.
         const { logger, log, stream } = useLogStream({
           id: job.id,
-          stage: entry.name,
+          stage: stage.name,
         });
+        // Attach the logger to the stage, and setup some timestamps.
+        stage.log = log;
+        stage.startedAt = Date.now();
+
         try {
-          entry.startedAt = Date.now();
-
-          entry.log = log;
-
-          const handler = await entry.handler({
+          // Run the handler with the context created for the Job and some
+          // Additional logging helpers.
+          const handlerResponse = await stage.handler({
             ...job.context,
             logger,
             logStream: stream,
           });
 
-          job.context = {
-            ...job.context,
-            ...handler,
-          };
+          // If the handler returns something, then let's merge this onto the ontext
+          // For the next stage to use as it might be relevant.
+          if (handlerResponse) {
+            job.context = {
+              ...job.context,
+              ...handlerResponse,
+            };
+          }
 
-          entry.status = 'COMPLETED';
+          // Complete the current stage
+          stage.status = 'COMPLETED';
         } catch (error) {
-          logger.error(error);
-          entry.status = 'FAILED';
+          // Log to the current stage the error that occured and fail the stage.
+          logger.error(`Stage failed with error: ${error.message}`);
+          stage.status = 'FAILED';
+
+          // Throw the error so the job can be failed too.
           throw error;
         } finally {
-          entry.endedAt = Date.now();
+          // Always set the stage end timestamp.
+          stage.endedAt = Date.now();
         }
       }
+
+      // If all went to plan, complete the job.
       job.status = 'COMPLETED';
     } catch (error) {
+      // If something went wrong, fail the job, and set the error property on the job.
       job.error = { name: error.name, message: error.message };
       job.status = 'FAILED';
-      job.context.logger.error(`Job failed with error ${error.message}`);
     }
   }
 }

plugins/scaffolder-backend/src/scaffolder/jobs/types.ts

@@ -48,7 +48,6 @@ export type Job = {
   context: StageContext;
   status: ProcessorStatus;
   stages: Stage[];
-  logStream: Writable;
   error?: Error;
 };
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -60,7 +60,7 @@ describe('GitHubPreparer', () => {
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',
       expect.any(String),
-      { checkoutOpts: { paths: ['template'] } },
+      {},
     );
   });
   it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
@@ -71,7 +71,7 @@ describe('GitHubPreparer', () => {
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',
       expect.any(String),
-      { checkoutOpts: {} },
+      {},
     );
   });
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -21,7 +21,7 @@ import { parseLocationAnnotation } from './helpers';
 import { InputError } from '@backstage/backend-common';
 import { PreparerBase } from './types';
 import GitUriParser from 'git-url-parse';
-import { Clone, CheckoutOptions } from 'nodegit';
+import { Clone } from 'nodegit';
 
 export class GithubPreparer implements PreparerBase {
   async prepare(template: TemplateEntityV1alpha1): Promise<string> {
@@ -45,13 +45,7 @@ export class GithubPreparer implements PreparerBase {
       template.spec.path ?? '.',
     );
 
-    const checkoutOptions = new CheckoutOptions();
-    if (template.spec.path) {
-      checkoutOptions.paths = [templateDirectory];
-    }
-
     await Clone.clone(repositoryCheckoutUrl, tempDir, {
-      checkoutOpts: checkoutOptions,
       // TODO(blam): Maybe need some auth here?
     });
 

plugins/scaffolder-backend/src/service/router.ts

@@ -21,7 +21,7 @@ import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import Docker from 'dockerode';
 import {} from '@backstage/backend-common';
-import { StageContext, Stage } from '../scaffolder/jobs/types';
+import { StageContext } from '../scaffolder/jobs/types';
 export interface RouterOptions {
   preparers: PreparerBuilder;
   templater: TemplaterBase;
@@ -39,6 +39,16 @@ export async function createRouter(
   const jobProcessor = new JobProcessor();
 
   router
+    .get('/v1/job/:jobId/stage/:index/log', ({ params }, res) => {
+      const job = jobProcessor.get(params.jobId);
+
+      if (!job) {
+        res.status(404).send({ error: 'job not found' });
+        return;
+      }
+
+      res.send(job.stages[Number(params.index)].log.join(''));
+    })
     .get('/v1/job/:jobId', ({ params }, res) => {
       const job = jobProcessor.get(params.jobId);
 
@@ -73,7 +83,7 @@ export async function createRouter(
         metadata: {
           annotations: {
             'backstage.io/managed-by-location':
-              'github:https://github.com/benjdlambert/backstage-graphqsl-template/blob/master/template.yaml',
+              'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
           },
           name: 'graphql-starter',
           title: 'GraphQL Service',
@@ -132,7 +142,7 @@ export async function createRouter(
         ],
       });
 
-      res.status(201).json({ jobId: job.id });
+      res.status(201).json({ id: job.id });
 
       jobProcessor.run(job);
     });