From c96075ce8c94158f9616e8b05e5ac3a68193987a Mon Sep 17 00:00:00 2001 From: blam Date: Sat, 27 Jun 2020 23:18:15 +0200 Subject: [PATCH] feat(scaffolder): added an endpoint for getting each stage logs as plaintext in case that's what is needed. --- .../src/scaffolder/index.ts | 2 +- .../src/scaffolder/jobs/index.ts | 1 + .../src/scaffolder/jobs/processor.ts | 48 ++++++++++++------- .../src/scaffolder/jobs/types.ts | 1 - .../scaffolder/stages/prepare/github.test.ts | 4 +- .../src/scaffolder/stages/prepare/github.ts | 8 +--- .../scaffolder-backend/src/service/router.ts | 16 +++++-- 7 files changed, 49 insertions(+), 31 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 6010d31e98..a1ee172184 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ export * from './stages/templater'; -export * from './stages/prepare'; export * from './stages/templater/cookiecutter'; +export * from './stages/prepare'; export * from './jobs'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts index 303987c5b1..683d9c2750 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export * from './processor'; +export * from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts index 89f2e89d97..1c76ae252d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts @@ -57,7 +57,6 @@ export class JobProcessor implements Processor { const job: Job = { id, - logStream: stream, context, stages: stages.map(stage => ({ handler: stage.handler, @@ -85,41 +84,56 @@ export class JobProcessor implements Processor { job.status = 'STARTED'; try { - for (const entry of job.stages) { + for (const stage of job.stages) { + // Create a logger for each stage so we can create seperate + // Streams for each step. const { logger, log, stream } = useLogStream({ id: job.id, - stage: entry.name, + stage: stage.name, }); + // Attach the logger to the stage, and setup some timestamps. + stage.log = log; + stage.startedAt = Date.now(); + try { - entry.startedAt = Date.now(); - - entry.log = log; - - const handler = await entry.handler({ + // Run the handler with the context created for the Job and some + // Additional logging helpers. + const handlerResponse = await stage.handler({ ...job.context, logger, logStream: stream, }); - job.context = { - ...job.context, - ...handler, - }; + // If the handler returns something, then let's merge this onto the ontext + // For the next stage to use as it might be relevant. + if (handlerResponse) { + job.context = { + ...job.context, + ...handlerResponse, + }; + } - entry.status = 'COMPLETED'; + // Complete the current stage + stage.status = 'COMPLETED'; } catch (error) { - logger.error(error); - entry.status = 'FAILED'; + // Log to the current stage the error that occured and fail the stage. + logger.error(`Stage failed with error: ${error.message}`); + stage.status = 'FAILED'; + + // Throw the error so the job can be failed too. throw error; } finally { - entry.endedAt = Date.now(); + // Always set the stage end timestamp. + stage.endedAt = Date.now(); } } + + // If all went to plan, complete the job. job.status = 'COMPLETED'; } catch (error) { + // If something went wrong, fail the job, and set the error property on the job. job.error = { name: error.name, message: error.message }; job.status = 'FAILED'; - job.context.logger.error(`Job failed with error ${error.message}`); } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts index 92e4e1cbf4..fd554e2033 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts @@ -48,7 +48,6 @@ export type Job = { context: StageContext; status: ProcessorStatus; stages: Stage[]; - logStream: Writable; error?: Error; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index 44d53248d0..86377be6d0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -60,7 +60,7 @@ describe('GitHubPreparer', () => { 1, 'https://github.com/benjdlambert/backstage-graphql-template', expect.any(String), - { checkoutOpts: { paths: ['template'] } }, + {}, ); }); it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { @@ -71,7 +71,7 @@ describe('GitHubPreparer', () => { 1, 'https://github.com/benjdlambert/backstage-graphql-template', expect.any(String), - { checkoutOpts: {} }, + {}, ); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index b0ced7db2b..f53151e55d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -21,7 +21,7 @@ import { parseLocationAnnotation } from './helpers'; import { InputError } from '@backstage/backend-common'; import { PreparerBase } from './types'; import GitUriParser from 'git-url-parse'; -import { Clone, CheckoutOptions } from 'nodegit'; +import { Clone } from 'nodegit'; export class GithubPreparer implements PreparerBase { async prepare(template: TemplateEntityV1alpha1): Promise { @@ -45,13 +45,7 @@ export class GithubPreparer implements PreparerBase { template.spec.path ?? '.', ); - const checkoutOptions = new CheckoutOptions(); - if (template.spec.path) { - checkoutOptions.paths = [templateDirectory]; - } - await Clone.clone(repositoryCheckoutUrl, tempDir, { - checkoutOpts: checkoutOptions, // TODO(blam): Maybe need some auth here? }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 0b4ec2a5b8..0f43c3e0de 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -21,7 +21,7 @@ import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import Docker from 'dockerode'; import {} from '@backstage/backend-common'; -import { StageContext, Stage } from '../scaffolder/jobs/types'; +import { StageContext } from '../scaffolder/jobs/types'; export interface RouterOptions { preparers: PreparerBuilder; templater: TemplaterBase; @@ -39,6 +39,16 @@ export async function createRouter( const jobProcessor = new JobProcessor(); router + .get('/v1/job/:jobId/stage/:index/log', ({ params }, res) => { + const job = jobProcessor.get(params.jobId); + + if (!job) { + res.status(404).send({ error: 'job not found' }); + return; + } + + res.send(job.stages[Number(params.index)].log.join('')); + }) .get('/v1/job/:jobId', ({ params }, res) => { const job = jobProcessor.get(params.jobId); @@ -73,7 +83,7 @@ export async function createRouter( metadata: { annotations: { 'backstage.io/managed-by-location': - 'github:https://github.com/benjdlambert/backstage-graphqsl-template/blob/master/template.yaml', + 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', }, name: 'graphql-starter', title: 'GraphQL Service', @@ -132,7 +142,7 @@ export async function createRouter( ], }); - res.status(201).json({ jobId: job.id }); + res.status(201).json({ id: job.id }); jobProcessor.run(job); });