diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index f2f7d3dad3..6010d31e98 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export * from './templater'; -export * from './prepare'; -export * from './templater/cookiecutter'; +export * from './stages/templater'; +export * from './stages/prepare'; +export * from './stages/templater/cookiecutter'; export * from './jobs'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts index 61d95b6b08..d2eaa13c3b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts @@ -30,6 +30,7 @@ export const useLogStream = (meta: Record) => { format: winston.format.combine( winston.format.colorize(), winston.format.timestamp(), + winston.format.simple(), ), defaultMeta: meta, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts index 69d02fa7d1..795e07291d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts @@ -52,13 +52,19 @@ export class JobProcessor implements Processor { entity, values, logger, + logStream: stream, }; const job: Job = { id, logStream: stream, context, - stages, + stages: stages.map(stage => ({ + handler: stage.handler, + log: [], + name: stage.name, + status: 'PENDING', + })), status: 'PENDING', }; @@ -71,45 +77,49 @@ export class JobProcessor implements Processor { return this.jobs.get(id); } - private async prepare(job: Job): Promise { - job.status = 'PREPARING'; - const entity = job.metadata.entity; - const preparer = this.preparers.get(entity); - return await preparer.prepare(entity); - } - - private async run(job: Job, directory: string): Promise { - job.status = 'TEMPLATING'; - return await this.templater.run({ - directory, - values: job.metadata.values, - dockerClient: this.dockerClient, - logStream: job.logStream, - }); - } - - private async store(job: Job): Promise { - job.status = 'STORING'; - } - - private async complete(job: Job): Promise { - job.status = 'COMPLETE'; - } - - async process(job: Job) { + async run(job: Job): Promise { if (job.status !== 'PENDING') { throw new Error("Job is not in a 'PENDING' state"); } + job.status = 'STARTED'; + try { - const skeletonPath = await this.prepare(job); - await this.run(job, skeletonPath); - await this.store(job); - await this.complete(job); + for (const entry of job.stages) { + const { logger, log, stream } = useLogStream({ + id: job.id, + stage: entry.name, + }); + try { + entry.startedAt = Date.now(); + + entry.log = log; + + const handler = await entry.handler({ + ...job.context, + logger, + logStream: stream, + }); + + job.context = { + ...job.context, + ...handler, + }; + + entry.status = 'COMPLETED'; + } catch (error) { + logger.error(error); + entry.status = 'FAILED'; + throw error; + } finally { + entry.endedAt = Date.now(); + } + } + job.status = 'COMPLETED'; } catch (error) { - job.error = error; + job.error = { name: error.name, message: error.message }; job.status = 'FAILED'; - job.logger.error(`Job failed with error ${error.message}`); + job.context.logger.error(`Job failed with error ${error.message}`); } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts index 6c66a65355..92e4e1cbf4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts @@ -64,4 +64,6 @@ export type Processor = { }): Job; get(id: string): Job | undefined; + + run(job: Job): Promise; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts index a6e42c465f..adbc38ef58 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { Logger } from 'winston'; export type PreparerBase = { /** @@ -21,7 +22,10 @@ export type PreparerBase = { * with contents from the remote location in temporary storage and return the path * @param template The template entity from the Service Catalog */ - prepare(template: TemplateEntityV1alpha1): Promise; + prepare( + template: TemplateEntityV1alpha1, + opts: { logger: Logger }, + ): Promise; }; export type PreparerBuilder = { diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index cb11bf04af..cef8a64b60 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -51,8 +51,15 @@ export class CookieCutter implements TemplaterBase { const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`); await runDockerContainer({ - imageName: 'backstage/cookiecutter', - args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + imageName: 'spotify/backstage-cookiecutter', + args: [ + 'cookiecutter', + '--no-input', + '-o', + '/result', + '/template', + '--verbose', + ], templateDir, resultDir, logStream: options.logStream, diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index b340190b41..0b4ec2a5b8 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -49,9 +49,16 @@ export async function createRouter( res.send({ id: job.id, - metadata: job.metadata, + metadata: { + ...job.context, + logger: undefined, + logStream: undefined, + }, status: job.status, - log: job.log, + stages: job.stages.map(stage => ({ + ...stage, + handler: undefined, + })), error: job.error, }); }) @@ -66,7 +73,7 @@ export async function createRouter( metadata: { annotations: { 'backstage.io/managed-by-location': - 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', + 'github:https://github.com/benjdlambert/backstage-graphqsl-template/blob/master/template.yaml', }, name: 'graphql-starter', title: 'GraphQL Service', @@ -91,7 +98,9 @@ export async function createRouter( name: 'Prepare the skeleton', handler: async ctx => { const preparer = preparers.get(ctx.entity); - const skeletonDir = await preparer.prepare(ctx.entity); + const skeletonDir = await preparer.prepare(ctx.entity, { + logger: ctx.logger, + }); return { skeletonDir }; }, }, @@ -122,9 +131,10 @@ export async function createRouter( }, ], }); + res.status(201).json({ jobId: job.id }); - jobProcessor.process(job); + jobProcessor.run(job); }); const app = express();