diff --git a/packages/backend-common/src/logging/rootLogger.ts b/packages/backend-common/src/logging/rootLogger.ts index 47379b027c..fd2f3a1c8b 100644 --- a/packages/backend-common/src/logging/rootLogger.ts +++ b/packages/backend-common/src/logging/rootLogger.ts @@ -13,31 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import * as winston from 'winston'; -export function createNewRootLogger(): winston.Logger { - return winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: - process.env.NODE_ENV === 'production' - ? winston.format.json() - : winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: { service: 'backstage' }, - transports: [ - new winston.transports.Console({ - silent: - process.env.JEST_WORKER_ID !== undefined && !process.env.LOG_LEVEL, - }), - ], - }); -} - -let rootLogger: winston.Logger = createNewRootLogger(); +let rootLogger: winston.Logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: + process.env.NODE_ENV === 'production' + ? winston.format.json() + : winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: { service: 'backstage' }, + transports: [ + new winston.transports.Console({ + silent: + process.env.JEST_WORKER_ID !== undefined && !process.env.LOG_LEVEL, + }), + ], +}); export function getRootLogger(): winston.Logger { return rootLogger; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts new file mode 100644 index 0000000000..61d95b6b08 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PassThrough } from 'stream'; +import winston from 'winston'; +import { JsonValue } from '@backstage/config'; + +export const useLogStream = (meta: Record) => { + const log: string[] = []; + + // Create an empty stream to collect all the log lines into + // one variable for the API. + const stream = new PassThrough(); + stream.on('data', chunk => log.push(chunk.toString())); + + const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + ), + defaultMeta: meta, + }); + + logger.add(new winston.transports.Stream({ stream })); + + return { + log, + stream, + logger, + }; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts index a1b8795156..69d02fa7d1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts @@ -13,16 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Processor, Job, Stage, StageContext } from './types'; +import { Processor, Job, Stage, StageContext, StageInput } from './types'; import { JsonValue } from '@backstage/config'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { PassThrough } from 'stream'; import * as uuid from 'uuid'; import Docker from 'dockerode'; -import winston from 'winston'; import { RequiredTemplateValues, TemplaterBase } from '../stages/templater'; -import { createNewRootLogger } from '@backstage/backend-common'; import { PreparerBuilder } from '../stages/prepare'; +import { useLogStream } from './logger'; export type JobProcessorArguments = { preparers: PreparerBuilder; @@ -37,29 +35,18 @@ export type JobAndDirectoryTuple = { export class JobProcessor implements Processor { private jobs = new Map(); - private stages: Stage[]; - constructor({ stages }: { stages: Stage[] }) { - this.stages = stages; - } - create( - entity: TemplateEntityV1alpha1, - values: RequiredTemplateValues & Record, - ): Job { + create({ + entity, + values, + stages, + }: { + entity: TemplateEntityV1alpha1; + values: RequiredTemplateValues & Record; + stages: StageInput[]; + }): Job { const id = uuid.v4(); - const log: string[] = []; - - // Create an empty stream to collect all the log lines into - // one variable for the API. - const logStream = new PassThrough(); - logStream.on('data', chunk => log.push(chunk.toString())); - - // TODO(blam): Maybe this is not the right way to build the logger - // Maybe we want to be more ux specific and drop the json support. - // Child loggers can not have specific transports which sucks, so we have to - // create another here. - const logger = createNewRootLogger(); - logger.add(new winston.transports.Stream({ stream: logStream })); + const { logger, stream } = useLogStream({ id }); const context: StageContext = { entity, @@ -69,14 +56,10 @@ export class JobProcessor implements Processor { const job: Job = { id, - logStream, + logStream: stream, context, - stages: this.stages.map((stage) => ({})) + stages, status: 'PENDING', - metadata: { - entity, - values, - }, }; this.jobs.set(job.id, job); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts index b01fae9625..6c66a65355 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts @@ -22,45 +22,46 @@ import { Logger } from 'winston'; // Context will be a mutable object which is passed between stages // To share data, but also thinking that we can pass in functions here too // To maybe create sub steps or fail the entire thing, or skip stages down the line. -export type StageContext = T & { +export type StageContext = { values: RequiredTemplateValues & Record; entity: TemplateEntityV1alpha1; logger: Logger; -}; + logStream: Writable; +} & T; -export type Stage = { +export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; + +export interface Stage extends StageInput { log: string[]; - status: 'PENDING' | 'STARTED' | 'COMPLETE' | 'FAILED'; - name: string; - handler: (ctx: StageContext) => Promise; -}; + status: ProcessorStatus; + startedAt?: number; + endedAt?: number; +} -export type Job = { +export interface StageInput { + name: string; + handler(ctx: StageContext): Promise; +} + +export type Job = { id: string; - context: StageContext; - status: 'PENDING' | 'STARTED' | 'COMPLETE' | 'FAILED'; + context: StageContext; + status: ProcessorStatus; stages: Stage[]; logStream: Writable; - logger: Logger; error?: Error; }; -export interface ProcessorConstructor { - new (t: string): Processor; -} - -export interface Processor { - create({ +export type Processor = { + create({ entity, values, stages, }: { entity: TemplateEntityV1alpha1; values: RequiredTemplateValues & Record; - stages: Stage[]; - }): Job; + stages: StageInput[]; + }): Job; get(id: string): Job | undefined; -} - -declare let Processor: ProcessorConstructor; +}; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 5957b66915..b340190b41 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -21,6 +21,7 @@ import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import Docker from 'dockerode'; import {} from '@backstage/backend-common'; +import { StageContext, Stage } from '../scaffolder/jobs/types'; export interface RouterOptions { preparers: PreparerBuilder; templater: TemplaterBase; @@ -35,11 +36,7 @@ export async function createRouter( const { preparers, templater, logger: parentLogger, dockerClient } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); - const jobProcessor = new JobProcessor({ - preparers, - templater, - dockerClient, - }); + const jobProcessor = new JobProcessor(); router .get('/v1/job/:jobId', ({ params }, res) => { @@ -86,7 +83,45 @@ export async function createRouter( }, }; - const job = jobProcessor.create(mockEntity, { component_id: 'test' }); + const job = jobProcessor.create({ + entity: mockEntity, + values: { component_id: 'blob' }, + stages: [ + { + name: 'Prepare the skeleton', + handler: async ctx => { + const preparer = preparers.get(ctx.entity); + const skeletonDir = await preparer.prepare(ctx.entity); + return { skeletonDir }; + }, + }, + { + name: 'Run the templater', + handler: async (ctx: StageContext<{ skeletonDir: string }>) => { + const resultDir = await templater.run({ + directory: ctx.skeletonDir, + dockerClient, + logStream: ctx.logStream, + values: ctx.values, + }); + + return { resultDir }; + }, + }, + { + name: 'Create VCS Repo', + handler: async (ctx: StageContext<{ resultDir: string }>) => { + ctx.logger.info('Should now create the VCS repo'); + }, + }, + { + name: 'Push to remote', + handler: async ctx => { + ctx.logger.info('Should now push to the remote'); + }, + }, + ], + }); res.status(201).json({ jobId: job.id }); jobProcessor.process(job);