diff --git a/packages/backend-common/src/logging/rootLogger.ts b/packages/backend-common/src/logging/rootLogger.ts index 11db57c1ed..fd2f3a1c8b 100644 --- a/packages/backend-common/src/logging/rootLogger.ts +++ b/packages/backend-common/src/logging/rootLogger.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import * as winston from 'winston'; let rootLogger: winston.Logger = winston.createLogger({ diff --git a/packages/backend-common/src/middleware/errorHandler.test.ts b/packages/backend-common/src/middleware/errorHandler.test.ts index 022802dff8..e8a5a7f47a 100644 --- a/packages/backend-common/src/middleware/errorHandler.test.ts +++ b/packages/backend-common/src/middleware/errorHandler.test.ts @@ -34,6 +34,29 @@ describe('errorHandler', () => { expect(response.text).toBe('some message'); }); + it('doesnt try to send the response again if its already been sent', async () => { + const app = express(); + const mockSend = jest.fn(); + + app.use('/works_with_async_fail', (_, res) => { + res.status(200).send('hello'); + + // mutate the response object to test the middlware. + // it's hard to catch errors inside middleware from the outside. + // @ts-ignore + res.send = mockSend; + throw new Error('some message'); + }); + + app.use(errorHandler()); + const response = await request(app).get('/works_with_async_fail'); + + expect(response.status).toBe(200); + expect(response.text).toBe('hello'); + + expect(mockSend).not.toHaveBeenCalled(); + }); + it('takes code from http-errors library errors', async () => { const app = express(); app.use('/breaks', () => { diff --git a/packages/backend-common/src/middleware/errorHandler.ts b/packages/backend-common/src/middleware/errorHandler.ts index 14f6cfa8d7..ad8f170dcd 100644 --- a/packages/backend-common/src/middleware/errorHandler.ts +++ b/packages/backend-common/src/middleware/errorHandler.ts @@ -53,7 +53,10 @@ export function errorHandler( next: NextFunction, ) => { if (response.headersSent) { + // If the headers have already been sent, do not send the response again + // as this will throw an error in the backend. next(error); + return; } const status = getStatusCode(error); diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index c8c40d7b67..47f5a31158 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -37,6 +37,7 @@ "helmet": "^3.22.0", "morgan": "^1.10.0", "nodegit": "0.26.5", + "uuid": "^8.2.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 4cc3f21a9d..a1ee172184 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export * from './templater'; -export * from './prepare'; -export * from './templater/cookiecutter'; +export * from './stages/templater'; +export * from './stages/templater/cookiecutter'; +export * from './stages/prepare'; +export * from './jobs'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts new file mode 100644 index 0000000000..683d9c2750 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './processor'; +export * from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts new file mode 100644 index 0000000000..d2eaa13c3b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PassThrough } from 'stream'; +import winston from 'winston'; +import { JsonValue } from '@backstage/config'; + +export const useLogStream = (meta: Record) => { + const log: string[] = []; + + // Create an empty stream to collect all the log lines into + // one variable for the API. + const stream = new PassThrough(); + stream.on('data', chunk => log.push(chunk.toString())); + + const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: meta, + }); + + logger.add(new winston.transports.Stream({ stream })); + + return { + log, + stream, + logger, + }; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts new file mode 100644 index 0000000000..501e72c078 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts @@ -0,0 +1,278 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { JobProcessor } from './processor'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { StageInput } from './types'; + +describe('JobProcessor', () => { + const mockEntity: TemplateEntityV1alpha1 = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Template', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', + }, + name: 'graphql-starter', + title: 'GraphQL Service', + description: + 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', + uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', + etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', + + generation: 1, + }, + spec: { + type: 'cookiecutter', + path: './template', + }, + }; + + const mockValues = { component_id: 'bob' }; + + describe('create', () => { + it('creates should create a new job with a unique id', async () => { + const processor = new JobProcessor(); + + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages: [], + }); + + expect(job.id).toMatch( + /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i, + ); + }); + + it('should setup the correct context for the job', async () => { + const processor = new JobProcessor(); + + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages: [], + }); + + expect(job.context.entity).toBe(mockEntity); + expect(job.context.values).toBe(mockValues); + }); + + it('should set the status as pending', async () => { + const processor = new JobProcessor(); + + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages: [], + }); + + expect(job.status).toBe('PENDING'); + }); + + it('should create the correct stages', async () => { + const stages: StageInput[] = [ + { + name: 'Do something cool step 1', + handler: jest.fn(), + }, + { + name: 'Do something cool step 2', + handler: jest.fn(), + }, + ]; + + const processor = new JobProcessor(); + + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages, + }); + + expect(job.stages).toHaveLength(stages.length); + + for (let i = 0; i < job.stages.length; i++) { + expect(job.stages[i].name).toBe(stages[i].name); + expect(job.stages[i].status).toBe('PENDING'); + } + }); + }); + + describe('get', () => { + it('return undefined for when the job does not exist', () => { + const processor = new JobProcessor(); + expect(processor.get('123')).not.toBeDefined(); + }); + + it('should return the exact same instance of the job when one is created', async () => { + const processor = new JobProcessor(); + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages: [], + }); + + expect(processor.get(job.id)).toBe(job); + }); + }); + describe('process', () => { + it('throws an error when the status of the job is not in pending state', async () => { + const processor = new JobProcessor(); + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages: [], + }); + + job.status = 'STARTED'; + + await expect(processor.run(job)).rejects.toThrow( + /Job is not in a 'PENDING' state/, + ); + }); + + it('will call each of the handlers in the stages', async () => { + const stages: StageInput[] = [ + { + name: 'c/o', + handler: jest.fn(), + }, + { + name: 'g/p', + handler: jest.fn(), + }, + ]; + + const processor = new JobProcessor(); + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages, + }); + + await processor.run(job); + + for (const stage of stages) { + expect(stage.handler).toHaveBeenCalled(); + } + }); + + it('should set all stages to complete and the job to complete when finishes without errors', async () => { + const stages: StageInput[] = [ + { + name: 'c/o', + handler: jest.fn(), + }, + { + name: 'g/p', + handler: jest.fn(), + }, + ]; + + const processor = new JobProcessor(); + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages, + }); + + await processor.run(job); + + for (const stage of job.stages) { + expect(stage.status).toBe('COMPLETED'); + } + + expect(job.status).toBe('COMPLETED'); + }); + + it('should merge the return value from previous steps into the context of the next step', async () => { + const stages: StageInput[] = [ + { + name: 'c/o', + handler: jest + .fn() + .mockResolvedValue({ first: 'ben', second: 'lambert' }), + }, + { + name: 'g/p', + handler: jest + .fn() + .mockResolvedValue({ second: 'linus', third: 'lambert' }), + }, + { + name: 'go', + handler: jest.fn(), + }, + ]; + + const processor = new JobProcessor(); + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages, + }); + + await processor.run(job); + + expect(stages[1].handler).toHaveBeenCalledWith( + expect.objectContaining({ first: 'ben', second: 'lambert' }), + ); + + expect(stages[2].handler).toHaveBeenCalledWith( + expect.objectContaining({ + first: 'ben', + second: 'linus', + third: 'lambert', + }), + ); + }); + + it('should fail the job and the step if one of them fails', async () => { + const fail = new Error('something went wrong here'); + const stages: StageInput[] = [ + { + name: 'c/o', + handler: jest.fn(), + }, + { + name: 'g/p', + handler: jest.fn().mockRejectedValue(fail), + }, + { + name: 'go', + handler: jest.fn(), + }, + ]; + + const processor = new JobProcessor(); + const job = processor.create({ + entity: mockEntity, + values: mockValues, + stages, + }); + + await processor.run(job); + + expect(job.status).toBe('FAILED'); + expect(job.stages[0].status).toBe('COMPLETED'); + expect(job.stages[1].status).toBe('FAILED'); + expect(job.stages[2].status).toBe('PENDING'); + expect(job.error?.message).toBe('something went wrong here'); + expect(job.stages[1].log.join()).toContain('something went wrong here'); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts new file mode 100644 index 0000000000..1c76ae252d --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts @@ -0,0 +1,139 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Processor, Job, StageContext, StageInput } from './types'; +import { JsonValue } from '@backstage/config'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import * as uuid from 'uuid'; +import Docker from 'dockerode'; +import { RequiredTemplateValues, TemplaterBase } from '../stages/templater'; +import { PreparerBuilder } from '../stages/prepare'; +import { useLogStream } from './logger'; + +export type JobProcessorArguments = { + preparers: PreparerBuilder; + templater: TemplaterBase; + dockerClient: Docker; +}; + +export type JobAndDirectoryTuple = { + job: Job; + directory: string; +}; + +export class JobProcessor implements Processor { + private jobs = new Map(); + + create({ + entity, + values, + stages, + }: { + entity: TemplateEntityV1alpha1; + values: RequiredTemplateValues & Record; + stages: StageInput[]; + }): Job { + const id = uuid.v4(); + const { logger, stream } = useLogStream({ id }); + + const context: StageContext = { + entity, + values, + logger, + logStream: stream, + }; + + const job: Job = { + id, + context, + stages: stages.map(stage => ({ + handler: stage.handler, + log: [], + name: stage.name, + status: 'PENDING', + })), + status: 'PENDING', + }; + + this.jobs.set(job.id, job); + + return job; + } + + get(id: string): Job | undefined { + return this.jobs.get(id); + } + + async run(job: Job): Promise { + if (job.status !== 'PENDING') { + throw new Error("Job is not in a 'PENDING' state"); + } + + job.status = 'STARTED'; + + try { + for (const stage of job.stages) { + // Create a logger for each stage so we can create seperate + // Streams for each step. + const { logger, log, stream } = useLogStream({ + id: job.id, + stage: stage.name, + }); + // Attach the logger to the stage, and setup some timestamps. + stage.log = log; + stage.startedAt = Date.now(); + + try { + // Run the handler with the context created for the Job and some + // Additional logging helpers. + const handlerResponse = await stage.handler({ + ...job.context, + logger, + logStream: stream, + }); + + // If the handler returns something, then let's merge this onto the ontext + // For the next stage to use as it might be relevant. + if (handlerResponse) { + job.context = { + ...job.context, + ...handlerResponse, + }; + } + + // Complete the current stage + stage.status = 'COMPLETED'; + } catch (error) { + // Log to the current stage the error that occured and fail the stage. + logger.error(`Stage failed with error: ${error.message}`); + stage.status = 'FAILED'; + + // Throw the error so the job can be failed too. + throw error; + } finally { + // Always set the stage end timestamp. + stage.endedAt = Date.now(); + } + } + + // If all went to plan, complete the job. + job.status = 'COMPLETED'; + } catch (error) { + // If something went wrong, fail the job, and set the error property on the job. + job.error = { name: error.name, message: error.message }; + job.status = 'FAILED'; + } + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts new file mode 100644 index 0000000000..fd554e2033 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { Writable } from 'stream'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { JsonValue } from '@backstage/config'; +import { RequiredTemplateValues } from '../stages/templater'; +import { Logger } from 'winston'; + +// Context will be a mutable object which is passed between stages +// To share data, but also thinking that we can pass in functions here too +// To maybe create sub steps or fail the entire thing, or skip stages down the line. +export type StageContext = { + values: RequiredTemplateValues & Record; + entity: TemplateEntityV1alpha1; + logger: Logger; + logStream: Writable; +} & T; + +export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; + +export interface Stage extends StageInput { + log: string[]; + status: ProcessorStatus; + startedAt?: number; + endedAt?: number; +} + +export interface StageInput { + name: string; + handler(ctx: StageContext): Promise; +} + +export type Job = { + id: string; + context: StageContext; + status: ProcessorStatus; + stages: Stage[]; + error?: Error; +}; + +export type Processor = { + create({ + entity, + values, + stages, + }: { + entity: TemplateEntityV1alpha1; + values: RequiredTemplateValues & Record; + stages: StageInput[]; + }): Job; + + get(id: string): Job | undefined; + + run(job: Job): Promise; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/file.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/file.test.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/file.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/file.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts similarity index 97% rename from plugins/scaffolder-backend/src/scaffolder/prepare/github.test.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index 44d53248d0..86377be6d0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -60,7 +60,7 @@ describe('GitHubPreparer', () => { 1, 'https://github.com/benjdlambert/backstage-graphql-template', expect.any(String), - { checkoutOpts: { paths: ['template'] } }, + {}, ); }); it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { @@ -71,7 +71,7 @@ describe('GitHubPreparer', () => { 1, 'https://github.com/benjdlambert/backstage-graphql-template', expect.any(String), - { checkoutOpts: {} }, + {}, ); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts similarity index 89% rename from plugins/scaffolder-backend/src/scaffolder/prepare/github.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index b0ced7db2b..f53151e55d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -21,7 +21,7 @@ import { parseLocationAnnotation } from './helpers'; import { InputError } from '@backstage/backend-common'; import { PreparerBase } from './types'; import GitUriParser from 'git-url-parse'; -import { Clone, CheckoutOptions } from 'nodegit'; +import { Clone } from 'nodegit'; export class GithubPreparer implements PreparerBase { async prepare(template: TemplateEntityV1alpha1): Promise { @@ -45,13 +45,7 @@ export class GithubPreparer implements PreparerBase { template.spec.path ?? '.', ); - const checkoutOptions = new CheckoutOptions(); - if (template.spec.path) { - checkoutOptions.paths = [templateDirectory]; - } - await Clone.clone(repositoryCheckoutUrl, tempDir, { - checkoutOpts: checkoutOptions, // TODO(blam): Maybe need some auth here? }); diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/helpers.test.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/helpers.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/index.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/preparers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/preparers.test.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/preparers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/prepare/preparers.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/prepare/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts similarity index 89% rename from plugins/scaffolder-backend/src/scaffolder/prepare/types.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts index a6e42c465f..adbc38ef58 100644 --- a/plugins/scaffolder-backend/src/scaffolder/prepare/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { Logger } from 'winston'; export type PreparerBase = { /** @@ -21,7 +22,10 @@ export type PreparerBase = { * with contents from the remote location in temporary storage and return the path * @param template The template entity from the Service Catalog */ - prepare(template: TemplateEntityV1alpha1): Promise; + prepare( + template: TemplateEntityV1alpha1, + opts: { logger: Logger }, + ): Promise; }; export type PreparerBuilder = { diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts similarity index 79% rename from plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts index 7c78badff1..994307e8b7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts @@ -18,6 +18,7 @@ jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() })); import { CookieCutter } from './cookiecutter'; import fs from 'fs-extra'; import os from 'os'; +import path from 'path'; import { RunDockerContainerOptions } from './helpers'; import { PassThrough } from 'stream'; import Docker from 'dockerode'; @@ -33,12 +34,15 @@ describe('CookieCutter Templater', () => { beforeEach(async () => { jest.clearAllMocks(); - - await fs.remove(`${os.tmpdir()}/cookiecutter.json`); }); + const mkTemp = async () => { + const tempDir = os.tmpdir(); + return await fs.promises.mkdtemp(path.join(tempDir, 'temp')); + }; + it('should write a cookiecutter.json file with the values from the entitiy', async () => { - const tempdir = os.tmpdir(); + const tempdir = await mkTemp(); const values = { component_id: 'test', @@ -53,10 +57,11 @@ describe('CookieCutter Templater', () => { }); it('should merge any value that is in the cookiecutter.json path already', async () => { - const tempdir = os.tmpdir(); + const tempdir = await mkTemp(); const existingJson = { _copy_without_render: ['./github/workflows/*'], }; + await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson); const values = { @@ -72,7 +77,7 @@ describe('CookieCutter Templater', () => { }); it('should throw an error if the cookiecutter json is malformed and not missing', async () => { - const tempdir = os.tmpdir(); + const tempdir = await mkTemp(); await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'"); @@ -87,7 +92,7 @@ describe('CookieCutter Templater', () => { }); it('should run the correct docker container with the correct bindings for the volumes', async () => { - const tempdir = os.tmpdir(); + const tempdir = await mkTemp(); const values = { component_id: 'test', @@ -97,35 +102,43 @@ describe('CookieCutter Templater', () => { await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); expect(runDockerContainer).toHaveBeenCalledWith({ - imageName: 'backstage/cookiecutter', - args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + imageName: 'spotify/backstage-cookiecutter', + args: [ + 'cookiecutter', + '--no-input', + '-o', + '/result', + '/template', + '--verbose', + ], templateDir: tempdir, - resultDir: `${tempdir}/result`, + resultDir: expect.stringContaining(`${tempdir}-result`), logStream: undefined, dockerClient: mockDocker, }); }); + it('should return the result path to the end templated folder', async () => { - const tempdir = os.tmpdir(); + const tempdir = await mkTemp(); const values = { component_id: 'test', description: 'description', }; - const path = await cookie.run({ + const returnPath = await cookie.run({ directory: tempdir, values, dockerClient: mockDocker, }); - expect(path).toBe(`${tempdir}/result`); + expect(returnPath.startsWith(`${tempdir}-result`)).toBeTruthy(); }); it('should pass through the streamer to the run docker helper', async () => { const stream = new PassThrough(); - const tempdir = os.tmpdir(); + const tempdir = await mkTemp(); const values = { component_id: 'test', @@ -140,10 +153,17 @@ describe('CookieCutter Templater', () => { }); expect(runDockerContainer).toHaveBeenCalledWith({ - imageName: 'backstage/cookiecutter', - args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + imageName: 'spotify/backstage-cookiecutter', + args: [ + 'cookiecutter', + '--no-input', + '-o', + '/result', + '/template', + '--verbose', + ], templateDir: tempdir, - resultDir: `${tempdir}/result`, + resultDir: expect.stringContaining(`${tempdir}-result`), logStream: stream, dockerClient: mockDocker, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts similarity index 85% rename from plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index a95eee7fb4..cef8a64b60 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -48,14 +48,18 @@ export class CookieCutter implements TemplaterBase { await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo); const templateDir = options.directory; - - // TODO(blam): This should be an entirely different directory on the host machine - // not in the template directory - const resultDir = `${templateDir}/result`; + const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`); await runDockerContainer({ - imageName: 'backstage/cookiecutter', - args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + imageName: 'spotify/backstage-cookiecutter', + args: [ + 'cookiecutter', + '--no-input', + '-o', + '/result', + '/template', + '--verbose', + ], templateDir, resultDir, logStream: options.logStream, diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts similarity index 93% rename from plugins/scaffolder-backend/src/scaffolder/templater/index.ts rename to plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts index 8885e65308..d0bbb0aa6c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts @@ -16,6 +16,7 @@ import type { Writable } from 'stream'; import Docker from 'dockerode'; +import { JsonValue } from '@backstage/config'; export interface RequiredTemplateValues { component_id: string; @@ -23,7 +24,7 @@ export interface RequiredTemplateValues { export interface TemplaterRunOptions { directory: string; - values: RequiredTemplateValues & object; + values: RequiredTemplateValues & Record; logStream?: Writable; dockerClient: Docker; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index ab39864e76..0f43c3e0de 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -17,10 +17,11 @@ import { Logger } from 'winston'; import Router from 'express-promise-router'; import express from 'express'; -import { PreparerBuilder, TemplaterBase } from '../scaffolder'; +import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import Docker from 'dockerode'; - +import {} from '@backstage/backend-common'; +import { StageContext } from '../scaffolder/jobs/types'; export interface RouterOptions { preparers: PreparerBuilder; templater: TemplaterBase; @@ -35,51 +36,117 @@ export async function createRouter( const { preparers, templater, logger: parentLogger, dockerClient } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); - router.post('/v1/jobs', async (_, res) => { - // TODO(blam): Create a unique job here and return the ID so that - // The end user can poll for updates on the current job - res.status(201).json({ accepted: true }); + const jobProcessor = new JobProcessor(); - // TODO(blam): Take this entity from the post body sent from the frontend - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', + router + .get('/v1/job/:jobId/stage/:index/log', ({ params }, res) => { + const job = jobProcessor.get(params.jobId); + + if (!job) { + res.status(404).send({ error: 'job not found' }); + return; + } + + res.send(job.stages[Number(params.index)].log.join('')); + }) + .get('/v1/job/:jobId', ({ params }, res) => { + const job = jobProcessor.get(params.jobId); + + if (!job) { + res.status(404).send({ error: 'job not found' }); + return; + } + + res.send({ + id: job.id, + metadata: { + ...job.context, + logger: undefined, + logStream: undefined, }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', + status: job.status, + stages: job.stages.map(stage => ({ + ...stage, + handler: undefined, + })), + error: job.error, + }); + }) + .post('/v1/jobs', async (_, res) => { + // TODO(blam): Create a unique job here and return the ID so that + // The end user can poll for updates on the current job - generation: 1, - }, - spec: { - type: 'cookiecutter', - path: './template', - }, - }; + // TODO(blam): Take this entity from the post body sent from the frontend + const mockEntity: TemplateEntityV1alpha1 = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Template', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', + }, + name: 'graphql-starter', + title: 'GraphQL Service', + description: + 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', + uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', + etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - // Get the preparer for the mock entity - const preparer = preparers.get(mockEntity); + generation: 1, + }, + spec: { + type: 'cookiecutter', + path: './template', + }, + }; - // Run the preparer for the mock entity to produce a temporary directory with template in - const skeletonPath = await preparer.prepare(mockEntity); + const job = jobProcessor.create({ + entity: mockEntity, + values: { component_id: 'blob' }, + stages: [ + { + name: 'Prepare the skeleton', + handler: async ctx => { + const preparer = preparers.get(ctx.entity); + const skeletonDir = await preparer.prepare(ctx.entity, { + logger: ctx.logger, + }); + return { skeletonDir }; + }, + }, + { + name: 'Run the templater', + handler: async (ctx: StageContext<{ skeletonDir: string }>) => { + const resultDir = await templater.run({ + directory: ctx.skeletonDir, + dockerClient, + logStream: ctx.logStream, + values: ctx.values, + }); - // Run the templater on the mock directory with values from the post body - const templatedPath = await templater.run({ - directory: skeletonPath, - values: { component_id: 'test' }, - dockerClient, + return { resultDir }; + }, + }, + { + name: 'Create VCS Repo', + handler: async (ctx: StageContext<{ resultDir: string }>) => { + ctx.logger.info('Should now create the VCS repo'); + }, + }, + { + name: 'Push to remote', + handler: async ctx => { + ctx.logger.info('Should now push to the remote'); + }, + }, + ], + }); + + res.status(201).json({ id: job.id }); + + jobProcessor.run(job); }); - console.warn(templatedPath); - }); - const app = express(); app.set('logger', logger); app.use('/', router); diff --git a/yarn.lock b/yarn.lock index f4fa34b993..4ee33d5b1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18674,6 +18674,11 @@ uuid@^8.0.0: resolved "https://registry.npmjs.org/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d" integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg== +uuid@^8.2.0: + version "8.2.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-8.2.0.tgz#cb10dd6b118e2dada7d0cd9730ba7417c93d920e" + integrity sha512-CYpGiFTUrmI6OBMkAdjSDM0k5h8SkkiTP4WAjQgDgNB1S3Ou9VBEvr6q0Kv2H1mMk7IWfxYGpMH5sd5AvcIV2Q== + v8-compile-cache@^2.0.3: version "2.1.0" resolved "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz#e14de37b31a6d194f5690d67efc4e7f6fc6ab30e"