From 4e23d2276c269ce4d4e6cd16ed57c58ec0c45357 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 7 Jul 2021 17:11:09 +0200 Subject: [PATCH] chore: moved out the cookiecutter action to its own repo Signed-off-by: blam --- .../package.json | 16 +- .../src/actions/fetch/cookiecutter.test.ts | 215 ++++++++++++++++ .../src/actions/fetch/cookiecutter.ts | 242 ++++++++++++++++++ .../{setupTests.ts => actions/fetch/index.ts} | 5 +- .../src/{ => actions}/index.ts | 5 +- .../src/run.ts | 33 --- .../src/service/router.test.ts | 45 ---- .../src/service/router.ts | 40 --- .../src/service/standaloneServer.ts | 52 ---- 9 files changed, 471 insertions(+), 182 deletions(-) create mode 100644 plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts create mode 100644 plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts rename plugins/scaffolder-backend-module-cookiecutter/src/{setupTests.ts => actions/fetch/index.ts} (84%) rename plugins/scaffolder-backend-module-cookiecutter/src/{ => actions}/index.ts (85%) delete mode 100644 plugins/scaffolder-backend-module-cookiecutter/src/run.ts delete mode 100644 plugins/scaffolder-backend-module-cookiecutter/src/service/router.test.ts delete mode 100644 plugins/scaffolder-backend-module-cookiecutter/src/service/router.ts delete mode 100644 plugins/scaffolder-backend-module-cookiecutter/src/service/standaloneServer.ts diff --git a/plugins/scaffolder-backend-module-cookiecutter/package.json b/plugins/scaffolder-backend-module-cookiecutter/package.json index 5b8c7d2d95..373b22c5c2 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/package.json +++ b/plugins/scaffolder-backend-module-cookiecutter/package.json @@ -1,6 +1,6 @@ { "name": "plugin-scaffolder-backend-module-cookiecutter", - "version": "0.1.1", + "version": "0.1.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,19 +21,23 @@ }, "dependencies": { "@backstage/backend-common": "^0.8.4", + "@backstage/errors": "^0.1.1", + "@backstage/integration": "^0.5.7", "@backstage/plugin-scaffolder-backend": "^0.12.2", "@backstage/config": "^0.1.5", - "@types/express": "^4.17.6", - "express": "^4.17.1", - "express-promise-router": "^4.1.0", + "command-exists": "^1.2.9", + "fs-extra": "^10.0.0", "winston": "^3.2.1", "cross-fetch": "^3.0.6", "yn": "^4.0.0" }, "devDependencies": { "@backstage/cli": "^0.7.3", - "@types/supertest": "^2.0.8", - "supertest": "^4.0.2", + "@types/fs-extra": "^9.0.1", + "@types/mock-fs": "^4.13.0", + "@types/jest": "^26.0.7", + "@types/command-exists": "^1.2.0", + "mock-fs": "^4.13.0", "msw": "^0.29.0" }, "files": [ diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts new file mode 100644 index 0000000000..bc3fe3eca2 --- /dev/null +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.test.ts @@ -0,0 +1,215 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const runCommand = jest.fn(); +const commandExists = jest.fn(); +const fetchContents = jest.fn(); + +jest.mock('@backstage/plugin-scaffolder-backend', () => ({ + ...jest.requireActual('@backstage/plugin-scaffolder-backend'), + fetchContents, + runCommand, +})); +jest.mock('command-exists', () => commandExists); + +import { + getVoidLogger, + UrlReader, + ContainerRunner, +} from '@backstage/backend-common'; +import { ConfigReader, JsonObject } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import mockFs from 'mock-fs'; +import os from 'os'; +import { PassThrough } from 'stream'; +import { createFetchCookiecutterAction } from './cookiecutter'; +import { join } from 'path'; +import type { ActionContext } from '@backstage/plugin-scaffolder-backend'; + +describe('fetch:cookiecutter', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azure: [ + { host: 'dev.azure.com', token: 'tokenlols' }, + { host: 'myazurehostnotoken.com' }, + ], + }, + }), + ); + + const mockTmpDir = os.tmpdir(); + + let mockContext: ActionContext<{ + url: string; + targetPath?: string; + values: JsonObject; + copyWithoutRender?: string[]; + extensions?: string[]; + imageName?: string; + }>; + + const containerRunner: jest.Mocked = { + runContainer: jest.fn(), + }; + + const mockReader: UrlReader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + const action = createFetchCookiecutterAction({ + integrations, + containerRunner, + reader: mockReader, + }); + + beforeEach(() => { + jest.resetAllMocks(); + + mockContext = { + input: { + url: 'https://google.com/cookie/cutter', + targetPath: 'something', + values: { + help: 'me', + }, + }, + baseUrl: 'somebase', + workspacePath: mockTmpDir, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), + }; + + // mock the temp directory + mockFs({ [mockTmpDir]: {} }); + mockFs({ [`${join(mockTmpDir, 'template')}`]: {} }); + + commandExists.mockResolvedValue(null); + + // Mock when run container is called it creates some new files in the mock filesystem + containerRunner.runContainer.mockImplementation(async () => { + mockFs({ + [`${join(mockTmpDir, 'intermediate')}`]: { + 'testfile.json': '{}', + }, + }); + }); + + // Mock when runCommand is called it creats some new files in the mock filesystem + runCommand.mockImplementation(async () => { + mockFs({ + [`${join(mockTmpDir, 'intermediate')}`]: { + 'testfile.json': '{}', + }, + }); + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should throw an error when copyWithoutRender is not an array', async () => { + (mockContext.input as any).copyWithoutRender = 'not an array'; + + await expect(action.handler(mockContext)).rejects.toThrowError( + /Fetch action input copyWithoutRender must be an Array/, + ); + }); + + it('should throw an error when extensions is not an array', async () => { + (mockContext.input as any).extensions = 'not an array'; + + await expect(action.handler(mockContext)).rejects.toThrowError( + /Fetch action input extensions must be an Array/, + ); + }); + + it('should call fetchContents with the correct variables', async () => { + fetchContents.mockImplementation(() => Promise.resolve()); + await action.handler(mockContext); + expect(fetchContents).toHaveBeenCalledWith( + expect.objectContaining({ + reader: mockReader, + integrations, + baseUrl: mockContext.baseUrl, + fetchUrl: mockContext.input.url, + outputPath: join( + mockTmpDir, + 'template', + "{{cookiecutter and 'contents'}}", + ), + }), + ); + }); + + it('should call out to cookiecutter using runCommand when cookiecutter is installed', async () => { + commandExists.mockResolvedValue(true); + + await action.handler(mockContext); + + expect(runCommand).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'cookiecutter', + args: [ + '--no-input', + '-o', + join(mockTmpDir, 'intermediate'), + join(mockTmpDir, 'template'), + '--verbose', + ], + logStream: mockContext.logStream, + }), + ); + }); + + it('should call out to the containerRunner when there is no cookiecutter installed', async () => { + commandExists.mockResolvedValue(false); + + await action.handler(mockContext); + + expect(containerRunner.runContainer).toHaveBeenCalledWith( + expect.objectContaining({ + imageName: 'spotify/backstage-cookiecutter', + command: 'cookiecutter', + args: ['--no-input', '-o', '/output', '/input', '--verbose'], + mountDirs: { + [join(mockTmpDir, 'intermediate')]: '/output', + [join(mockTmpDir, 'template')]: '/input', + }, + workingDir: '/input', + envVars: { HOME: '/tmp' }, + logStream: mockContext.logStream, + }), + ); + }); + + it('should use a custom imageName when there is an image supplied to the context', async () => { + const imageName = 'test-image'; + mockContext.input.imageName = imageName; + + await action.handler(mockContext); + + expect(containerRunner.runContainer).toHaveBeenCalledWith( + expect.objectContaining({ + imageName, + }), + ); + }); +}); diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts new file mode 100644 index 0000000000..1f6fbdb7d8 --- /dev/null +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts @@ -0,0 +1,242 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ContainerRunner, + UrlReader, + resolveSafeChildPath, +} from '@backstage/backend-common'; +import { JsonObject, JsonValue } from '@backstage/config'; +import { InputError } from '@backstage/errors'; +import { ScmIntegrations } from '@backstage/integration'; +import commandExists from 'command-exists'; +import fs from 'fs-extra'; +import path, { resolve as resolvePath } from 'path'; +import { Writable } from 'stream'; +import { + runCommand, + createTemplateAction, + fetchContents, +} from '@backstage/plugin-scaffolder-backend'; + +export class CookiecutterRunner { + private readonly containerRunner: ContainerRunner; + + constructor({ containerRunner }: { containerRunner: ContainerRunner }) { + this.containerRunner = containerRunner; + } + + private async fetchTemplateCookieCutter( + directory: string, + ): Promise> { + try { + return await fs.readJSON(path.join(directory, 'cookiecutter.json')); + } catch (ex) { + if (ex.code !== 'ENOENT') { + throw ex; + } + + return {}; + } + } + + public async run({ + workspacePath, + values, + logStream, + }: { + workspacePath: string; + values: JsonObject; + logStream: Writable; + }): Promise { + const templateDir = path.join(workspacePath, 'template'); + const intermediateDir = path.join(workspacePath, 'intermediate'); + await fs.ensureDir(intermediateDir); + const resultDir = path.join(workspacePath, 'result'); + + // First lets grab the default cookiecutter.json file + const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir); + + const { imageName, ...valuesForCookieCutterJson } = values; + const cookieInfo = { + ...cookieCutterJson, + ...valuesForCookieCutterJson, + }; + + await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); + + // Directories to bind on container + const mountDirs = { + [templateDir]: '/input', + [intermediateDir]: '/output', + }; + + // the command-exists package returns `true` or throws an error + const cookieCutterInstalled = await commandExists('cookiecutter').catch( + () => false, + ); + if (cookieCutterInstalled) { + await runCommand({ + command: 'cookiecutter', + args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'], + logStream, + }); + } else { + await this.containerRunner.runContainer({ + imageName: (imageName as string) ?? 'spotify/backstage-cookiecutter', + command: 'cookiecutter', + args: ['--no-input', '-o', '/output', '/input', '--verbose'], + mountDirs, + workingDir: '/input', + // Set the home directory inside the container as something that applications can + // write to, otherwise they will just fail trying to write to / + envVars: { HOME: '/tmp' }, + logStream, + }); + } + + // if cookiecutter was successful, intermediateDir will contain + // exactly one directory. + + const [generated] = await fs.readdir(intermediateDir); + + if (generated === undefined) { + throw new Error('No data generated by cookiecutter'); + } + + await fs.move(path.join(intermediateDir, generated), resultDir); + } +} + +export function createFetchCookiecutterAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; + containerRunner: ContainerRunner; +}) { + const { reader, containerRunner, integrations } = options; + + return createTemplateAction<{ + url: string; + targetPath?: string; + values: JsonObject; + copyWithoutRender?: string[]; + extensions?: string[]; + imageName?: string; + }>({ + id: 'fetch:cookiecutter', + description: + 'Downloads a template from the given URL into the workspace, and runs cookiecutter on it.', + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to cookiecutter for templating', + type: 'object', + }, + copyWithoutRender: { + title: 'Copy Without Render', + description: + 'Avoid rendering directories and files in the template', + type: 'array', + items: { + type: 'string', + }, + }, + extensions: { + title: 'Template Extensions', + description: + "Jinja2 extensions to add filters, tests, globals or extend the parser. Extensions must be installed in the container or on the host where Cookiecutter executes. See the contrib directory in Backstage's repo for more information", + type: 'array', + items: { + type: 'string', + }, + }, + imageName: { + title: 'Cookiecutter Docker image', + description: + "Specify a custom Docker image to run cookiecutter, to override the default: 'spotify/backstage-cookiecutter'. This can be used to execute cookiecutter with Template Extensions. Used only when a local cookiecutter is not found.", + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + ctx.logger.info('Fetching and then templating using cookiecutter'); + const workDir = await ctx.createTemporaryDirectory(); + const templateDir = resolvePath(workDir, 'template'); + const templateContentsDir = resolvePath( + templateDir, + "{{cookiecutter and 'contents'}}", + ); + const resultDir = resolvePath(workDir, 'result'); + + if ( + ctx.input.copyWithoutRender && + !Array.isArray(ctx.input.copyWithoutRender) + ) { + throw new InputError( + 'Fetch action input copyWithoutRender must be an Array', + ); + } + if (ctx.input.extensions && !Array.isArray(ctx.input.extensions)) { + throw new InputError('Fetch action input extensions must be an Array'); + } + + await fetchContents({ + reader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.input.url, + outputPath: templateContentsDir, + }); + + const cookiecutter = new CookiecutterRunner({ containerRunner }); + const values = { + ...ctx.input.values, + _copy_without_render: ctx.input.copyWithoutRender, + _extensions: ctx.input.extensions, + imageName: ctx.input.imageName, + }; + + // Will execute the template in ./template and put the result in ./result + await cookiecutter.run({ + workspacePath: workDir, + logStream: ctx.logStream, + values, + }); + + // Finally move the template result into the task workspace + const targetPath = ctx.input.targetPath ?? './'; + const outputPath = resolveSafeChildPath(ctx.workspacePath, targetPath); + await fs.copy(resultDir, outputPath); + }, + }); +} diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/setupTests.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/index.ts similarity index 84% rename from plugins/scaffolder-backend-module-cookiecutter/src/setupTests.ts rename to plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/index.ts index d3232290a7..bbe5a16a04 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/setupTests.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -export {}; +export { createFetchCookiecutterAction } from './cookiecutter'; diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/index.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/index.ts similarity index 85% rename from plugins/scaffolder-backend-module-cookiecutter/src/index.ts rename to plugins/scaffolder-backend-module-cookiecutter/src/actions/index.ts index ca73cb27ba..1d70b91a34 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/index.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -export * from './service/router'; +export { createFetchCookiecutterAction } from './fetch'; diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/run.ts b/plugins/scaffolder-backend-module-cookiecutter/src/run.ts deleted file mode 100644 index 54d2716290..0000000000 --- a/plugins/scaffolder-backend-module-cookiecutter/src/run.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getRootLogger } from '@backstage/backend-common'; -import yn from 'yn'; -import { startStandaloneServer } from './service/standaloneServer'; - -const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 7000; -const enableCors = yn(process.env.PLUGIN_CORS, { default: false }); -const logger = getRootLogger(); - -startStandaloneServer({ port, enableCors, logger }).catch(err => { - logger.error(err); - process.exit(1); -}); - -process.on('SIGINT', () => { - logger.info('CTRL+C pressed; exiting.'); - process.exit(0); -}); diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/service/router.test.ts b/plugins/scaffolder-backend-module-cookiecutter/src/service/router.test.ts deleted file mode 100644 index 8b77a04348..0000000000 --- a/plugins/scaffolder-backend-module-cookiecutter/src/service/router.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger } from '@backstage/backend-common'; -import express from 'express'; -import request from 'supertest'; - -import { createRouter } from './router'; - -describe('createRouter', () => { - let app: express.Express; - - beforeAll(async () => { - const router = await createRouter({ - logger: getVoidLogger(), - }); - app = express().use(router); - }); - - beforeEach(() => { - jest.resetAllMocks(); - }); - - describe('GET /health', () => { - it('returns ok', async () => { - const response = await request(app).get('/health'); - - expect(response.status).toEqual(200); - expect(response.body).toEqual({ status: 'ok' }); - }); - }); -}); diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/service/router.ts b/plugins/scaffolder-backend-module-cookiecutter/src/service/router.ts deleted file mode 100644 index 9ceaa47627..0000000000 --- a/plugins/scaffolder-backend-module-cookiecutter/src/service/router.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { errorHandler } from '@backstage/backend-common'; -import express from 'express'; -import Router from 'express-promise-router'; -import { Logger } from 'winston'; - -export interface RouterOptions { - logger: Logger; -} - -export async function createRouter( - options: RouterOptions, -): Promise { - const { logger } = options; - - const router = Router(); - router.use(express.json()); - - router.get('/health', (_, response) => { - logger.info('PONG!'); - response.send({ status: 'ok' }); - }); - router.use(errorHandler()); - return router; -} diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/service/standaloneServer.ts b/plugins/scaffolder-backend-module-cookiecutter/src/service/standaloneServer.ts deleted file mode 100644 index d35fb1e5ad..0000000000 --- a/plugins/scaffolder-backend-module-cookiecutter/src/service/standaloneServer.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { createServiceBuilder } from '@backstage/backend-common'; -import { Server } from 'http'; -import { Logger } from 'winston'; -import { createRouter } from './router'; - -export interface ServerOptions { - port: number; - enableCors: boolean; - logger: Logger; -} - -export async function startStandaloneServer( - options: ServerOptions, -): Promise { - const logger = options.logger.child({ - service: 'scaffolder-backend-module-cookiecutter-backend', - }); - logger.debug('Starting application server...'); - const router = await createRouter({ - logger, - }); - - let service = createServiceBuilder(module) - .setPort(options.port) - .addRouter('/scaffolder-backend-module-cookiecutter', router); - if (options.enableCors) { - service = service.enableCors({ origin: 'http://localhost:3000' }); - } - - return await service.start().catch(err => { - logger.error(err); - process.exit(1); - }); -} - -module.hot?.accept();