From 4e9453d6ed0c5099b28fcc3a81f192de8b123628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Fri, 23 Oct 2020 14:42:59 +0200 Subject: [PATCH] Add workdir config support to scaffolder --- app-config.yaml | 1 + packages/backend/src/plugins/scaffolder.ts | 1 + .../templates/default-app/app-config.yaml.hbs | 1 + .../scaffolder/stages/prepare/azure.test.ts | 19 +++++++++----- .../src/scaffolder/stages/prepare/azure.ts | 15 ++++++----- .../scaffolder/stages/prepare/file.test.ts | 5 +++- .../src/scaffolder/stages/prepare/file.ts | 9 ++++--- .../scaffolder/stages/prepare/github.test.ts | 20 ++++++++++----- .../src/scaffolder/stages/prepare/github.ts | 9 ++++--- .../scaffolder/stages/prepare/gitlab.test.ts | 19 +++++++++----- .../src/scaffolder/stages/prepare/gitlab.ts | 10 +++++--- .../src/scaffolder/stages/prepare/types.ts | 2 +- .../src/service/router.test.ts | 2 ++ .../scaffolder-backend/src/service/router.ts | 25 ++++++++++++++++++- 14 files changed, 99 insertions(+), 39 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 6e3a68a229..009876a64b 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -155,6 +155,7 @@ catalog: target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme-corp.yaml scaffolder: + # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir github: token: $env: GITHUB_TOKEN diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index c84928ea68..6eacd143c1 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -45,6 +45,7 @@ export default async function createPlugin({ templaters, publishers, logger, + config, dockerClient, }); } diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 1f265d7e7d..9f66e7db0a 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -69,6 +69,7 @@ auth: providers: {} scaffolder: + # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir github: token: $env: GITHUB_TOKEN diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts index 7a7973e981..065ebaefa5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts @@ -19,6 +19,11 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); import { AzurePreparer } from './azure'; import { @@ -73,7 +78,7 @@ describe('AzurePreparer', () => { it('calls the clone command with the correct arguments for a repository', async () => { const preparer = new AzurePreparer(ConfigReader.fromConfigs([])); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', @@ -99,7 +104,7 @@ describe('AzurePreparer', () => { }, ]), ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', @@ -117,7 +122,7 @@ describe('AzurePreparer', () => { it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { const preparer = new AzurePreparer(ConfigReader.fromConfigs([])); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', @@ -129,10 +134,12 @@ describe('AzurePreparer', () => { it('return the temp directory with the path to the folder if it is specified', async () => { const preparer = new AzurePreparer(ConfigReader.fromConfigs([])); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + workingDirectory: '/workDir', + }); - expect(response.split('\\').join('/')).toMatch( - /\/template\/test\/1\/2\/3$/, + expect(response).toBe( + `/workDir/graphql-starter-static/template/test/1/2/3`, ); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts index dc74ec0f75..aaf8508c8e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts @@ -15,7 +15,6 @@ */ import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; @@ -32,8 +31,12 @@ export class AzurePreparer implements PreparerBase { config.getOptionalString('scaffolder.azure.api.token') ?? ''; } - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts: { workingDirectory: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const { workingDirectory } = opts; if (!['azure/api', 'url'].includes(protocol)) { throw new InputError( @@ -42,18 +45,14 @@ export class AzurePreparer implements PreparerBase { } const templateId = template.metadata.name; - const url = new URL(location); // Need to extract filepath from search parameter const parsedGitLocation = GitUriParser(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); - const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const templateDirectory = path.join( - `${path - .dirname(url.searchParams.get('path') || '') - .replace(/^\/+/g, '')}`, // Strip leading slash + `${path.dirname(parsedGitLocation.filepath)}`, template.spec.path ?? '.', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts index b9472fbc8d..d226736efe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import os from 'os'; import fs from 'fs-extra'; import YAML from 'yaml'; import { FilePreparer } from './file'; @@ -42,7 +43,9 @@ const setupTest = async (fixturePath: string) => { }; const filePreparer = new FilePreparer(); - const resultDir = await filePreparer.prepare(template); + const resultDir = await filePreparer.prepare(template, { + workingDirectory: os.tmpdir(), + }); return { filePreparer, template, resultDir }; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts index b5ac846cf2..828238ae08 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts @@ -15,15 +15,18 @@ */ import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; import { PreparerBase } from './types'; export class FilePreparer implements PreparerBase { - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts: { workingDirectory: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const { workingDirectory } = opts; if (protocol !== 'file') { throw new InputError( @@ -33,7 +36,7 @@ export class FilePreparer implements PreparerBase { const templateId = template.metadata.name; const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const parentDirectory = path.resolve( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index de57043a50..29655f7ee9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -19,6 +19,11 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); import { GithubPreparer } from './github'; import { @@ -71,7 +76,7 @@ describe('GitHubPreparer', () => { }); it('calls the clone command with the correct arguments for a repository', async () => { const preparer = new GithubPreparer(); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://github.com/benjdlambert/backstage-graphql-template', @@ -84,7 +89,7 @@ describe('GitHubPreparer', () => { it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { const preparer = new GithubPreparer(); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://github.com/benjdlambert/backstage-graphql-template', @@ -97,15 +102,18 @@ describe('GitHubPreparer', () => { it('return the temp directory with the path to the folder if it is specified', async () => { const preparer = new GithubPreparer(); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + workingDirectory: '/workDir', + }); - expect(response.split('\\').join('/')).toMatch( - /\/template\/test\/1\/2\/3$/, + expect(response).toBe( + `/workDir/graphql-starter-static/template/test/1/2/3`, ); }); + it('calls the clone command with the token when provided', async () => { const preparer = new GithubPreparer({ token: 'abc' }); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://github.com/benjdlambert/backstage-graphql-template', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index 0f064d22fa..863b760d03 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -15,7 +15,6 @@ */ import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; @@ -30,8 +29,12 @@ export class GithubPreparer implements PreparerBase { this.token = params.token; } - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts: { workingDirectory: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const { workingDirectory } = opts; const { token } = this; if (!['github', 'url'].includes(protocol)) { @@ -44,7 +47,7 @@ export class GithubPreparer implements PreparerBase { const parsedGitLocation = GitUriParser(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const templateDirectory = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts index b25886b027..8df853ac06 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts @@ -18,6 +18,11 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); import { GitlabPreparer } from './gitlab'; import { @@ -74,7 +79,7 @@ describe('GitLabPreparer', () => { it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => { const preparer = new GitlabPreparer(ConfigReader.fromConfigs([])); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://gitlab.com/benjdlambert/backstage-graphql-template', @@ -101,7 +106,7 @@ describe('GitLabPreparer', () => { ]), ); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://gitlab.com/benjdlambert/backstage-graphql-template', @@ -120,7 +125,7 @@ describe('GitLabPreparer', () => { const preparer = new GitlabPreparer(ConfigReader.fromConfigs([])); mockEntity = mockEntityWithProtocol(protocol); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { workingDirectory: '/workDir' }); expect(mocks.Clone.clone).toHaveBeenNthCalledWith( 1, 'https://gitlab.com/benjdlambert/backstage-graphql-template', @@ -133,10 +138,12 @@ describe('GitLabPreparer', () => { const preparer = new GitlabPreparer(ConfigReader.fromConfigs([])); mockEntity = mockEntityWithProtocol(protocol); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + workingDirectory: '/workDir', + }); - expect(response.split('\\').join('/')).toMatch( - /\/template\/test\/1\/2\/3$/, + expect(response).toBe( + `/workDir/graphql-starter-static/template/test/1/2/3`, ); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts index 8ccde0b626..5be72f85b5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts @@ -15,7 +15,6 @@ */ import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; @@ -33,8 +32,12 @@ export class GitlabPreparer implements PreparerBase { ''; } - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts: { workingDirectory: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const { workingDirectory } = opts; if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) { throw new InputError( @@ -45,9 +48,8 @@ export class GitlabPreparer implements PreparerBase { const parsedGitLocation = GitUriParser(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); - const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const templateDirectory = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts index c9dd14c0a3..940d2b4178 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts @@ -25,7 +25,7 @@ export type PreparerBase = { */ prepare( template: TemplateEntityV1alpha1, - opts: { logger: Logger }, + opts: { logger: Logger; workingDirectory: string }, ): Promise; }; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 35d6b50ece..9e44fb8cbe 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -15,6 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { createRouter } from './router'; @@ -32,6 +33,7 @@ describe('createRouter', () => { preparers: new Preparers(), templaters: new Templaters(), publishers: new Publishers(), + config: ConfigReader.fromConfigs([]), dockerClient: new Docker(), }); app = express().use(router); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f61d2566b9..60c17a3e80 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -15,7 +15,9 @@ */ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { JsonValue } from '@backstage/config'; +import { Config, JsonValue } from '@backstage/config'; +import os from 'os'; +import fs from 'fs-extra'; import Docker from 'dockerode'; import express from 'express'; import Router from 'express-promise-router'; @@ -36,6 +38,7 @@ export interface RouterOptions { publishers: PublisherBuilder; logger: Logger; + config: Config; dockerClient: Docker; } @@ -50,12 +53,31 @@ export async function createRouter( templaters, publishers, logger: parentLogger, + config, dockerClient, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); const jobProcessor = new JobProcessor(); + const workingDirectory = + config.getOptionalString('scaffolder.workingDirectory') ?? os.tmpdir(); + try { + // Check if working directory exists and is writable + await fs.promises.access( + workingDirectory, + fs.constants.F_OK | fs.constants.W_OK, + ); + logger.info(`using working directory: ${workingDirectory}`); + } catch (err) { + logger.error( + `working directory ${workingDirectory} ${ + err.code === 'ENOENT' ? 'does not exist' : 'is not writable' + }`, + ); + throw err; + } + router .get('/v1/job/:jobId', ({ params }, res) => { const job = jobProcessor.get(params.jobId); @@ -104,6 +126,7 @@ export async function createRouter( const preparer = preparers.get(ctx.entity); const skeletonDir = await preparer.prepare(ctx.entity, { logger: ctx.logger, + workingDirectory, }); return { skeletonDir }; },