diff --git a/.changeset/spicy-rockets-ring.md b/.changeset/spicy-rockets-ring.md new file mode 100644 index 0000000000..b39b433c1b --- /dev/null +++ b/.changeset/spicy-rockets-ring.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Added support for configuring the working directory of the Scaffolder: + +```yaml +backend: + workingDirectory: /some-dir # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir +``` diff --git a/app-config.yaml b/app-config.yaml index 6e3a68a229..4746e822da 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -16,6 +16,7 @@ backend: credentials: true csp: connect-src: ["'self'", 'http:', 'https:'] + # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir # See README.md in the proxy-backend plugin for information on the configuration format proxy: diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index c84928ea68..6eacd143c1 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -45,6 +45,7 @@ export default async function createPlugin({ templaters, publishers, logger, + config, dockerClient, }); } diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 1f265d7e7d..d643c72e0c 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -38,6 +38,7 @@ backend: #ca: # if you have a CA file and want to verify it you can uncomment this section # $file: /ca/server.crt {{/if}} + # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir integrations: github: diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index 240309fbd1..8de15920e9 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -102,6 +102,7 @@ export default async function createPlugin({ templaters, publishers, logger, + config, dockerClient, }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts index 7a7973e981..97f01e6b7d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts @@ -19,6 +19,11 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); import { AzurePreparer } from './azure'; import { @@ -135,4 +140,16 @@ describe('AzurePreparer', () => { /\/template\/test\/1\/2\/3$/, ); }); + + it('return the working directory with the path to the folder if it is specified', async () => { + const preparer = new AzurePreparer(ConfigReader.fromConfigs([])); + mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { + workingDirectory: '/workDir', + }); + + expect(response).toBe( + '/workDir/graphql-starter-static/template/test/1/2/3', + ); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts index dc74ec0f75..bdcc7e07de 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import os from 'os'; import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; @@ -32,8 +32,12 @@ export class AzurePreparer implements PreparerBase { config.getOptionalString('scaffolder.azure.api.token') ?? ''; } - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts?: { workingDirectory?: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); if (!['azure/api', 'url'].includes(protocol)) { throw new InputError( @@ -42,18 +46,14 @@ export class AzurePreparer implements PreparerBase { } const templateId = template.metadata.name; - const url = new URL(location); // Need to extract filepath from search parameter const parsedGitLocation = GitUriParser(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); - const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const templateDirectory = path.join( - `${path - .dirname(url.searchParams.get('path') || '') - .replace(/^\/+/g, '')}`, // Strip leading slash + `${path.dirname(parsedGitLocation.filepath)}`, template.spec.path ?? '.', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts index b9472fbc8d..d226736efe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import os from 'os'; import fs from 'fs-extra'; import YAML from 'yaml'; import { FilePreparer } from './file'; @@ -42,7 +43,9 @@ const setupTest = async (fixturePath: string) => { }; const filePreparer = new FilePreparer(); - const resultDir = await filePreparer.prepare(template); + const resultDir = await filePreparer.prepare(template, { + workingDirectory: os.tmpdir(), + }); return { filePreparer, template, resultDir }; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts index b5ac846cf2..683d0c8cfb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts @@ -13,17 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import os from 'os'; import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; import { PreparerBase } from './types'; export class FilePreparer implements PreparerBase { - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts?: { workingDirectory?: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); if (protocol !== 'file') { throw new InputError( @@ -33,7 +37,7 @@ export class FilePreparer implements PreparerBase { const templateId = template.metadata.name; const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const parentDirectory = path.resolve( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index de57043a50..f251a39d28 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -19,6 +19,11 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); import { GithubPreparer } from './github'; import { @@ -94,6 +99,7 @@ describe('GitHubPreparer', () => { }, ); }); + it('return the temp directory with the path to the folder if it is specified', async () => { const preparer = new GithubPreparer(); mockEntity.spec.path = './template/test/1/2/3'; @@ -103,6 +109,19 @@ describe('GitHubPreparer', () => { /\/template\/test\/1\/2\/3$/, ); }); + + it('return the working directory with the path to the folder if it is specified', async () => { + const preparer = new GithubPreparer(); + mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { + workingDirectory: '/workDir', + }); + + expect(response).toBe( + '/workDir/graphql-starter-static/template/test/1/2/3', + ); + }); + it('calls the clone command with the token when provided', async () => { const preparer = new GithubPreparer({ token: 'abc' }); await preparer.prepare(mockEntity); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index 0f064d22fa..04b0bbcce1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import os from 'os'; import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; @@ -30,8 +30,12 @@ export class GithubPreparer implements PreparerBase { this.token = params.token; } - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts?: { workingDirectory?: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); const { token } = this; if (!['github', 'url'].includes(protocol)) { @@ -44,7 +48,7 @@ export class GithubPreparer implements PreparerBase { const parsedGitLocation = GitUriParser(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const templateDirectory = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts index b25886b027..a229c4f394 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts @@ -18,6 +18,11 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); import { GitlabPreparer } from './gitlab'; import { @@ -139,5 +144,17 @@ describe('GitLabPreparer', () => { /\/template\/test\/1\/2\/3$/, ); }); + + it('return the working directory with the path to the folder if it is specified', async () => { + const preparer = new GitlabPreparer(ConfigReader.fromConfigs([])); + mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { + workingDirectory: '/workDir', + }); + + expect(response).toBe( + '/workDir/graphql-starter-static/template/test/1/2/3', + ); + }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts index 8ccde0b626..4553137c26 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import os from 'os'; import fs from 'fs-extra'; import path from 'path'; -import os from 'os'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { parseLocationAnnotation } from '../helpers'; import { InputError } from '@backstage/backend-common'; @@ -33,8 +33,12 @@ export class GitlabPreparer implements PreparerBase { ''; } - async prepare(template: TemplateEntityV1alpha1): Promise { + async prepare( + template: TemplateEntityV1alpha1, + opts?: { workingDirectory?: string }, + ): Promise { const { protocol, location } = parseLocationAnnotation(template); + const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) { throw new InputError( @@ -45,9 +49,8 @@ export class GitlabPreparer implements PreparerBase { const parsedGitLocation = GitUriParser(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); - const tempDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), templateId), + path.join(workingDirectory, templateId), ); const templateDirectory = path.join( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts index c9dd14c0a3..c17edc24a4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts @@ -25,7 +25,7 @@ export type PreparerBase = { */ prepare( template: TemplateEntityV1alpha1, - opts: { logger: Logger }, + opts?: { logger: Logger; workingDirectory?: string }, ): Promise; }; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 35d6b50ece..fae7ee8beb 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -14,7 +14,19 @@ * limitations under the License. */ +const mockAccess = jest.fn(); +jest.doMock('fs-extra', () => ({ + promises: { + access: mockAccess, + }, + constants: { + F_OK: 0, + W_OK: 1, + }, +})); + import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import { createRouter } from './router'; @@ -23,6 +35,106 @@ import Docker from 'dockerode'; jest.mock('dockerode'); +describe('createRouter - working directory', () => { + const mockPrepare = jest.fn(); + const mockPreparers = new Preparers(); + + beforeAll(() => { + const mockPreparer = { + prepare: mockPrepare, + }; + mockPreparers.register('azure/api', mockPreparer); + }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + const workDirConfig = (path: string) => ({ + context: '', + data: { + backend: { + workingDirectory: path, + }, + }, + }); + + const template = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Template', + metadata: { + annotations: { + 'backstage.io/managed-by-location': 'azure/api:dev.azure.com', + }, + }, + spec: { + owner: 'template@backstage.io', + path: '.', + schema: {}, + }, + }; + + it('should throw an error when working directory does not exist or is not writable', async () => { + mockAccess.mockImplementation(() => { + throw new Error('access error'); + }); + + await expect( + createRouter({ + logger: getVoidLogger(), + preparers: new Preparers(), + templaters: new Templaters(), + publishers: new Publishers(), + config: ConfigReader.fromConfigs([workDirConfig('/path')]), + dockerClient: new Docker(), + }), + ).rejects.toThrow('access error'); + }); + + it('should use the working directory when configured', async () => { + const router = await createRouter({ + logger: getVoidLogger(), + preparers: mockPreparers, + templaters: new Templaters(), + publishers: new Publishers(), + config: ConfigReader.fromConfigs([workDirConfig('/path')]), + dockerClient: new Docker(), + }); + + const app = express().use(router); + await request(app).post('/v1/jobs').send({ + template, + values: {}, + }); + + expect(mockPrepare).toBeCalledWith(expect.anything(), { + logger: expect.anything(), + workingDirectory: '/path', + }); + }); + + it('should not pass along anything when no working directory is configured', async () => { + const router = await createRouter({ + logger: getVoidLogger(), + preparers: mockPreparers, + templaters: new Templaters(), + publishers: new Publishers(), + config: ConfigReader.fromConfigs([]), + dockerClient: new Docker(), + }); + + const app = express().use(router); + await request(app).post('/v1/jobs').send({ + template, + values: {}, + }); + + expect(mockPrepare).toBeCalledWith(expect.anything(), { + logger: expect.anything(), + }); + }); +}); + describe('createRouter', () => { let app: express.Express; @@ -32,6 +144,7 @@ describe('createRouter', () => { preparers: new Preparers(), templaters: new Templaters(), publishers: new Publishers(), + config: ConfigReader.fromConfigs([]), dockerClient: new Docker(), }); app = express().use(router); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f61d2566b9..fb40489622 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -15,7 +15,8 @@ */ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { JsonValue } from '@backstage/config'; +import { Config, JsonValue } from '@backstage/config'; +import fs from 'fs-extra'; import Docker from 'dockerode'; import express from 'express'; import Router from 'express-promise-router'; @@ -36,6 +37,7 @@ export interface RouterOptions { publishers: PublisherBuilder; logger: Logger; + config: Config; dockerClient: Docker; } @@ -50,12 +52,33 @@ export async function createRouter( templaters, publishers, logger: parentLogger, + config, dockerClient, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); const jobProcessor = new JobProcessor(); + let workingDirectory: string; + if (config.has('backend.workingDirectory')) { + workingDirectory = config.getString('backend.workingDirectory'); + try { + // Check if working directory exists and is writable + await fs.promises.access( + workingDirectory, + fs.constants.F_OK | fs.constants.W_OK, + ); + logger.info(`using working directory: ${workingDirectory}`); + } catch (err) { + logger.error( + `working directory ${workingDirectory} ${ + err.code === 'ENOENT' ? 'does not exist' : 'is not writable' + }`, + ); + throw err; + } + } + router .get('/v1/job/:jobId', ({ params }, res) => { const job = jobProcessor.get(params.jobId); @@ -104,6 +127,7 @@ export async function createRouter( const preparer = preparers.get(ctx.entity); const skeletonDir = await preparer.prepare(ctx.entity, { logger: ctx.logger, + workingDirectory, }); return { skeletonDir }; },