Add workdir config support to scaffolder

app-config.yaml

@@ -155,6 +155,7 @@ catalog:
       target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/acme-corp.yaml
 
 scaffolder:
+  # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir
   github:
     token:
       $env: GITHUB_TOKEN

packages/backend/src/plugins/scaffolder.ts

@@ -45,6 +45,7 @@ export default async function createPlugin({
     templaters,
     publishers,
     logger,
+    config,
     dockerClient,
   });
 }

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -69,6 +69,7 @@ auth:
   providers: {}
 
 scaffolder:
+  # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir
   github:
     token:
       $env: GITHUB_TOKEN

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts

@@ -19,6 +19,11 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
 
 import { AzurePreparer } from './azure';
 import {
@@ -73,7 +78,7 @@ describe('AzurePreparer', () => {
 
   it('calls the clone command with the correct arguments for a repository', async () => {
     const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
-    await preparer.prepare(mockEntity);
+    await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
     expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
       1,
       'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
@@ -99,7 +104,7 @@ describe('AzurePreparer', () => {
         },
       ]),
     );
-    await preparer.prepare(mockEntity);
+    await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
     expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
       1,
       'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
@@ -117,7 +122,7 @@ describe('AzurePreparer', () => {
   it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
     const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
     delete mockEntity.spec.path;
-    await preparer.prepare(mockEntity);
+    await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
     expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
       1,
       'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
@@ -129,10 +134,12 @@ describe('AzurePreparer', () => {
   it('return the temp directory with the path to the folder if it is specified', async () => {
     const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
     mockEntity.spec.path = './template/test/1/2/3';
-    const response = await preparer.prepare(mockEntity);
+    const response = await preparer.prepare(mockEntity, {
+      workingDirectory: '/workDir',
+    });
 
-    expect(response.split('\\').join('/')).toMatch(
-      /\/template\/test\/1\/2\/3$/,
+    expect(response).toBe(
+      `/workDir/graphql-starter-static/template/test/1/2/3`,
     );
   });
 });

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts

@@ -15,7 +15,6 @@
  */
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
@@ -32,8 +31,12 @@ export class AzurePreparer implements PreparerBase {
       config.getOptionalString('scaffolder.azure.api.token') ?? '';
   }
 
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts: { workingDirectory: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const { workingDirectory } = opts;
 
     if (!['azure/api', 'url'].includes(protocol)) {
       throw new InputError(
@@ -42,18 +45,14 @@ export class AzurePreparer implements PreparerBase {
     }
     const templateId = template.metadata.name;
 
-    const url = new URL(location); // Need to extract filepath from search parameter
     const parsedGitLocation = GitUriParser(location);
     const repositoryCheckoutUrl = parsedGitLocation.toString('https');
-
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const templateDirectory = path.join(
-      `${path
-        .dirname(url.searchParams.get('path') || '')
-        .replace(/^\/+/g, '')}`, // Strip leading slash
+      `${path.dirname(parsedGitLocation.filepath)}`,
       template.spec.path ?? '.',
     );
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import os from 'os';
 import fs from 'fs-extra';
 import YAML from 'yaml';
 import { FilePreparer } from './file';
@@ -42,7 +43,9 @@ const setupTest = async (fixturePath: string) => {
   };
 
   const filePreparer = new FilePreparer();
-  const resultDir = await filePreparer.prepare(template);
+  const resultDir = await filePreparer.prepare(template, {
+    workingDirectory: os.tmpdir(),
+  });
 
   return { filePreparer, template, resultDir };
 };

plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts

@@ -15,15 +15,18 @@
  */
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
 import { PreparerBase } from './types';
 
 export class FilePreparer implements PreparerBase {
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts: { workingDirectory: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const { workingDirectory } = opts;
 
     if (protocol !== 'file') {
       throw new InputError(
@@ -33,7 +36,7 @@ export class FilePreparer implements PreparerBase {
     const templateId = template.metadata.name;
 
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const parentDirectory = path.resolve(

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -19,6 +19,11 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
 
 import { GithubPreparer } from './github';
 import {
@@ -71,7 +76,7 @@ describe('GitHubPreparer', () => {
   });
   it('calls the clone command with the correct arguments for a repository', async () => {
     const preparer = new GithubPreparer();
-    await preparer.prepare(mockEntity);
+    await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
     expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',
@@ -84,7 +89,7 @@ describe('GitHubPreparer', () => {
   it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
     const preparer = new GithubPreparer();
     delete mockEntity.spec.path;
-    await preparer.prepare(mockEntity);
+    await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
     expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',
@@ -97,15 +102,18 @@ describe('GitHubPreparer', () => {
   it('return the temp directory with the path to the folder if it is specified', async () => {
     const preparer = new GithubPreparer();
     mockEntity.spec.path = './template/test/1/2/3';
-    const response = await preparer.prepare(mockEntity);
+    const response = await preparer.prepare(mockEntity, {
+      workingDirectory: '/workDir',
+    });
 
-    expect(response.split('\\').join('/')).toMatch(
-      /\/template\/test\/1\/2\/3$/,
+    expect(response).toBe(
+      `/workDir/graphql-starter-static/template/test/1/2/3`,
     );
   });
+
   it('calls the clone command with the token when provided', async () => {
     const preparer = new GithubPreparer({ token: 'abc' });
-    await preparer.prepare(mockEntity);
+    await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
     expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
       1,
       'https://github.com/benjdlambert/backstage-graphql-template',

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -15,7 +15,6 @@
  */
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
@@ -30,8 +29,12 @@ export class GithubPreparer implements PreparerBase {
     this.token = params.token;
   }
 
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts: { workingDirectory: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const { workingDirectory } = opts;
     const { token } = this;
 
     if (!['github', 'url'].includes(protocol)) {
@@ -44,7 +47,7 @@ export class GithubPreparer implements PreparerBase {
     const parsedGitLocation = GitUriParser(location);
     const repositoryCheckoutUrl = parsedGitLocation.toString('https');
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const templateDirectory = path.join(

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts

@@ -18,6 +18,11 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
 
 import { GitlabPreparer } from './gitlab';
 import {
@@ -74,7 +79,7 @@ describe('GitLabPreparer', () => {
     it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => {
       const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
       mockEntity = mockEntityWithProtocol(protocol);
-      await preparer.prepare(mockEntity);
+      await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
       expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
         1,
         'https://gitlab.com/benjdlambert/backstage-graphql-template',
@@ -101,7 +106,7 @@ describe('GitLabPreparer', () => {
         ]),
       );
       mockEntity = mockEntityWithProtocol(protocol);
-      await preparer.prepare(mockEntity);
+      await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
       expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
         1,
         'https://gitlab.com/benjdlambert/backstage-graphql-template',
@@ -120,7 +125,7 @@ describe('GitLabPreparer', () => {
       const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
       mockEntity = mockEntityWithProtocol(protocol);
       delete mockEntity.spec.path;
-      await preparer.prepare(mockEntity);
+      await preparer.prepare(mockEntity, { workingDirectory: '/workDir' });
       expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
         1,
         'https://gitlab.com/benjdlambert/backstage-graphql-template',
@@ -133,10 +138,12 @@ describe('GitLabPreparer', () => {
       const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
       mockEntity = mockEntityWithProtocol(protocol);
       mockEntity.spec.path = './template/test/1/2/3';
-      const response = await preparer.prepare(mockEntity);
+      const response = await preparer.prepare(mockEntity, {
+        workingDirectory: '/workDir',
+      });
 
-      expect(response.split('\\').join('/')).toMatch(
-        /\/template\/test\/1\/2\/3$/,
+      expect(response).toBe(
+        `/workDir/graphql-starter-static/template/test/1/2/3`,
       );
     });
   });

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts

@@ -15,7 +15,6 @@
  */
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
@@ -33,8 +32,12 @@ export class GitlabPreparer implements PreparerBase {
       '';
   }
 
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts: { workingDirectory: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const { workingDirectory } = opts;
 
     if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) {
       throw new InputError(
@@ -45,9 +48,8 @@ export class GitlabPreparer implements PreparerBase {
 
     const parsedGitLocation = GitUriParser(location);
     const repositoryCheckoutUrl = parsedGitLocation.toString('https');
-
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const templateDirectory = path.join(

plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts

@@ -25,7 +25,7 @@ export type PreparerBase = {
    */
   prepare(
     template: TemplateEntityV1alpha1,
-    opts: { logger: Logger },
+    opts: { logger: Logger; workingDirectory: string },
   ): Promise<string>;
 };
 

plugins/scaffolder-backend/src/service/router.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
 import express from 'express';
 import request from 'supertest';
 import { createRouter } from './router';
@@ -32,6 +33,7 @@ describe('createRouter', () => {
       preparers: new Preparers(),
       templaters: new Templaters(),
       publishers: new Publishers(),
+      config: ConfigReader.fromConfigs([]),
       dockerClient: new Docker(),
     });
     app = express().use(router);

plugins/scaffolder-backend/src/service/router.ts

@@ -15,7 +15,9 @@
  */
 
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { JsonValue } from '@backstage/config';
+import { Config, JsonValue } from '@backstage/config';
+import os from 'os';
+import fs from 'fs-extra';
 import Docker from 'dockerode';
 import express from 'express';
 import Router from 'express-promise-router';
@@ -36,6 +38,7 @@ export interface RouterOptions {
   publishers: PublisherBuilder;
 
   logger: Logger;
+  config: Config;
   dockerClient: Docker;
 }
 
@@ -50,12 +53,31 @@ export async function createRouter(
     templaters,
     publishers,
     logger: parentLogger,
+    config,
     dockerClient,
   } = options;
 
   const logger = parentLogger.child({ plugin: 'scaffolder' });
   const jobProcessor = new JobProcessor();
 
+  const workingDirectory =
+    config.getOptionalString('scaffolder.workingDirectory') ?? os.tmpdir();
+  try {
+    // Check if working directory exists and is writable
+    await fs.promises.access(
+      workingDirectory,
+      fs.constants.F_OK | fs.constants.W_OK,
+    );
+    logger.info(`using working directory: ${workingDirectory}`);
+  } catch (err) {
+    logger.error(
+      `working directory ${workingDirectory} ${
+        err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
+      }`,
+    );
+    throw err;
+  }
+
   router
     .get('/v1/job/:jobId', ({ params }, res) => {
       const job = jobProcessor.get(params.jobId);
@@ -104,6 +126,7 @@ export async function createRouter(
               const preparer = preparers.get(ctx.entity);
               const skeletonDir = await preparer.prepare(ctx.entity, {
                 logger: ctx.logger,
+                workingDirectory,
               });
               return { skeletonDir };
             },