Merge pull request #3062 from mfrinnstrom/scaffolder-working-directory

[Scaffolder] Add workdir config support to scaffolder

.changeset/spicy-rockets-ring.md

@@ -0,0 +1,10 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Added support for configuring the working directory of the Scaffolder:
+
+```yaml
+backend:
+  workingDirectory: /some-dir # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir
+```

app-config.yaml

@@ -16,6 +16,7 @@ backend:
     credentials: true
   csp:
     connect-src: ["'self'", 'http:', 'https:']
+  # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir
 
 # See README.md in the proxy-backend plugin for information on the configuration format
 proxy:

packages/backend/src/plugins/scaffolder.ts

@@ -45,6 +45,7 @@ export default async function createPlugin({
     templaters,
     publishers,
     logger,
+    config,
     dockerClient,
   });
 }

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -38,6 +38,7 @@ backend:
         #ca: # if you have a CA file and want to verify it you can uncomment this section
         #  $file: <file-path>/ca/server.crt
   {{/if}}
+  # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir
 
 integrations:
   github:

packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts

@@ -102,6 +102,7 @@ export default async function createPlugin({
     templaters,
     publishers,
     logger,
+    config,
     dockerClient,
   });
 }

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts

@@ -19,6 +19,11 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
 
 import { AzurePreparer } from './azure';
 import {
@@ -135,4 +140,16 @@ describe('AzurePreparer', () => {
       /\/template\/test\/1\/2\/3$/,
     );
   });
+
+  it('return the working directory with the path to the folder if it is specified', async () => {
+    const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
+    mockEntity.spec.path = './template/test/1/2/3';
+    const response = await preparer.prepare(mockEntity, {
+      workingDirectory: '/workDir',
+    });
+
+    expect(response).toBe(
+      '/workDir/graphql-starter-static/template/test/1/2/3',
+    );
+  });
 });

plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import os from 'os';
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
@@ -32,8 +32,12 @@ export class AzurePreparer implements PreparerBase {
       config.getOptionalString('scaffolder.azure.api.token') ?? '';
   }
 
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts?: { workingDirectory?: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
 
     if (!['azure/api', 'url'].includes(protocol)) {
       throw new InputError(
@@ -42,18 +46,14 @@ export class AzurePreparer implements PreparerBase {
     }
     const templateId = template.metadata.name;
 
-    const url = new URL(location); // Need to extract filepath from search parameter
     const parsedGitLocation = GitUriParser(location);
     const repositoryCheckoutUrl = parsedGitLocation.toString('https');
-
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const templateDirectory = path.join(
-      `${path
-        .dirname(url.searchParams.get('path') || '')
-        .replace(/^\/+/g, '')}`, // Strip leading slash
+      `${path.dirname(parsedGitLocation.filepath)}`,
       template.spec.path ?? '.',
     );
 

plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import os from 'os';
 import fs from 'fs-extra';
 import YAML from 'yaml';
 import { FilePreparer } from './file';
@@ -42,7 +43,9 @@ const setupTest = async (fixturePath: string) => {
   };
 
   const filePreparer = new FilePreparer();
-  const resultDir = await filePreparer.prepare(template);
+  const resultDir = await filePreparer.prepare(template, {
+    workingDirectory: os.tmpdir(),
+  });
 
   return { filePreparer, template, resultDir };
 };

plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts

@@ -13,17 +13,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import os from 'os';
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
 import { PreparerBase } from './types';
 
 export class FilePreparer implements PreparerBase {
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts?: { workingDirectory?: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
 
     if (protocol !== 'file') {
       throw new InputError(
@@ -33,7 +37,7 @@ export class FilePreparer implements PreparerBase {
     const templateId = template.metadata.name;
 
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const parentDirectory = path.resolve(

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -19,6 +19,11 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
 
 import { GithubPreparer } from './github';
 import {
@@ -94,6 +99,7 @@ describe('GitHubPreparer', () => {
       },
     );
   });
+
   it('return the temp directory with the path to the folder if it is specified', async () => {
     const preparer = new GithubPreparer();
     mockEntity.spec.path = './template/test/1/2/3';
@@ -103,6 +109,19 @@ describe('GitHubPreparer', () => {
       /\/template\/test\/1\/2\/3$/,
     );
   });
+
+  it('return the working directory with the path to the folder if it is specified', async () => {
+    const preparer = new GithubPreparer();
+    mockEntity.spec.path = './template/test/1/2/3';
+    const response = await preparer.prepare(mockEntity, {
+      workingDirectory: '/workDir',
+    });
+
+    expect(response).toBe(
+      '/workDir/graphql-starter-static/template/test/1/2/3',
+    );
+  });
+
   it('calls the clone command with the token when provided', async () => {
     const preparer = new GithubPreparer({ token: 'abc' });
     await preparer.prepare(mockEntity);

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import os from 'os';
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
@@ -30,8 +30,12 @@ export class GithubPreparer implements PreparerBase {
     this.token = params.token;
   }
 
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts?: { workingDirectory?: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
     const { token } = this;
 
     if (!['github', 'url'].includes(protocol)) {
@@ -44,7 +48,7 @@ export class GithubPreparer implements PreparerBase {
     const parsedGitLocation = GitUriParser(location);
     const repositoryCheckoutUrl = parsedGitLocation.toString('https');
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const templateDirectory = path.join(

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts

@@ -18,6 +18,11 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
 
 import { GitlabPreparer } from './gitlab';
 import {
@@ -139,5 +144,17 @@ describe('GitLabPreparer', () => {
         /\/template\/test\/1\/2\/3$/,
       );
     });
+
+    it('return the working directory with the path to the folder if it is specified', async () => {
+      const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
+      mockEntity.spec.path = './template/test/1/2/3';
+      const response = await preparer.prepare(mockEntity, {
+        workingDirectory: '/workDir',
+      });
+
+      expect(response).toBe(
+        '/workDir/graphql-starter-static/template/test/1/2/3',
+      );
+    });
   });
 });

plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import os from 'os';
 import fs from 'fs-extra';
 import path from 'path';
-import os from 'os';
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
 import { parseLocationAnnotation } from '../helpers';
 import { InputError } from '@backstage/backend-common';
@@ -33,8 +33,12 @@ export class GitlabPreparer implements PreparerBase {
       '';
   }
 
-  async prepare(template: TemplateEntityV1alpha1): Promise<string> {
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts?: { workingDirectory?: string },
+  ): Promise<string> {
     const { protocol, location } = parseLocationAnnotation(template);
+    const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
 
     if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) {
       throw new InputError(
@@ -45,9 +49,8 @@ export class GitlabPreparer implements PreparerBase {
 
     const parsedGitLocation = GitUriParser(location);
     const repositoryCheckoutUrl = parsedGitLocation.toString('https');
-
     const tempDir = await fs.promises.mkdtemp(
-      path.join(os.tmpdir(), templateId),
+      path.join(workingDirectory, templateId),
     );
 
     const templateDirectory = path.join(

plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts

@@ -25,7 +25,7 @@ export type PreparerBase = {
    */
   prepare(
     template: TemplateEntityV1alpha1,
-    opts: { logger: Logger },
+    opts?: { logger: Logger; workingDirectory?: string },
   ): Promise<string>;
 };
 

plugins/scaffolder-backend/src/service/router.test.ts

@@ -14,7 +14,19 @@
  * limitations under the License.
  */
 
+const mockAccess = jest.fn();
+jest.doMock('fs-extra', () => ({
+  promises: {
+    access: mockAccess,
+  },
+  constants: {
+    F_OK: 0,
+    W_OK: 1,
+  },
+}));
+
 import { getVoidLogger } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
 import express from 'express';
 import request from 'supertest';
 import { createRouter } from './router';
@@ -23,6 +35,106 @@ import Docker from 'dockerode';
 
 jest.mock('dockerode');
 
+describe('createRouter - working directory', () => {
+  const mockPrepare = jest.fn();
+  const mockPreparers = new Preparers();
+
+  beforeAll(() => {
+    const mockPreparer = {
+      prepare: mockPrepare,
+    };
+    mockPreparers.register('azure/api', mockPreparer);
+  });
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  const workDirConfig = (path: string) => ({
+    context: '',
+    data: {
+      backend: {
+        workingDirectory: path,
+      },
+    },
+  });
+
+  const template = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Template',
+    metadata: {
+      annotations: {
+        'backstage.io/managed-by-location': 'azure/api:dev.azure.com',
+      },
+    },
+    spec: {
+      owner: 'template@backstage.io',
+      path: '.',
+      schema: {},
+    },
+  };
+
+  it('should throw an error when working directory does not exist or is not writable', async () => {
+    mockAccess.mockImplementation(() => {
+      throw new Error('access error');
+    });
+
+    await expect(
+      createRouter({
+        logger: getVoidLogger(),
+        preparers: new Preparers(),
+        templaters: new Templaters(),
+        publishers: new Publishers(),
+        config: ConfigReader.fromConfigs([workDirConfig('/path')]),
+        dockerClient: new Docker(),
+      }),
+    ).rejects.toThrow('access error');
+  });
+
+  it('should use the working directory when configured', async () => {
+    const router = await createRouter({
+      logger: getVoidLogger(),
+      preparers: mockPreparers,
+      templaters: new Templaters(),
+      publishers: new Publishers(),
+      config: ConfigReader.fromConfigs([workDirConfig('/path')]),
+      dockerClient: new Docker(),
+    });
+
+    const app = express().use(router);
+    await request(app).post('/v1/jobs').send({
+      template,
+      values: {},
+    });
+
+    expect(mockPrepare).toBeCalledWith(expect.anything(), {
+      logger: expect.anything(),
+      workingDirectory: '/path',
+    });
+  });
+
+  it('should not pass along anything when no working directory is configured', async () => {
+    const router = await createRouter({
+      logger: getVoidLogger(),
+      preparers: mockPreparers,
+      templaters: new Templaters(),
+      publishers: new Publishers(),
+      config: ConfigReader.fromConfigs([]),
+      dockerClient: new Docker(),
+    });
+
+    const app = express().use(router);
+    await request(app).post('/v1/jobs').send({
+      template,
+      values: {},
+    });
+
+    expect(mockPrepare).toBeCalledWith(expect.anything(), {
+      logger: expect.anything(),
+    });
+  });
+});
+
 describe('createRouter', () => {
   let app: express.Express;
 
@@ -32,6 +144,7 @@ describe('createRouter', () => {
       preparers: new Preparers(),
       templaters: new Templaters(),
       publishers: new Publishers(),
+      config: ConfigReader.fromConfigs([]),
       dockerClient: new Docker(),
     });
     app = express().use(router);

plugins/scaffolder-backend/src/service/router.ts

@@ -15,7 +15,8 @@
  */
 
 import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
-import { JsonValue } from '@backstage/config';
+import { Config, JsonValue } from '@backstage/config';
+import fs from 'fs-extra';
 import Docker from 'dockerode';
 import express from 'express';
 import Router from 'express-promise-router';
@@ -36,6 +37,7 @@ export interface RouterOptions {
   publishers: PublisherBuilder;
 
   logger: Logger;
+  config: Config;
   dockerClient: Docker;
 }
 
@@ -50,12 +52,33 @@ export async function createRouter(
     templaters,
     publishers,
     logger: parentLogger,
+    config,
     dockerClient,
   } = options;
 
   const logger = parentLogger.child({ plugin: 'scaffolder' });
   const jobProcessor = new JobProcessor();
 
+  let workingDirectory: string;
+  if (config.has('backend.workingDirectory')) {
+    workingDirectory = config.getString('backend.workingDirectory');
+    try {
+      // Check if working directory exists and is writable
+      await fs.promises.access(
+        workingDirectory,
+        fs.constants.F_OK | fs.constants.W_OK,
+      );
+      logger.info(`using working directory: ${workingDirectory}`);
+    } catch (err) {
+      logger.error(
+        `working directory ${workingDirectory} ${
+          err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
+        }`,
+      );
+      throw err;
+    }
+  }
+
   router
     .get('/v1/job/:jobId', ({ params }, res) => {
       const job = jobProcessor.get(params.jobId);
@@ -104,6 +127,7 @@ export async function createRouter(
               const preparer = preparers.get(ctx.entity);
               const skeletonDir = await preparer.prepare(ctx.entity, {
                 logger: ctx.logger,
+                workingDirectory,
               });
               return { skeletonDir };
             },