Merge pull request #21752 from backstage/blam/scaffolder-vendor-modules

scaffolder: Split out Actions into smaller modules

plugins/scaffolder-node/api-report.md

@@ -10,6 +10,7 @@ import { JsonValue } from '@backstage/types';
 import { Logger } from 'winston';
 import { Observable } from '@backstage/types';
 import { Schema } from 'jsonschema';
+import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmIntegrations } from '@backstage/integration';
 import { SpawnOptionsWithoutStdio } from 'child_process';
 import { TaskSpec } from '@backstage/plugin-scaffolder-common';
@@ -44,6 +45,29 @@ export type ActionContext<
   each?: JsonObject;
 };
 
+// @public (undocumented)
+export function commitAndPushRepo(input: {
+  dir: string;
+  auth:
+    | {
+        username: string;
+        password: string;
+      }
+    | {
+        token: string;
+      };
+  logger: Logger;
+  commitMessage: string;
+  gitAuthorInfo?: {
+    name?: string;
+    email?: string;
+  };
+  branch?: string;
+  remoteRef?: string;
+}): Promise<{
+  commitHash: string;
+}>;
+
 // @public
 export const createTemplateAction: <
   TInputParams extends JsonObject = JsonObject,
@@ -73,6 +97,12 @@ export const createTemplateAction: <
   >,
 ) => TemplateAction<TActionInput, TActionOutput>;
 
+// @public
+export function deserializeDirectoryContents(
+  targetPath: string,
+  files: SerializedFile[],
+): Promise<void>;
+
 // @public
 export function executeShellCommand(
   options: ExecuteShellCommandOptions,
@@ -104,6 +134,69 @@ export function fetchFile(options: {
   outputPath: string;
 }): Promise<void>;
 
+// @public (undocumented)
+export const getRepoSourceDirectory: (
+  workspacePath: string,
+  sourcePath: string | undefined,
+) => string;
+
+// @public (undocumented)
+export function initRepoAndPush(input: {
+  dir: string;
+  remoteUrl: string;
+  auth:
+    | {
+        username: string;
+        password: string;
+      }
+    | {
+        token: string;
+      };
+  logger: Logger;
+  defaultBranch?: string;
+  commitMessage?: string;
+  gitAuthorInfo?: {
+    name?: string;
+    email?: string;
+  };
+}): Promise<{
+  commitHash: string;
+}>;
+
+// @public (undocumented)
+export const parseRepoUrl: (
+  repoUrl: string,
+  integrations: ScmIntegrationRegistry,
+) => {
+  repo: string;
+  host: string;
+  owner?: string | undefined;
+  organization?: string | undefined;
+  workspace?: string | undefined;
+  project?: string | undefined;
+};
+
+// @public (undocumented)
+export interface SerializedFile {
+  // (undocumented)
+  content: Buffer;
+  // (undocumented)
+  executable?: boolean;
+  // (undocumented)
+  path: string;
+  // (undocumented)
+  symlink?: boolean;
+}
+
+// @public (undocumented)
+export function serializeDirectoryContents(
+  sourcePath: string,
+  options?: {
+    gitignore?: boolean;
+    globPatterns?: string[];
+  },
+): Promise<SerializedFile[]>;
+
 // @public
 export type SerializedTask = {
   id: string;

plugins/scaffolder-node/package.json

@@ -50,12 +50,15 @@
     "@backstage/plugin-scaffolder-common": "workspace:^",
     "@backstage/types": "workspace:^",
     "fs-extra": "10.1.0",
+    "globby": "^11.0.0",
     "jsonschema": "^1.2.6",
+    "p-limit": "^3.1.0",
     "winston": "^3.2.1",
     "zod": "^3.21.4",
     "zod-to-json-schema": "^3.20.4"
   },
   "devDependencies": {
+    "@backstage/backend-test-utils": "workspace:^",
     "@backstage/cli": "workspace:^",
     "@backstage/config": "workspace:^"
   },

plugins/scaffolder-node/src/actions/gitHelpers.test.ts

@@ -0,0 +1,305 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Git, getVoidLogger } from '@backstage/backend-common';
+import { commitAndPushRepo, initRepoAndPush } from './gitHelpers';
+
+jest.mock('@backstage/backend-common', () => ({
+  Git: {
+    fromAuth: jest.fn().mockReturnValue({
+      init: jest.fn(),
+      add: jest.fn(),
+      checkout: jest.fn(),
+      commit: jest
+        .fn()
+        .mockResolvedValue('220f19cc36b551763d157f1b5e4a4b446165dbd6'),
+      fetch: jest.fn(),
+      addRemote: jest.fn(),
+      push: jest.fn(),
+    }),
+  },
+  getVoidLogger: jest.requireActual('@backstage/backend-common').getVoidLogger,
+}));
+
+const mockedGit = Git.fromAuth({
+  logger: getVoidLogger(),
+});
+
+describe('initRepoAndPush', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('with minimal parameters', () => {
+    beforeEach(async () => {
+      await initRepoAndPush({
+        dir: '/test/repo/dir/',
+        remoteUrl: 'git@github.com:test/repo.git',
+        auth: {
+          username: 'test-user',
+          password: 'test-password',
+        },
+        logger: getVoidLogger(),
+      });
+    });
+
+    it('initializes the repo', () => {
+      expect(mockedGit.init).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        defaultBranch: 'master',
+      });
+    });
+
+    it('stages all files in the repo', () => {
+      expect(mockedGit.add).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        filepath: '.',
+      });
+    });
+
+    it('creates an initial commit', () => {
+      expect(mockedGit.commit).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        message: 'Initial commit',
+        author: {
+          name: 'Scaffolder',
+          email: 'scaffolder@backstage.io',
+        },
+        committer: {
+          name: 'Scaffolder',
+          email: 'scaffolder@backstage.io',
+        },
+      });
+    });
+
+    it('adds the appropriate remote', () => {
+      expect(mockedGit.addRemote).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        url: 'git@github.com:test/repo.git',
+        remote: 'origin',
+      });
+    });
+
+    it('pushes to the remote', () => {
+      expect(mockedGit.push).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        remote: 'origin',
+      });
+    });
+  });
+
+  it('with token', async () => {
+    await initRepoAndPush({
+      dir: '/test/repo/dir/',
+      remoteUrl: 'git@github.com:test/repo.git',
+      auth: {
+        token: 'test-token',
+      },
+      logger: getVoidLogger(),
+    });
+
+    expect(mockedGit.init).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      defaultBranch: 'master',
+    });
+  });
+
+  it('allows overriding the default branch', async () => {
+    await initRepoAndPush({
+      dir: '/test/repo/dir/',
+      defaultBranch: 'trunk',
+      remoteUrl: 'git@github.com:test/repo.git',
+      auth: {
+        username: 'test-user',
+        password: 'test-password',
+      },
+      logger: getVoidLogger(),
+    });
+
+    expect(mockedGit.init).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      defaultBranch: 'trunk',
+    });
+  });
+
+  it('allows overriding the author', async () => {
+    await initRepoAndPush({
+      dir: '/test/repo/dir/',
+      gitAuthorInfo: {
+        name: 'Custom Scaffolder Author',
+        email: 'scaffolder@example.org',
+      },
+      remoteUrl: 'git@github.com:test/repo.git',
+      auth: {
+        username: 'test-user',
+        password: 'test-password',
+      },
+      logger: getVoidLogger(),
+    });
+
+    expect(mockedGit.commit).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      message: 'Initial commit',
+      author: {
+        name: 'Custom Scaffolder Author',
+        email: 'scaffolder@example.org',
+      },
+      committer: {
+        name: 'Custom Scaffolder Author',
+        email: 'scaffolder@example.org',
+      },
+    });
+  });
+});
+
+describe('commitAndPushRepo', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('with minimal parameters', () => {
+    beforeEach(async () => {
+      await commitAndPushRepo({
+        dir: '/test/repo/dir/',
+        auth: {
+          username: 'test-user',
+          password: 'test-password',
+        },
+        logger: getVoidLogger(),
+        commitMessage: 'commit message',
+      });
+    });
+
+    it('fetches commits', () => {
+      expect(mockedGit.fetch).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+      });
+    });
+
+    it('checkouts to master', () => {
+      expect(mockedGit.checkout).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        ref: 'master',
+      });
+    });
+
+    it('stages all files in the repo', () => {
+      expect(mockedGit.add).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        filepath: '.',
+      });
+    });
+
+    it('creates a commit', () => {
+      expect(mockedGit.commit).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        message: 'commit message',
+        author: {
+          name: 'Scaffolder',
+          email: 'scaffolder@backstage.io',
+        },
+        committer: {
+          name: 'Scaffolder',
+          email: 'scaffolder@backstage.io',
+        },
+      });
+    });
+
+    it('pushes to the remote', () => {
+      expect(mockedGit.push).toHaveBeenCalledWith({
+        dir: '/test/repo/dir/',
+        remote: 'origin',
+        remoteRef: 'refs/heads/master',
+      });
+    });
+  });
+
+  it('allows overriding the default branch', async () => {
+    await commitAndPushRepo({
+      dir: '/test/repo/dir/',
+      auth: {
+        username: 'test-user',
+        password: 'test-password',
+      },
+      logger: getVoidLogger(),
+      commitMessage: 'commit message',
+      branch: 'otherbranch',
+    });
+
+    expect(mockedGit.checkout).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      ref: 'otherbranch',
+    });
+    expect(mockedGit.push).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      remote: 'origin',
+      remoteRef: 'refs/heads/otherbranch',
+    });
+  });
+
+  it('allows overriding the remote ref', async () => {
+    await commitAndPushRepo({
+      dir: '/test/repo/dir/',
+      auth: {
+        username: 'test-user',
+        password: 'test-password',
+      },
+      logger: getVoidLogger(),
+      commitMessage: 'commit message',
+      remoteRef: 'refs/for/master',
+    });
+
+    expect(mockedGit.checkout).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      ref: 'master',
+    });
+    expect(mockedGit.push).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      remote: 'origin',
+      remoteRef: 'refs/for/master',
+    });
+  });
+
+  it('allows overriding the author', async () => {
+    await commitAndPushRepo({
+      dir: '/test/repo/dir/',
+      commitMessage: 'commit message',
+      gitAuthorInfo: {
+        name: 'Custom Scaffolder Author',
+        email: 'scaffolder@example.org',
+      },
+      auth: {
+        username: 'test-user',
+        password: 'test-password',
+      },
+      logger: getVoidLogger(),
+      branch: 'master',
+    });
+
+    expect(mockedGit.commit).toHaveBeenCalledWith({
+      dir: '/test/repo/dir/',
+      message: 'commit message',
+      author: {
+        name: 'Custom Scaffolder Author',
+        email: 'scaffolder@example.org',
+      },
+      committer: {
+        name: 'Custom Scaffolder Author',
+        email: 'scaffolder@example.org',
+      },
+    });
+  });
+});

plugins/scaffolder-node/src/actions/gitHelpers.ts

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Git } from '@backstage/backend-common';
+import { Logger } from 'winston';
+
+/**
+ * @public
+ */
+export async function initRepoAndPush(input: {
+  dir: string;
+  remoteUrl: string;
+  // For use cases where token has to be used with Basic Auth
+  // it has to be provided as password together with a username
+  // which may be a fixed value defined by the provider.
+  auth: { username: string; password: string } | { token: string };
+  logger: Logger;
+  defaultBranch?: string;
+  commitMessage?: string;
+  gitAuthorInfo?: { name?: string; email?: string };
+}): Promise<{ commitHash: string }> {
+  const {
+    dir,
+    remoteUrl,
+    auth,
+    logger,
+    defaultBranch = 'master',
+    commitMessage = 'Initial commit',
+    gitAuthorInfo,
+  } = input;
+  const git = Git.fromAuth({
+    ...auth,
+    logger,
+  });
+
+  await git.init({
+    dir,
+    defaultBranch,
+  });
+
+  await git.add({ dir, filepath: '.' });
+
+  // use provided info if possible, otherwise use fallbacks
+  const authorInfo = {
+    name: gitAuthorInfo?.name ?? 'Scaffolder',
+    email: gitAuthorInfo?.email ?? 'scaffolder@backstage.io',
+  };
+
+  const commitHash = await git.commit({
+    dir,
+    message: commitMessage,
+    author: authorInfo,
+    committer: authorInfo,
+  });
+  await git.addRemote({
+    dir,
+    url: remoteUrl,
+    remote: 'origin',
+  });
+
+  await git.push({
+    dir,
+    remote: 'origin',
+  });
+
+  return { commitHash };
+}
+
+/**
+ * @public
+ */
+export async function commitAndPushRepo(input: {
+  dir: string;
+  // For use cases where token has to be used with Basic Auth
+  // it has to be provided as password together with a username
+  // which may be a fixed value defined by the provider.
+  auth: { username: string; password: string } | { token: string };
+  logger: Logger;
+  commitMessage: string;
+  gitAuthorInfo?: { name?: string; email?: string };
+  branch?: string;
+  remoteRef?: string;
+}): Promise<{ commitHash: string }> {
+  const {
+    dir,
+    auth,
+    logger,
+    commitMessage,
+    gitAuthorInfo,
+    branch = 'master',
+    remoteRef,
+  } = input;
+
+  const git = Git.fromAuth({
+    ...auth,
+    logger,
+  });
+
+  await git.fetch({ dir });
+  await git.checkout({ dir, ref: branch });
+  await git.add({ dir, filepath: '.' });
+
+  // use provided info if possible, otherwise use fallbacks
+  const authorInfo = {
+    name: gitAuthorInfo?.name ?? 'Scaffolder',
+    email: gitAuthorInfo?.email ?? 'scaffolder@backstage.io',
+  };
+
+  const commitHash = await git.commit({
+    dir,
+    message: commitMessage,
+    author: authorInfo,
+    committer: authorInfo,
+  });
+
+  await git.push({
+    dir,
+    remote: 'origin',
+    remoteRef: remoteRef ?? `refs/heads/${branch}`,
+  });
+
+  return { commitHash };
+}

plugins/scaffolder-node/src/actions/index.ts

@@ -25,3 +25,5 @@ export {
 } from './executeShellCommand';
 export { fetchContents, fetchFile } from './fetch';
 export { type ActionContext, type TemplateAction } from './types';
+export { initRepoAndPush, commitAndPushRepo } from './gitHelpers';
+export { parseRepoUrl, getRepoSourceDirectory } from './util';

plugins/scaffolder-node/src/actions/util.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { InputError } from '@backstage/errors';
+import { isChildPath } from '@backstage/backend-common';
+import { join as joinPath, normalize as normalizePath } from 'path';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+
+/**
+ * @public
+ */
+export const getRepoSourceDirectory = (
+  workspacePath: string,
+  sourcePath: string | undefined,
+) => {
+  if (sourcePath) {
+    const safeSuffix = normalizePath(sourcePath).replace(
+      /^(\.\.(\/|\\|$))+/,
+      '',
+    );
+    const path = joinPath(workspacePath, safeSuffix);
+    if (!isChildPath(workspacePath, path)) {
+      throw new Error('Invalid source path');
+    }
+    return path;
+  }
+  return workspacePath;
+};
+
+/**
+ * @public
+ */
+export const parseRepoUrl = (
+  repoUrl: string,
+  integrations: ScmIntegrationRegistry,
+): {
+  repo: string;
+  host: string;
+  owner?: string;
+  organization?: string;
+  workspace?: string;
+  project?: string;
+} => {
+  let parsed;
+  try {
+    parsed = new URL(`https://${repoUrl}`);
+  } catch (error) {
+    throw new InputError(
+      `Invalid repo URL passed to publisher, got ${repoUrl}, ${error}`,
+    );
+  }
+  const host = parsed.host;
+  const owner = parsed.searchParams.get('owner') ?? undefined;
+  const organization = parsed.searchParams.get('organization') ?? undefined;
+  const workspace = parsed.searchParams.get('workspace') ?? undefined;
+  const project = parsed.searchParams.get('project') ?? undefined;
+
+  const type = integrations.byHost(host)?.type;
+
+  if (!type) {
+    throw new InputError(
+      `No matching integration configuration for host ${host}, please check your integrations config`,
+    );
+  }
+
+  const repo: string = parsed.searchParams.get('repo')!;
+  switch (type) {
+    case 'bitbucket': {
+      if (host === 'www.bitbucket.org') {
+        checkRequiredParams(parsed, 'workspace');
+      }
+      checkRequiredParams(parsed, 'project', 'repo');
+      break;
+    }
+    case 'gitlab': {
+      // project is the projectID, and if defined, owner and repo won't be needed.
+      if (!project) {
+        checkRequiredParams(parsed, 'owner', 'repo');
+      }
+      break;
+    }
+    case 'gerrit': {
+      checkRequiredParams(parsed, 'repo');
+      break;
+    }
+    default: {
+      checkRequiredParams(parsed, 'repo', 'owner');
+      break;
+    }
+  }
+
+  return { host, owner, repo, organization, workspace, project };
+};
+
+function checkRequiredParams(repoUrl: URL, ...params: string[]) {
+  for (let i = 0; i < params.length; i++) {
+    if (!repoUrl.searchParams.get(params[i])) {
+      throw new InputError(
+        `Invalid repo URL passed to publisher: ${repoUrl.toString()}, missing ${
+          params[i]
+        }`,
+      );
+    }
+  }
+}

plugins/scaffolder-node/src/files/deserializeDirectoryContents.test.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import { deserializeDirectoryContents } from './deserializeDirectoryContents';
+import { serializeDirectoryContents } from './serializeDirectoryContents';
+
+describe('deserializeDirectoryContents', () => {
+  const mockDir = createMockDirectory();
+
+  beforeEach(() => {
+    mockDir.clear();
+  });
+
+  it('deserializes contents into a directory', async () => {
+    await deserializeDirectoryContents(mockDir.path, [
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+      },
+    ]);
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+    ]);
+  });
+
+  it('deserializes contents into a deep directory structure', async () => {
+    await deserializeDirectoryContents(mockDir.path, [
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+      },
+      {
+        path: 'a/b.txt',
+        content: Buffer.from('b', 'utf8'),
+      },
+      {
+        path: 'a/b/c.txt',
+        content: Buffer.from('c', 'utf8'),
+      },
+    ]);
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+      {
+        path: 'a/b.txt',
+        content: Buffer.from('b', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+      {
+        path: 'a/b/c.txt',
+        content: Buffer.from('c', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+    ]);
+  });
+});

plugins/scaffolder-node/src/files/deserializeDirectoryContents.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { dirname } from 'path';
+import { resolveSafeChildPath } from '@backstage/backend-common';
+import { SerializedFile } from './types';
+
+/**
+ * Deserializes a list of serialized files into the target directory.
+ *
+ * This method uses `resolveSafeChildPath` to make sure that files are
+ * not written outside of the target directory.
+ *
+ * @public
+ */
+export async function deserializeDirectoryContents(
+  targetPath: string,
+  files: SerializedFile[],
+): Promise<void> {
+  for (const file of files) {
+    const filePath = resolveSafeChildPath(targetPath, file.path);
+    await fs.ensureDir(dirname(filePath));
+    await fs.writeFile(filePath, file.content); // Ignore file mode
+  }
+}

plugins/scaffolder-node/src/files/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { serializeDirectoryContents } from './serializeDirectoryContents';
+export { deserializeDirectoryContents } from './deserializeDirectoryContents';
+export type { SerializedFile } from './types';

plugins/scaffolder-node/src/files/serializeDirectoryContents.test.ts

@@ -0,0 +1,243 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import { serializeDirectoryContents } from './serializeDirectoryContents';
+
+describe('serializeDirectoryContents', () => {
+  const mockDir = createMockDirectory();
+
+  it('should list files in this directory', async () => {
+    await expect(serializeDirectoryContents(__dirname)).resolves.toEqual(
+      expect.arrayContaining([
+        {
+          path: 'index.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+        {
+          path: 'types.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+        {
+          path: 'serializeDirectoryContents.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+        {
+          path: 'serializeDirectoryContents.test.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+      ]),
+    );
+  });
+
+  it('should list files in a mock directory', async () => {
+    mockDir.setContent({
+      'a.txt': 'a',
+      b: {
+        'b1.txt': 'b1',
+        'b2.txt': 'b2',
+      },
+      c: {
+        c1: {
+          'c11.txt': 'c11',
+          c11: {
+            'c111.txt': 'c111',
+          },
+        },
+      },
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('a', 'utf8'),
+      },
+      {
+        path: 'b/b1.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b1', 'utf8'),
+      },
+      {
+        path: 'b/b2.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b2', 'utf8'),
+      },
+      {
+        path: 'c/c1/c11.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c11', 'utf8'),
+      },
+      {
+        path: 'c/c1/c11/c111.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c111', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should ignore symlinked files', async () => {
+    mockDir.setContent({
+      'a.txt': 'some text',
+      sym: ctx => ctx.symlink('./a.txt'),
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('some text', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should pick up broken symlinks', async () => {
+    mockDir.setContent({
+      'b.txt': ctx => ctx.symlink('./a.txt'),
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'b.txt',
+        executable: true,
+        symlink: true,
+        content: Buffer.from('./a.txt', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should ignore symlinked folder files', async () => {
+    mockDir.setContent({
+      'a.txt': 'some text',
+      linkme: {
+        'b.txt': 'lols',
+      },
+      sym: ctx => ctx.symlink('./linkme'),
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('some text', 'utf8'),
+      },
+      {
+        path: 'linkme/b.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('lols', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should ignore gitignored files', async () => {
+    mockDir.setContent({
+      '.gitignore': '*.txt',
+      'a.txt': 'a',
+      'a.log': 'a',
+    });
+
+    await expect(
+      serializeDirectoryContents(mockDir.path, {
+        gitignore: true,
+      }),
+    ).resolves.toEqual([
+      {
+        path: '.gitignore',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('*.txt', 'utf8'),
+      },
+      {
+        path: 'a.log',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('a', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should use custom glob patterns', async () => {
+    mockDir.setContent({
+      '.a': 'a',
+      'a.log': 'a',
+      'a.txt': 'a',
+      b: {
+        '.b': 'b',
+        'b.log': 'b',
+        'b.txt': 'b',
+      },
+      c: {
+        '.c': 'c',
+        'c.log': 'c',
+        'c.txt': 'c',
+      },
+    });
+
+    await expect(
+      serializeDirectoryContents(mockDir.path, {
+        gitignore: true,
+        globPatterns: ['**/*.txt', '*/.?', '*/*.log', '!c/**/.*', '!b/*.log'],
+      }).then(files => files.sort((a, b) => a.path.localeCompare(b.path))),
+    ).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('a', 'utf8'),
+      },
+      {
+        path: 'b/.b',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b', 'utf8'),
+      },
+      {
+        path: 'b/b.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b', 'utf8'),
+      },
+      {
+        path: 'c/c.log',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c', 'utf8'),
+      },
+      {
+        path: 'c/c.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c', 'utf8'),
+      },
+    ]);
+  });
+});

plugins/scaffolder-node/src/files/serializeDirectoryContents.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { promises as fs } from 'fs';
+import globby from 'globby';
+import limiterFactory from 'p-limit';
+import { resolveSafeChildPath } from '@backstage/backend-common';
+import { SerializedFile } from './types';
+import { isError } from '@backstage/errors';
+
+const DEFAULT_GLOB_PATTERNS = ['./**', '!.git'];
+
+export const isExecutable = (fileMode: number | undefined) => {
+  if (!fileMode) {
+    return false;
+  }
+
+  const executeBitMask = 0o000111;
+  const res = fileMode & executeBitMask;
+  return res > 0;
+};
+
+async function asyncFilter<T>(
+  array: T[],
+  callback: (value: T, index: number, array: T[]) => Promise<boolean>,
+): Promise<T[]> {
+  const filterMap = await Promise.all(array.map(callback));
+  return array.filter((_value, index) => filterMap[index]);
+}
+
+/**
+ * @public
+ */
+export async function serializeDirectoryContents(
+  sourcePath: string,
+  options?: {
+    gitignore?: boolean;
+    globPatterns?: string[];
+  },
+): Promise<SerializedFile[]> {
+  const paths = await globby(options?.globPatterns ?? DEFAULT_GLOB_PATTERNS, {
+    cwd: sourcePath,
+    dot: true,
+    gitignore: options?.gitignore,
+    followSymbolicLinks: false,
+    // In order to pick up 'broken' symlinks, we oxymoronically request files AND folders yet we filter out folders
+    // This is because broken symlinks aren't classed as files so we need to glob everything
+    onlyFiles: false,
+    objectMode: true,
+    stats: true,
+  });
+
+  const limiter = limiterFactory(10);
+
+  const valid = await asyncFilter(paths, async ({ dirent, path }) => {
+    if (dirent.isDirectory()) return false;
+    if (!dirent.isSymbolicLink()) return true;
+
+    const safePath = resolveSafeChildPath(sourcePath, path);
+
+    // we only want files that don't exist
+    try {
+      await fs.stat(safePath);
+      return false;
+    } catch (e) {
+      return isError(e) && e.code === 'ENOENT';
+    }
+  });
+
+  return Promise.all(
+    valid.map(async ({ dirent, path, stats }) => ({
+      path,
+      content: await limiter(async () => {
+        const absFilePath = resolveSafeChildPath(sourcePath, path);
+        if (dirent.isSymbolicLink()) {
+          return fs.readlink(absFilePath, 'buffer');
+        }
+        return fs.readFile(absFilePath);
+      }),
+      executable: isExecutable(stats?.mode),
+      symlink: dirent.isSymbolicLink(),
+    })),
+  );
+}

plugins/scaffolder-node/src/files/types.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @public
+ */
+export interface SerializedFile {
+  path: string;
+  content: Buffer;
+  executable?: boolean;
+  symlink?: boolean;
+}

plugins/scaffolder-node/src/index.ts

@@ -22,4 +22,5 @@
 
 export * from './actions';
 export * from './tasks';
+export * from './files';
 export type { TemplateFilter, TemplateGlobal } from './types';