chore: move some other things around a little bit

Signed-off-by: blam <ben@blam.sh>

plugins/scaffolder-node/src/actions/index.ts

@@ -25,3 +25,5 @@ export {
 } from './executeShellCommand';
 export { fetchContents, fetchFile } from './fetch';
 export { type ActionContext, type TemplateAction } from './types';
+export { initRepoAndPush, commitAndPushRepo } from './repoHelpers';
+export { parseRepoUrl, getRepoSourceDirectory } from './util';

plugins/scaffolder-node/src/actions/repoHelpers.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Git } from '@backstage/backend-common';
+import { Logger } from 'winston';
+
+export async function initRepoAndPush({
+  dir,
+  remoteUrl,
+  auth,
+  logger,
+  defaultBranch = 'master',
+  commitMessage = 'Initial commit',
+  gitAuthorInfo,
+}: {
+  dir: string;
+  remoteUrl: string;
+  // For use cases where token has to be used with Basic Auth
+  // it has to be provided as password together with a username
+  // which may be a fixed value defined by the provider.
+  auth: { username: string; password: string } | { token: string };
+  logger: Logger;
+  defaultBranch?: string;
+  commitMessage?: string;
+  gitAuthorInfo?: { name?: string; email?: string };
+}): Promise<{ commitHash: string }> {
+  const git = Git.fromAuth({
+    ...auth,
+    logger,
+  });
+
+  await git.init({
+    dir,
+    defaultBranch,
+  });
+
+  await git.add({ dir, filepath: '.' });
+
+  // use provided info if possible, otherwise use fallbacks
+  const authorInfo = {
+    name: gitAuthorInfo?.name ?? 'Scaffolder',
+    email: gitAuthorInfo?.email ?? 'scaffolder@backstage.io',
+  };
+
+  const commitHash = await git.commit({
+    dir,
+    message: commitMessage,
+    author: authorInfo,
+    committer: authorInfo,
+  });
+  await git.addRemote({
+    dir,
+    url: remoteUrl,
+    remote: 'origin',
+  });
+
+  await git.push({
+    dir,
+    remote: 'origin',
+  });
+
+  return { commitHash };
+}
+
+export async function commitAndPushRepo({
+  dir,
+  auth,
+  logger,
+  commitMessage,
+  gitAuthorInfo,
+  branch = 'master',
+  remoteRef,
+}: {
+  dir: string;
+  // For use cases where token has to be used with Basic Auth
+  // it has to be provided as password together with a username
+  // which may be a fixed value defined by the provider.
+  auth: { username: string; password: string } | { token: string };
+  logger: Logger;
+  commitMessage: string;
+  gitAuthorInfo?: { name?: string; email?: string };
+  branch?: string;
+  remoteRef?: string;
+}): Promise<{ commitHash: string }> {
+  const git = Git.fromAuth({
+    ...auth,
+    logger,
+  });
+
+  await git.fetch({ dir });
+  await git.checkout({ dir, ref: branch });
+  await git.add({ dir, filepath: '.' });
+
+  // use provided info if possible, otherwise use fallbacks
+  const authorInfo = {
+    name: gitAuthorInfo?.name ?? 'Scaffolder',
+    email: gitAuthorInfo?.email ?? 'scaffolder@backstage.io',
+  };
+
+  const commitHash = await git.commit({
+    dir,
+    message: commitMessage,
+    author: authorInfo,
+    committer: authorInfo,
+  });
+
+  await git.push({
+    dir,
+    remote: 'origin',
+    remoteRef: remoteRef ?? `refs/heads/${branch}`,
+  });
+
+  return { commitHash };
+}

plugins/scaffolder-node/src/actions/util.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { InputError } from '@backstage/errors';
+import { isChildPath } from '@backstage/backend-common';
+import { join as joinPath, normalize as normalizePath } from 'path';
+import { ScmIntegrationRegistry } from '@backstage/integration';
+
+export const getRepoSourceDirectory = (
+  workspacePath: string,
+  sourcePath: string | undefined,
+) => {
+  if (sourcePath) {
+    const safeSuffix = normalizePath(sourcePath).replace(
+      /^(\.\.(\/|\\|$))+/,
+      '',
+    );
+    const path = joinPath(workspacePath, safeSuffix);
+    if (!isChildPath(workspacePath, path)) {
+      throw new Error('Invalid source path');
+    }
+    return path;
+  }
+  return workspacePath;
+};
+
+export const parseRepoUrl = (
+  repoUrl: string,
+  integrations: ScmIntegrationRegistry,
+): {
+  repo: string;
+  host: string;
+  owner?: string;
+  organization?: string;
+  workspace?: string;
+  project?: string;
+} => {
+  let parsed;
+  try {
+    parsed = new URL(`https://${repoUrl}`);
+  } catch (error) {
+    throw new InputError(
+      `Invalid repo URL passed to publisher, got ${repoUrl}, ${error}`,
+    );
+  }
+  const host = parsed.host;
+  const owner = parsed.searchParams.get('owner') ?? undefined;
+  const organization = parsed.searchParams.get('organization') ?? undefined;
+  const workspace = parsed.searchParams.get('workspace') ?? undefined;
+  const project = parsed.searchParams.get('project') ?? undefined;
+
+  const type = integrations.byHost(host)?.type;
+
+  if (!type) {
+    throw new InputError(
+      `No matching integration configuration for host ${host}, please check your integrations config`,
+    );
+  }
+
+  const repo: string = parsed.searchParams.get('repo')!;
+  switch (type) {
+    case 'bitbucket': {
+      if (host === 'www.bitbucket.org') {
+        checkRequiredParams(parsed, 'workspace');
+      }
+      checkRequiredParams(parsed, 'project', 'repo');
+      break;
+    }
+    case 'gitlab': {
+      // project is the projectID, and if defined, owner and repo won't be needed.
+      if (!project) {
+        checkRequiredParams(parsed, 'owner', 'repo');
+      }
+      break;
+    }
+    case 'gerrit': {
+      checkRequiredParams(parsed, 'repo');
+      break;
+    }
+    default: {
+      checkRequiredParams(parsed, 'repo', 'owner');
+      break;
+    }
+  }
+
+  return { host, owner, repo, organization, workspace, project };
+};
+
+function checkRequiredParams(repoUrl: URL, ...params: string[]) {
+  for (let i = 0; i < params.length; i++) {
+    if (!repoUrl.searchParams.get(params[i])) {
+      throw new InputError(
+        `Invalid repo URL passed to publisher: ${repoUrl.toString()}, missing ${
+          params[i]
+        }`,
+      );
+    }
+  }
+}

plugins/scaffolder-node/src/files/deserializeDirectoryContents.test.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import { deserializeDirectoryContents } from './deserializeDirectoryContents';
+import { serializeDirectoryContents } from './serializeDirectoryContents';
+
+describe('deserializeDirectoryContents', () => {
+  const mockDir = createMockDirectory();
+
+  beforeEach(() => {
+    mockDir.clear();
+  });
+
+  it('deserializes contents into a directory', async () => {
+    await deserializeDirectoryContents(mockDir.path, [
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+      },
+    ]);
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+    ]);
+  });
+
+  it('deserializes contents into a deep directory structure', async () => {
+    await deserializeDirectoryContents(mockDir.path, [
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+      },
+      {
+        path: 'a/b.txt',
+        content: Buffer.from('b', 'utf8'),
+      },
+      {
+        path: 'a/b/c.txt',
+        content: Buffer.from('c', 'utf8'),
+      },
+    ]);
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        content: Buffer.from('a', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+      {
+        path: 'a/b.txt',
+        content: Buffer.from('b', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+      {
+        path: 'a/b/c.txt',
+        content: Buffer.from('c', 'utf8'),
+        executable: false,
+        symlink: false,
+      },
+    ]);
+  });
+});

plugins/scaffolder-node/src/files/deserializeDirectoryContents.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import { dirname } from 'path';
+import { resolveSafeChildPath } from '@backstage/backend-common';
+import { SerializedFile } from './types';
+
+/**
+ * Deserializes a list of serialized files into the target directory.
+ *
+ * This method uses `resolveSafeChildPath` to make sure that files are
+ * not written outside of the target directory.
+ *
+ * @internal
+ */
+export async function deserializeDirectoryContents(
+  targetPath: string,
+  files: SerializedFile[],
+): Promise<void> {
+  for (const file of files) {
+    const filePath = resolveSafeChildPath(targetPath, file.path);
+    await fs.ensureDir(dirname(filePath));
+    await fs.writeFile(filePath, file.content); // Ignore file mode
+  }
+}

plugins/scaffolder-node/src/files/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { serializeDirectoryContents } from './serializeDirectoryContents';
+export { deserializeDirectoryContents } from './deserializeDirectoryContents';
+export type { SerializedFile } from './types';

plugins/scaffolder-node/src/files/serializeDirectoryContents.test.ts

@@ -0,0 +1,243 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createMockDirectory } from '@backstage/backend-test-utils';
+import { serializeDirectoryContents } from './serializeDirectoryContents';
+
+describe('serializeDirectoryContents', () => {
+  const mockDir = createMockDirectory();
+
+  it('should list files in this directory', async () => {
+    await expect(serializeDirectoryContents(__dirname)).resolves.toEqual(
+      expect.arrayContaining([
+        {
+          path: 'index.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+        {
+          path: 'types.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+        {
+          path: 'serializeDirectoryContents.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+        {
+          path: 'serializeDirectoryContents.test.ts',
+          executable: false,
+          symlink: false,
+          content: expect.any(Buffer),
+        },
+      ]),
+    );
+  });
+
+  it('should list files in a mock directory', async () => {
+    mockDir.setContent({
+      'a.txt': 'a',
+      b: {
+        'b1.txt': 'b1',
+        'b2.txt': 'b2',
+      },
+      c: {
+        c1: {
+          'c11.txt': 'c11',
+          c11: {
+            'c111.txt': 'c111',
+          },
+        },
+      },
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('a', 'utf8'),
+      },
+      {
+        path: 'b/b1.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b1', 'utf8'),
+      },
+      {
+        path: 'b/b2.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b2', 'utf8'),
+      },
+      {
+        path: 'c/c1/c11.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c11', 'utf8'),
+      },
+      {
+        path: 'c/c1/c11/c111.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c111', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should ignore symlinked files', async () => {
+    mockDir.setContent({
+      'a.txt': 'some text',
+      sym: ctx => ctx.symlink('./a.txt'),
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('some text', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should pick up broken symlinks', async () => {
+    mockDir.setContent({
+      'b.txt': ctx => ctx.symlink('./a.txt'),
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'b.txt',
+        executable: true,
+        symlink: true,
+        content: Buffer.from('./a.txt', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should ignore symlinked folder files', async () => {
+    mockDir.setContent({
+      'a.txt': 'some text',
+      linkme: {
+        'b.txt': 'lols',
+      },
+      sym: ctx => ctx.symlink('./linkme'),
+    });
+
+    await expect(serializeDirectoryContents(mockDir.path)).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('some text', 'utf8'),
+      },
+      {
+        path: 'linkme/b.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('lols', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should ignore gitignored files', async () => {
+    mockDir.setContent({
+      '.gitignore': '*.txt',
+      'a.txt': 'a',
+      'a.log': 'a',
+    });
+
+    await expect(
+      serializeDirectoryContents(mockDir.path, {
+        gitignore: true,
+      }),
+    ).resolves.toEqual([
+      {
+        path: '.gitignore',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('*.txt', 'utf8'),
+      },
+      {
+        path: 'a.log',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('a', 'utf8'),
+      },
+    ]);
+  });
+
+  it('should use custom glob patterns', async () => {
+    mockDir.setContent({
+      '.a': 'a',
+      'a.log': 'a',
+      'a.txt': 'a',
+      b: {
+        '.b': 'b',
+        'b.log': 'b',
+        'b.txt': 'b',
+      },
+      c: {
+        '.c': 'c',
+        'c.log': 'c',
+        'c.txt': 'c',
+      },
+    });
+
+    await expect(
+      serializeDirectoryContents(mockDir.path, {
+        gitignore: true,
+        globPatterns: ['**/*.txt', '*/.?', '*/*.log', '!c/**/.*', '!b/*.log'],
+      }).then(files => files.sort((a, b) => a.path.localeCompare(b.path))),
+    ).resolves.toEqual([
+      {
+        path: 'a.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('a', 'utf8'),
+      },
+      {
+        path: 'b/.b',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b', 'utf8'),
+      },
+      {
+        path: 'b/b.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('b', 'utf8'),
+      },
+      {
+        path: 'c/c.log',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c', 'utf8'),
+      },
+      {
+        path: 'c/c.txt',
+        executable: false,
+        symlink: false,
+        content: Buffer.from('c', 'utf8'),
+      },
+    ]);
+  });
+});

plugins/scaffolder-node/src/files/serializeDirectoryContents.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { promises as fs } from 'fs';
+import globby from 'globby';
+import limiterFactory from 'p-limit';
+import { resolveSafeChildPath } from '@backstage/backend-common';
+import { SerializedFile } from './types';
+import { isError } from '@backstage/errors';
+
+const DEFAULT_GLOB_PATTERNS = ['./**', '!.git'];
+
+export const isExecutable = (fileMode: number | undefined) => {
+  if (!fileMode) {
+    return false;
+  }
+
+  const executeBitMask = 0o000111;
+  const res = fileMode & executeBitMask;
+  return res > 0;
+};
+
+async function asyncFilter<T>(
+  array: T[],
+  callback: (value: T, index: number, array: T[]) => Promise<boolean>,
+): Promise<T[]> {
+  const filterMap = await Promise.all(array.map(callback));
+  return array.filter((_value, index) => filterMap[index]);
+}
+
+export async function serializeDirectoryContents(
+  sourcePath: string,
+  options?: {
+    gitignore?: boolean;
+    globPatterns?: string[];
+  },
+): Promise<SerializedFile[]> {
+  const paths = await globby(options?.globPatterns ?? DEFAULT_GLOB_PATTERNS, {
+    cwd: sourcePath,
+    dot: true,
+    gitignore: options?.gitignore,
+    followSymbolicLinks: false,
+    // In order to pick up 'broken' symlinks, we oxymoronically request files AND folders yet we filter out folders
+    // This is because broken symlinks aren't classed as files so we need to glob everything
+    onlyFiles: false,
+    objectMode: true,
+    stats: true,
+  });
+
+  const limiter = limiterFactory(10);
+
+  const valid = await asyncFilter(paths, async ({ dirent, path }) => {
+    if (dirent.isDirectory()) return false;
+    if (!dirent.isSymbolicLink()) return true;
+
+    const safePath = resolveSafeChildPath(sourcePath, path);
+
+    // we only want files that don't exist
+    try {
+      await fs.stat(safePath);
+      return false;
+    } catch (e) {
+      return isError(e) && e.code === 'ENOENT';
+    }
+  });
+
+  return Promise.all(
+    valid.map(async ({ dirent, path, stats }) => ({
+      path,
+      content: await limiter(async () => {
+        const absFilePath = resolveSafeChildPath(sourcePath, path);
+        if (dirent.isSymbolicLink()) {
+          return fs.readlink(absFilePath, 'buffer');
+        }
+        return fs.readFile(absFilePath);
+      }),
+      executable: isExecutable(stats?.mode),
+      symlink: dirent.isSymbolicLink(),
+    })),
+  );
+}

plugins/scaffolder-node/src/files/types.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface SerializedFile {
+  path: string;
+  content: Buffer;
+  executable?: boolean;
+  symlink?: boolean;
+}

plugins/scaffolder-node/src/index.ts

@@ -22,4 +22,5 @@
 
 export * from './actions';
 export * from './tasks';
+export * from './files';
 export type { TemplateFilter, TemplateGlobal } from './types';