From 03a467ebc9fda69117e2f681cd8a3d6aba453ea3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 31 May 2022 11:23:40 +0200 Subject: [PATCH] scaffolder-backend: refactor actions to use serializeDirectoryContents Signed-off-by: Patrik Oldsberg --- .../actions/builtin/fetch/template.ts | 2 + .../builtin/publish/githubPullRequest.ts | 73 +++++++------------ .../builtin/publish/gitlabMergeRequest.ts | 38 ++++------ 3 files changed, 43 insertions(+), 70 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts index 041263c90c..c6d6b86e7b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts @@ -154,6 +154,7 @@ export function createFetchTemplateAction(options: { dot: true, onlyFiles: false, markDirectories: true, + followSymbolicLinks: false, }); const nonTemplatedEntries = new Set( @@ -165,6 +166,7 @@ export function createFetchTemplateAction(options: { dot: true, onlyFiles: false, markDirectories: true, + followSymbolicLinks: false, }), ), ) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts index 3bf90d1e30..f8f9b22d61 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts @@ -14,21 +14,19 @@ * limitations under the License. */ -import fs from 'fs-extra'; -import { parseRepoUrl, isExecutable } from './util'; - +import path from 'path'; +import { parseRepoUrl } from './util'; import { GithubCredentialsProvider, ScmIntegrationRegistry, } from '@backstage/integration'; -import { zipObject } from 'lodash'; import { createTemplateAction } from '../../createTemplateAction'; import { Octokit } from 'octokit'; import { InputError, CustomErrorBase } from '@backstage/errors'; import { createPullRequest } from 'octokit-plugin-create-pull-request'; -import globby from 'globby'; import { resolveSafeChildPath } from '@backstage/backend-common'; import { getOctokitOptions } from '../github/helpers'; +import { serializeDirectoryContents } from '../../../../lib/files'; export type Encoding = 'utf-8' | 'base64'; @@ -219,56 +217,39 @@ export const createPublishGithubPullRequestAction = ({ ? resolveSafeChildPath(ctx.workspacePath, sourcePath) : ctx.workspacePath; - const localFilePaths = await globby(['./**', './**/.*', '!.git'], { - cwd: fileRoot, + const directoryContents = await serializeDirectoryContents(fileRoot, { gitignore: true, - dot: true, }); - - const fileContents = await Promise.all( - localFilePaths.map(filePath => { - const absPath = resolveSafeChildPath(fileRoot, filePath); - const base64EncodedContent = fs - .readFileSync(absPath) - .toString('base64'); - const fileStat = fs.statSync(absPath); - // See the properties of tree items - // in https://docs.github.com/en/rest/reference/git#trees - const githubTreeItemMode = isExecutable(fileStat.mode) - ? '100755' - : '100644'; - // Always use base64 encoding to avoid doubling a binary file in size - // due to interpreting a binary file as utf-8 and sending github - // the utf-8 encoded content. - // - // For example, the original gradle-wrapper.jar is 57.8k in https://github.com/kennethzfeng/pull-request-test/pull/5/files. - // Its size could be doubled to 98.3K (See https://github.com/kennethzfeng/pull-request-test/pull/4/files) - const encoding: Encoding = 'base64'; - return { - encoding: encoding, - content: base64EncodedContent, - mode: githubTreeItemMode, - }; - }), + const files = Object.fromEntries( + directoryContents.map(file => [ + targetPath ? path.posix.join(targetPath, file.path) : file.path, + { + // See the properties of tree items + // in https://docs.github.com/en/rest/reference/git#trees + mode: file.executable ? '100755' : '100644', + // Always use base64 encoding to avoid doubling a binary file in size + // due to interpreting a binary file as utf-8 and sending github + // the utf-8 encoded content. + // + // For example, the original gradle-wrapper.jar is 57.8k in https://github.com/kennethzfeng/pull-request-test/pull/5/files. + // Its size could be doubled to 98.3K (See https://github.com/kennethzfeng/pull-request-test/pull/4/files) + encoding: 'base64' as const, + content: file.content.toString('base64'), + }, + ]), ); - const repoFilePaths = localFilePaths.map(repoFilePath => { - return targetPath ? `${targetPath}/${repoFilePath}` : repoFilePath; - }); - - const changes = [ - { - files: zipObject(repoFilePaths, fileContents), - commit: title, - }, - ]; - try { const response = await client.createPullRequest({ owner, repo, title, - changes, + changes: [ + { + files, + commit: title, + }, + ], body: description, head: branchName, draft, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlabMergeRequest.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlabMergeRequest.ts index 63cfeaa8b3..7259336b23 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlabMergeRequest.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlabMergeRequest.ts @@ -14,15 +14,14 @@ * limitations under the License. */ import { createTemplateAction } from '../../createTemplateAction'; -import { readFile } from 'fs-extra'; import { Gitlab } from '@gitbeaker/node'; -import globby from 'globby'; import { Types } from '@gitbeaker/core'; - +import path from 'path'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { InputError } from '@backstage/errors'; import { parseRepoUrl } from './util'; import { resolveSafeChildPath } from '@backstage/backend-common'; +import { serializeDirectoryContents } from '../../../../lib/files'; /** * Create a new action that creates a gitlab merge request. @@ -106,8 +105,6 @@ export const createPublishGitlabMergeRequestAction = (options: { const { host } = parseRepoUrl(repoUrl, integrations); const integrationConfig = integrations.gitlab.byHost(host); - const actions: Types.CommitAction[] = []; - const destinationBranch = ctx.input.branchName; if (!integrationConfig) { @@ -128,28 +125,21 @@ export const createPublishGitlabMergeRequestAction = (options: { [tokenType]: token, }); - const fileRoot = ctx.workspacePath; - const localFilePaths = await globby([`${ctx.input.targetPath}/**`], { - cwd: fileRoot, - gitignore: true, - dot: true, - }); - - const fileContents = await Promise.all( - localFilePaths.map(p => readFile(resolveSafeChildPath(fileRoot, p))), + const targetPath = resolveSafeChildPath( + ctx.workspacePath, + ctx.input.targetPath, ); - - const repoFilePaths = localFilePaths.map(repoFilePath => { - return repoFilePath; + const fileContents = await serializeDirectoryContents(targetPath, { + gitignore: true, }); - for (let i = 0; i < repoFilePaths.length; i++) { - actions.push({ - action: 'create', - filePath: repoFilePaths[i], - content: fileContents[i].toString(), - }); - } + const actions: Types.CommitAction[] = fileContents.map(file => ({ + action: 'create', + filePath: path.posix.join(ctx.input.targetPath, file.path), + encoding: 'base64', + content: file.content.toString('base64'), + execute_filemode: file.executable, + })); const projects = await api.Projects.show(ctx.input.projectid);