diff --git a/.changeset/tasty-spoons-beg.md b/.changeset/tasty-spoons-beg.md new file mode 100644 index 0000000000..651c70e03c --- /dev/null +++ b/.changeset/tasty-spoons-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Rather than calling `yarn pack`, the `build-workspace` and `backend-bundle` commands now move files directly whenever possible. This cuts out several `yarn` invocations and speeds the packing process up by several orders of magnitude. diff --git a/packages/cli/package.json b/packages/cli/package.json index 41cefe6c53..0f6c521ec4 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -86,6 +86,7 @@ "lodash": "^4.17.21", "minimatch": "3.0.4", "mini-css-extract-plugin": "^2.4.2", + "npm-packlist": "^3.0.0", "node-libs-browser": "^2.2.1", "ora": "^5.3.0", "postcss": "^8.1.0", @@ -132,6 +133,7 @@ "@types/minimatch": "^3.0.5", "@types/mock-fs": "^4.13.0", "@types/node": "^14.14.32", + "@types/npm-packlist": "^1.1.2", "@types/recursive-readdir": "^2.2.0", "@types/rollup-plugin-peer-deps-external": "^2.2.0", "@types/rollup-plugin-postcss": "^2.0.0", diff --git a/packages/cli/src/lib/packager/copyPackageDist.ts b/packages/cli/src/lib/packager/copyPackageDist.ts new file mode 100644 index 0000000000..03ac361006 --- /dev/null +++ b/packages/cli/src/lib/packager/copyPackageDist.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import npmPackList from 'npm-packlist'; +import { join as joinPath, resolve as resolvePath } from 'path'; + +const SKIPPED_KEYS = ['access', 'registry', 'tag', 'alphaTypes', 'betaTypes']; + +// Writes e.g. alpha/package.json +async function writeReleaseStageEntrypoint( + pkg: any, + stage: 'alpha' | 'beta', + targetDir: string, +) { + await fs.ensureDir(resolvePath(targetDir, stage)); + await fs.writeJson( + resolvePath(targetDir, stage, 'package.json'), + { + name: pkg.name, + version: pkg.version, + main: (pkg.publishConfig.main || pkg.main) && '..', + module: (pkg.publishConfig.module || pkg.module) && '..', + browser: (pkg.publishConfig.browser || pkg.browser) && '..', + types: joinPath('..', pkg.publishConfig[`${stage}Types`]), + }, + { encoding: 'utf8', spaces: 2 }, + ); +} + +export async function copyPackageDist(packageDir: string, targetDir: string) { + const pkgPath = resolvePath(packageDir, 'package.json'); + const pkgContent = await fs.readFile(pkgPath, 'utf8'); + const pkg = JSON.parse(pkgContent); + + const publishConfig = pkg.publishConfig ?? {}; + for (const key of Object.keys(publishConfig)) { + if (!SKIPPED_KEYS.includes(key)) { + pkg[key] = publishConfig[key]; + } + } + + // We remove the dependencies from package.json of packages that are marked + // as bundled, so that yarn doesn't try to install them. + if (pkg.bundled) { + delete pkg.dependencies; + delete pkg.devDependencies; + delete pkg.peerDependencies; + delete pkg.optionalDependencies; + } + + // Write the modified package.json so that the file listing is correct + await fs.writeJson(pkgPath, pkg, { encoding: 'utf8', spaces: 2 }); + + // Lists all dist files, respecting .npmignore, files field in package.json, etc. + const filePaths = await npmPackList({ path: packageDir }); + + await fs.ensureDir(targetDir); + for (const filePath of filePaths.sort()) { + await fs.copy( + resolvePath(packageDir, filePath), + resolvePath(targetDir, filePath), + ); + } + + if (publishConfig.alphaTypes) { + await writeReleaseStageEntrypoint(pkg, 'alpha', targetDir); + } + if (publishConfig.betaTypes) { + await writeReleaseStageEntrypoint(pkg, 'beta', targetDir); + } + + // Restore package.json + await fs.writeFile(pkgPath, pkgContent, 'utf8'); +} diff --git a/packages/cli/src/lib/packager/createDistWorkspace.ts b/packages/cli/src/lib/packager/createDistWorkspace.ts new file mode 100644 index 0000000000..15ccda30c2 --- /dev/null +++ b/packages/cli/src/lib/packager/createDistWorkspace.ts @@ -0,0 +1,303 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import chalk from 'chalk'; +import fs from 'fs-extra'; +import { + join as joinPath, + resolve as resolvePath, + relative as relativePath, +} from 'path'; +import { tmpdir } from 'os'; +import tar, { CreateOptions } from 'tar'; +import partition from 'lodash/partition'; +import { paths } from '../paths'; +import { run } from '../run'; +import { + dependencies as cliDependencies, + devDependencies as cliDevDependencies, +} from '../../../package.json'; +import { PackageGraph, PackageGraphNode } from '../monorepo'; +import { + BuildOptions, + buildPackages, + getOutputsForRole, + Output, +} from '../builder'; +import { copyPackageDist } from './copyPackageDist'; + +// These packages aren't safe to pack in parallel since the CLI depends on them +const UNSAFE_PACKAGES = [ + ...Object.keys(cliDependencies), + ...Object.keys(cliDevDependencies), +]; + +type FileEntry = + | string + | { + src: string; + dest: string; + }; + +type Options = { + /** + * Target directory for the dist workspace, defaults to a temporary directory + */ + targetDir?: string; + + /** + * Files to copy into the target workspace. + * + * Defaults to ['yarn.lock', 'package.json']. + */ + files?: FileEntry[]; + + /** + * If set to true, the target packages are built before they are packaged into the workspace. + */ + buildDependencies?: boolean; + + /** + * When `buildDependencies` is set, this list of packages will not be built even if they are dependencies. + */ + buildExcludes?: string[]; + + /** + * Controls amount of parallelism in some build steps. + */ + parallelism?: number; + + /** + * If set, creates a skeleton tarball that contains all package.json files + * with the same structure as the workspace dir. + */ + skeleton?: 'skeleton.tar' | 'skeleton.tar.gz'; +}; + +/** + * Uses `yarn pack` to package local packages and unpacks them into a dist workspace. + * The target workspace will end up containing dist version of each package and + * will be suitable for packaging e.g. into a docker image. + * + * This creates a structure that is functionally similar to if the packages were + * installed from npm, but uses Yarn workspaces to link to them at runtime. + */ +export async function createDistWorkspace( + packageNames: string[], + options: Options = {}, +) { + const targetDir = + options.targetDir ?? + (await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace'))); + + const packages = await PackageGraph.listTargetPackages(); + const packageGraph = PackageGraph.fromPackages(packages); + const targetNames = packageGraph.collectPackageNames(packageNames, node => { + // Don't include dependencies of packages that are marked as bundled + if (node.packageJson.bundled) { + return undefined; + } + + return node.publishedLocalDependencies.keys(); + }); + const targets = Array.from(targetNames).map(name => packageGraph.get(name)!); + + if (options.buildDependencies) { + const exclude = options.buildExcludes ?? []; + + const toBuild = new Set( + targets.map(_ => _.name).filter(name => !exclude.includes(name)), + ); + + const standardBuilds = new Array(); + const customBuild = new Array(); + + for (const pkg of packages) { + if (!toBuild.has(pkg.packageJson.name)) { + continue; + } + const role = pkg.packageJson.backstage?.role; + if (!role) { + console.warn(`Ignored ${pkg.packageJson.name} because it has no role`); + customBuild.push(pkg.packageJson.name); + continue; + } + + const buildScript = pkg.packageJson.scripts?.build; + if (!buildScript) { + customBuild.push(pkg.packageJson.name); + continue; + } + + if (!buildScript.startsWith('backstage-cli script build')) { + console.warn( + `Ignored ${pkg.packageJson.name} because it has a custom build script, '${buildScript}'`, + ); + customBuild.push(pkg.packageJson.name); + continue; + } + + const outputs = getOutputsForRole(role); + + // No need to build and include types in the production runtime + outputs.delete(Output.types); + + if (outputs.size > 0) { + standardBuilds.push({ + targetDir: pkg.dir, + outputs: outputs, + logPrefix: `${chalk.cyan(relativePath(paths.targetRoot, pkg.dir))}: `, + // No need to detect these for the backend builds, we assume no minification or types + minify: false, + useApiExtractor: false, + }); + } + } + + await buildPackages(standardBuilds); + + if (customBuild.length > 0) { + const scopeArgs = customBuild.flatMap(name => ['--scope', name]); + const lernaArgs = + options.parallelism && Number.isInteger(options.parallelism) + ? ['--concurrency', options.parallelism.toString()] + : []; + + await run('yarn', ['lerna', ...lernaArgs, 'run', ...scopeArgs, 'build'], { + cwd: paths.targetRoot, + }); + } + } + + await moveToDistWorkspace(targetDir, targets); + + const files: FileEntry[] = options.files ?? ['yarn.lock', 'package.json']; + + for (const file of files) { + const src = typeof file === 'string' ? file : file.src; + const dest = typeof file === 'string' ? file : file.dest; + await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest)); + } + + if (options.skeleton) { + const skeletonFiles = targets.map(target => { + const dir = relativePath(paths.targetRoot, target.dir); + return joinPath(dir, 'package.json'); + }); + + await tar.create( + { + file: resolvePath(targetDir, options.skeleton), + cwd: targetDir, + portable: true, + noMtime: true, + gzip: options.skeleton.endsWith('.gz'), + } as CreateOptions & { noMtime: boolean }, + skeletonFiles, + ); + } + + return targetDir; +} + +const FAST_PACK_SCRIPTS = [ + undefined, + 'backstage-cli prepack', + 'backstage-cli script prepack', +]; + +async function moveToDistWorkspace( + workspaceDir: string, + localPackages: PackageGraphNode[], +): Promise { + const [fastPackPackages, slowPackPackages] = partition(localPackages, pkg => + FAST_PACK_SCRIPTS.includes(pkg.packageJson.scripts?.prepack), + ); + + // New an improved flow where we avoid calling `yarn pack` + await Promise.all( + fastPackPackages.map(async target => { + console.log(`Moving ${target.name} into dist workspace`); + + const outputDir = relativePath(paths.targetRoot, target.dir); + const absoluteOutputPath = resolvePath(workspaceDir, outputDir); + await copyPackageDist(target.dir, absoluteOutputPath); + }), + ); + + // Old flow is below, which calls `yarn pack` and extracts the tarball + + async function pack(target: PackageGraphNode, archive: string) { + console.log(`Repacking ${target.name} into dist workspace`); + const archivePath = resolvePath(workspaceDir, archive); + + await run('yarn', ['pack', '--filename', archivePath], { + cwd: target.dir, + }); + // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed + if (target.packageJson?.scripts?.postpack) { + await run('yarn', ['postpack'], { cwd: target.dir }); + } + + const outputDir = relativePath(paths.targetRoot, target.dir); + const absoluteOutputPath = resolvePath(workspaceDir, outputDir); + await fs.ensureDir(absoluteOutputPath); + + await tar.extract({ + file: archivePath, + cwd: absoluteOutputPath, + strip: 1, + }); + await fs.remove(archivePath); + + // We remove the dependencies from package.json of packages that are marked + // as bundled, so that yarn doesn't try to install them. + if (target.packageJson.bundled) { + const pkgJson = await fs.readJson( + resolvePath(absoluteOutputPath, 'package.json'), + ); + delete pkgJson.dependencies; + delete pkgJson.devDependencies; + delete pkgJson.peerDependencies; + delete pkgJson.optionalDependencies; + + await fs.writeJson( + resolvePath(absoluteOutputPath, 'package.json'), + pkgJson, + { + spaces: 2, + }, + ); + } + } + + const [unsafePackages, safePackages] = partition(slowPackPackages, p => + UNSAFE_PACKAGES.includes(p.name), + ); + + // The unsafe package are packed first one by one in order to avoid race conditions + // where the CLI is being executed with broken dependencies. + for (const target of unsafePackages) { + await pack(target, `temp-package.tgz`); + } + + // Repacking in parallel is much faster and safe for all packages outside of the Backstage repo + await Promise.all( + safePackages.map(async (target, index) => + pack(target, `temp-package-${index}.tgz`), + ), + ); +} diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 40af21bc5e..75f3fdf71d 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,212 +14,4 @@ * limitations under the License. */ -import fs from 'fs-extra'; -import { - join as joinPath, - resolve as resolvePath, - relative as relativePath, -} from 'path'; -import { tmpdir } from 'os'; -import tar, { CreateOptions } from 'tar'; -import { paths } from '../paths'; -import { run } from '../run'; -import { - dependencies as cliDependencies, - devDependencies as cliDevDependencies, -} from '../../../package.json'; -import { getPackages } from '@manypkg/get-packages'; -import { PackageGraph, PackageGraphNode } from '../monorepo'; - -// These packages aren't safe to pack in parallel since the CLI depends on them -const UNSAFE_PACKAGES = [ - ...Object.keys(cliDependencies), - ...Object.keys(cliDevDependencies), -]; - -type FileEntry = - | string - | { - src: string; - dest: string; - }; - -type Options = { - /** - * Target directory for the dist workspace, defaults to a temporary directory - */ - targetDir?: string; - - /** - * Files to copy into the target workspace. - * - * Defaults to ['yarn.lock', 'package.json']. - */ - files?: FileEntry[]; - - /** - * If set to true, the target packages are built before they are packaged into the workspace. - */ - buildDependencies?: boolean; - - /** - * When `buildDependencies` is set, this list of packages will not be built even if they are dependencies. - */ - buildExcludes?: string[]; - - /** - * Controls amount of parallelism in some build steps. - */ - parallelism?: number; - - /** - * If set, creates a skeleton tarball that contains all package.json files - * with the same structure as the workspace dir. - */ - skeleton?: 'skeleton.tar' | 'skeleton.tar.gz'; -}; - -/** - * Uses `yarn pack` to package local packages and unpacks them into a dist workspace. - * The target workspace will end up containing dist version of each package and - * will be suitable for packaging e.g. into a docker image. - * - * This creates a structure that is functionally similar to if the packages where - * installed from npm, but uses Yarn workspaces to link to them at runtime. - */ -export async function createDistWorkspace( - packageNames: string[], - options: Options = {}, -) { - const targetDir = - options.targetDir ?? - (await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace'))); - - const { packages } = await getPackages(paths.targetDir); - const packageGraph = PackageGraph.fromPackages(packages); - const targetNames = packageGraph.collectPackageNames(packageNames, node => { - // Don't include dependencies of packages that are marked as bundled - if (node.packageJson.bundled) { - return undefined; - } - - return node.publishedLocalDependencies.keys(); - }); - const targets = Array.from(targetNames).map(name => packageGraph.get(name)!); - - if (options.buildDependencies) { - const exclude = options.buildExcludes ?? []; - - const toBuild = targets.filter(target => !exclude.includes(target.name)); - if (toBuild.length > 0) { - const scopeArgs = toBuild.flatMap(target => ['--scope', target.name]); - const lernaArgs = - options.parallelism && Number.isInteger(options.parallelism) - ? ['--concurrency', options.parallelism.toString()] - : []; - - await run('yarn', ['lerna', ...lernaArgs, 'run', ...scopeArgs, 'build'], { - cwd: paths.targetRoot, - }); - } - } - - await moveToDistWorkspace(targetDir, targets); - - const files: FileEntry[] = options.files ?? ['yarn.lock', 'package.json']; - - for (const file of files) { - const src = typeof file === 'string' ? file : file.src; - const dest = typeof file === 'string' ? file : file.dest; - await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest)); - } - - if (options.skeleton) { - const skeletonFiles = targets.map(target => { - const dir = relativePath(paths.targetRoot, target.dir); - return joinPath(dir, 'package.json'); - }); - - await tar.create( - { - file: resolvePath(targetDir, options.skeleton), - cwd: targetDir, - portable: true, - noMtime: true, - gzip: options.skeleton.endsWith('.gz'), - } as CreateOptions & { noMtime: boolean }, - skeletonFiles, - ); - } - - return targetDir; -} - -async function moveToDistWorkspace( - workspaceDir: string, - localPackages: PackageGraphNode[], -): Promise { - async function pack(target: PackageGraphNode, archive: string) { - console.log(`Repacking ${target.name} into dist workspace`); - const archivePath = resolvePath(workspaceDir, archive); - - await run('yarn', ['pack', '--filename', archivePath], { - cwd: target.dir, - }); - // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed - if (target.packageJson?.scripts?.postpack) { - await run('yarn', ['postpack'], { cwd: target.dir }); - } - - const outputDir = relativePath(paths.targetRoot, target.dir); - const absoluteOutputPath = resolvePath(workspaceDir, outputDir); - await fs.ensureDir(absoluteOutputPath); - - await tar.extract({ - file: archivePath, - cwd: absoluteOutputPath, - strip: 1, - }); - await fs.remove(archivePath); - - // We remove the dependencies from package.json of packages that are marked - // as bundled, so that yarn doesn't try to install them. - if (target.packageJson.bundled) { - const pkgJson = await fs.readJson( - resolvePath(absoluteOutputPath, 'package.json'), - ); - delete pkgJson.dependencies; - delete pkgJson.devDependencies; - delete pkgJson.peerDependencies; - delete pkgJson.optionalDependencies; - - await fs.writeJson( - resolvePath(absoluteOutputPath, 'package.json'), - pkgJson, - { - spaces: 2, - }, - ); - } - } - - const unsafePackages = localPackages.filter(p => - UNSAFE_PACKAGES.includes(p.name), - ); - const safePackages = localPackages.filter( - p => !UNSAFE_PACKAGES.includes(p.name), - ); - - // The unsafe package are packed first one by one in order to avoid race conditions - // where the CLI is being executed with broken dependencies. - for (const target of unsafePackages) { - await pack(target, `temp-package.tgz`); - } - - // Repacking in parallel is much faster and safe for all packages outside of the Backstage repo - await Promise.all( - safePackages.map(async (target, index) => - pack(target, `temp-package-${index}.tgz`), - ), - ); -} +export { createDistWorkspace } from './createDistWorkspace'; diff --git a/yarn.lock b/yarn.lock index f1bce4f4fa..abfd54f3c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5761,6 +5761,11 @@ resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== +"@types/npm-packlist@^1.1.2": + version "1.1.2" + resolved "https://registry.npmjs.org/@types/npm-packlist/-/npm-packlist-1.1.2.tgz#285978c9023ce68fa0641ca606c7c3b7b0e851c5" + integrity sha512-9NYoEH87t90e6dkaQOuUTY/R1xUE0a67sXzJBuAB+b+/z4FysHFD19g/O154ToGjyWqKYkezVUtuBdtfd4hyfw== + "@types/nunjucks@^3.1.4": version "3.2.1" resolved "https://registry.npmjs.org/@types/nunjucks/-/nunjucks-3.2.1.tgz#02a3ade3dc4d3950029c6466a4034565dba7cf8c" @@ -13545,6 +13550,13 @@ ignore-walk@^3.0.1, ignore-walk@^3.0.3: dependencies: minimatch "^3.0.4" +ignore-walk@^4.0.1: + version "4.0.1" + resolved "https://registry.npmjs.org/ignore-walk/-/ignore-walk-4.0.1.tgz#fc840e8346cf88a3a9380c5b17933cd8f4d39fa3" + integrity sha512-rzDQLaW4jQbh2YrOFlJdCtX8qgJTehFRYiUB2r1osqTeDzV/3+Jh8fz1oAPzUThf3iku8Ds4IDqawI5d8mUiQw== + dependencies: + minimatch "^3.0.4" + ignore@^3.3.5: version "3.3.10" resolved "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043" @@ -18012,6 +18024,16 @@ npm-packlist@^2.1.4: npm-bundled "^1.1.1" npm-normalize-package-bin "^1.0.1" +npm-packlist@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/npm-packlist/-/npm-packlist-3.0.0.tgz#0370df5cfc2fcc8f79b8f42b37798dd9ee32c2a9" + integrity sha512-L/cbzmutAwII5glUcf2DBRNY/d0TFd4e/FnaZigJV6JD85RHZXJFGwCndjMWiiViiWSsWt3tiOLpI3ByTnIdFQ== + dependencies: + glob "^7.1.6" + ignore-walk "^4.0.1" + npm-bundled "^1.1.1" + npm-normalize-package-bin "^1.0.1" + npm-pick-manifest@^6.0.0: version "6.1.0" resolved "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-6.1.0.tgz#2befed87b0fce956790f62d32afb56d7539c022a"