From 2375ef7c3c5f63be60090cf010540b00118f75d0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 30 Jan 2022 19:41:44 +0100 Subject: [PATCH 1/7] cli: packager, rename index to createDistWorkspace Signed-off-by: Patrik Oldsberg --- .../cli/src/lib/packager/{index.ts => createDistWorkspace.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/cli/src/lib/packager/{index.ts => createDistWorkspace.ts} (100%) diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/createDistWorkspace.ts similarity index 100% rename from packages/cli/src/lib/packager/index.ts rename to packages/cli/src/lib/packager/createDistWorkspace.ts From 46855aaffa7f958518410b933632c6594bc09fbd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 30 Jan 2022 19:42:00 +0100 Subject: [PATCH 2/7] cli: packager, add back index Signed-off-by: Patrik Oldsberg --- packages/cli/src/lib/packager/index.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/cli/src/lib/packager/index.ts diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts new file mode 100644 index 0000000000..75f3fdf71d --- /dev/null +++ b/packages/cli/src/lib/packager/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createDistWorkspace } from './createDistWorkspace'; From c039c184c570b6fb7b6c5de6c7946f4078cc1fa9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 29 Jan 2022 19:10:03 +0100 Subject: [PATCH 3/7] cli: update createDistWorkspace to inline builds when possible Signed-off-by: Patrik Oldsberg --- .../src/lib/packager/createDistWorkspace.ts | 67 +++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/lib/packager/createDistWorkspace.ts b/packages/cli/src/lib/packager/createDistWorkspace.ts index 40af21bc5e..bcbcdb29cc 100644 --- a/packages/cli/src/lib/packager/createDistWorkspace.ts +++ b/packages/cli/src/lib/packager/createDistWorkspace.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import chalk from 'chalk'; import fs from 'fs-extra'; import { join as joinPath, @@ -28,8 +29,14 @@ import { dependencies as cliDependencies, devDependencies as cliDevDependencies, } from '../../../package.json'; -import { getPackages } from '@manypkg/get-packages'; import { PackageGraph, PackageGraphNode } from '../monorepo'; +import { + BuildOptions, + buildPackages, + getOutputsForRole, + Output, +} from '../builder'; +import { copyPackageDist } from './copyPackageDist'; // These packages aren't safe to pack in parallel since the CLI depends on them const UNSAFE_PACKAGES = [ @@ -95,7 +102,7 @@ export async function createDistWorkspace( options.targetDir ?? (await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace'))); - const { packages } = await getPackages(paths.targetDir); + const packages = await PackageGraph.listTargetPackages(); const packageGraph = PackageGraph.fromPackages(packages); const targetNames = packageGraph.collectPackageNames(packageNames, node => { // Don't include dependencies of packages that are marked as bundled @@ -110,9 +117,59 @@ export async function createDistWorkspace( if (options.buildDependencies) { const exclude = options.buildExcludes ?? []; - const toBuild = targets.filter(target => !exclude.includes(target.name)); - if (toBuild.length > 0) { - const scopeArgs = toBuild.flatMap(target => ['--scope', target.name]); + const toBuild = new Set( + targets.map(_ => _.name).filter(name => !exclude.includes(name)), + ); + + const standardBuilds = new Array(); + const customBuild = new Array(); + + for (const pkg of packages) { + if (!toBuild.has(pkg.packageJson.name)) { + continue; + } + const role = pkg.packageJson.backstage?.role; + if (!role) { + console.warn(`Ignored ${pkg.packageJson.name} because it has no role`); + customBuild.push(pkg.packageJson.name); + continue; + } + + const buildScript = pkg.packageJson.scripts?.build; + if (!buildScript) { + customBuild.push(pkg.packageJson.name); + continue; + } + + if (!buildScript.startsWith('backstage-cli script build')) { + console.warn( + `Ignored ${pkg.packageJson.name} because it has a custom build script, '${buildScript}'`, + ); + customBuild.push(pkg.packageJson.name); + continue; + } + + const outputs = getOutputsForRole(role); + + // No need to build and include types in the production runtime + outputs.delete(Output.types); + + if (outputs.size > 0) { + standardBuilds.push({ + targetDir: pkg.dir, + outputs: outputs, + logPrefix: `${chalk.cyan(relativePath(paths.targetRoot, pkg.dir))}: `, + // No need to detect these for the backend builds, we assume no minification or types + minify: false, + useApiExtractor: false, + }); + } + } + + await buildPackages(standardBuilds); + + if (customBuild.length > 0) { + const scopeArgs = customBuild.flatMap(name => ['--scope', name]); const lernaArgs = options.parallelism && Number.isInteger(options.parallelism) ? ['--concurrency', options.parallelism.toString()] From a41f50f9705b58e7e86255cebaa2644595efc3aa Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 29 Jan 2022 19:01:19 +0100 Subject: [PATCH 4/7] cli: added utility for copying package dist files Signed-off-by: Patrik Oldsberg --- packages/cli/package.json | 2 + .../cli/src/lib/packager/copyPackageDist.ts | 88 +++++++++++++++++++ yarn.lock | 22 +++++ 3 files changed, 112 insertions(+) create mode 100644 packages/cli/src/lib/packager/copyPackageDist.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 41cefe6c53..0f6c521ec4 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -86,6 +86,7 @@ "lodash": "^4.17.21", "minimatch": "3.0.4", "mini-css-extract-plugin": "^2.4.2", + "npm-packlist": "^3.0.0", "node-libs-browser": "^2.2.1", "ora": "^5.3.0", "postcss": "^8.1.0", @@ -132,6 +133,7 @@ "@types/minimatch": "^3.0.5", "@types/mock-fs": "^4.13.0", "@types/node": "^14.14.32", + "@types/npm-packlist": "^1.1.2", "@types/recursive-readdir": "^2.2.0", "@types/rollup-plugin-peer-deps-external": "^2.2.0", "@types/rollup-plugin-postcss": "^2.0.0", diff --git a/packages/cli/src/lib/packager/copyPackageDist.ts b/packages/cli/src/lib/packager/copyPackageDist.ts new file mode 100644 index 0000000000..03ac361006 --- /dev/null +++ b/packages/cli/src/lib/packager/copyPackageDist.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import npmPackList from 'npm-packlist'; +import { join as joinPath, resolve as resolvePath } from 'path'; + +const SKIPPED_KEYS = ['access', 'registry', 'tag', 'alphaTypes', 'betaTypes']; + +// Writes e.g. alpha/package.json +async function writeReleaseStageEntrypoint( + pkg: any, + stage: 'alpha' | 'beta', + targetDir: string, +) { + await fs.ensureDir(resolvePath(targetDir, stage)); + await fs.writeJson( + resolvePath(targetDir, stage, 'package.json'), + { + name: pkg.name, + version: pkg.version, + main: (pkg.publishConfig.main || pkg.main) && '..', + module: (pkg.publishConfig.module || pkg.module) && '..', + browser: (pkg.publishConfig.browser || pkg.browser) && '..', + types: joinPath('..', pkg.publishConfig[`${stage}Types`]), + }, + { encoding: 'utf8', spaces: 2 }, + ); +} + +export async function copyPackageDist(packageDir: string, targetDir: string) { + const pkgPath = resolvePath(packageDir, 'package.json'); + const pkgContent = await fs.readFile(pkgPath, 'utf8'); + const pkg = JSON.parse(pkgContent); + + const publishConfig = pkg.publishConfig ?? {}; + for (const key of Object.keys(publishConfig)) { + if (!SKIPPED_KEYS.includes(key)) { + pkg[key] = publishConfig[key]; + } + } + + // We remove the dependencies from package.json of packages that are marked + // as bundled, so that yarn doesn't try to install them. + if (pkg.bundled) { + delete pkg.dependencies; + delete pkg.devDependencies; + delete pkg.peerDependencies; + delete pkg.optionalDependencies; + } + + // Write the modified package.json so that the file listing is correct + await fs.writeJson(pkgPath, pkg, { encoding: 'utf8', spaces: 2 }); + + // Lists all dist files, respecting .npmignore, files field in package.json, etc. + const filePaths = await npmPackList({ path: packageDir }); + + await fs.ensureDir(targetDir); + for (const filePath of filePaths.sort()) { + await fs.copy( + resolvePath(packageDir, filePath), + resolvePath(targetDir, filePath), + ); + } + + if (publishConfig.alphaTypes) { + await writeReleaseStageEntrypoint(pkg, 'alpha', targetDir); + } + if (publishConfig.betaTypes) { + await writeReleaseStageEntrypoint(pkg, 'beta', targetDir); + } + + // Restore package.json + await fs.writeFile(pkgPath, pkgContent, 'utf8'); +} diff --git a/yarn.lock b/yarn.lock index f1bce4f4fa..abfd54f3c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5761,6 +5761,11 @@ resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== +"@types/npm-packlist@^1.1.2": + version "1.1.2" + resolved "https://registry.npmjs.org/@types/npm-packlist/-/npm-packlist-1.1.2.tgz#285978c9023ce68fa0641ca606c7c3b7b0e851c5" + integrity sha512-9NYoEH87t90e6dkaQOuUTY/R1xUE0a67sXzJBuAB+b+/z4FysHFD19g/O154ToGjyWqKYkezVUtuBdtfd4hyfw== + "@types/nunjucks@^3.1.4": version "3.2.1" resolved "https://registry.npmjs.org/@types/nunjucks/-/nunjucks-3.2.1.tgz#02a3ade3dc4d3950029c6466a4034565dba7cf8c" @@ -13545,6 +13550,13 @@ ignore-walk@^3.0.1, ignore-walk@^3.0.3: dependencies: minimatch "^3.0.4" +ignore-walk@^4.0.1: + version "4.0.1" + resolved "https://registry.npmjs.org/ignore-walk/-/ignore-walk-4.0.1.tgz#fc840e8346cf88a3a9380c5b17933cd8f4d39fa3" + integrity sha512-rzDQLaW4jQbh2YrOFlJdCtX8qgJTehFRYiUB2r1osqTeDzV/3+Jh8fz1oAPzUThf3iku8Ds4IDqawI5d8mUiQw== + dependencies: + minimatch "^3.0.4" + ignore@^3.3.5: version "3.3.10" resolved "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043" @@ -18012,6 +18024,16 @@ npm-packlist@^2.1.4: npm-bundled "^1.1.1" npm-normalize-package-bin "^1.0.1" +npm-packlist@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/npm-packlist/-/npm-packlist-3.0.0.tgz#0370df5cfc2fcc8f79b8f42b37798dd9ee32c2a9" + integrity sha512-L/cbzmutAwII5glUcf2DBRNY/d0TFd4e/FnaZigJV6JD85RHZXJFGwCndjMWiiViiWSsWt3tiOLpI3ByTnIdFQ== + dependencies: + glob "^7.1.6" + ignore-walk "^4.0.1" + npm-bundled "^1.1.1" + npm-normalize-package-bin "^1.0.1" + npm-pick-manifest@^6.0.0: version "6.1.0" resolved "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-6.1.0.tgz#2befed87b0fce956790f62d32afb56d7539c022a" From f65550a08d85dfea1344cfb9176e7403957371b3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 29 Jan 2022 19:10:33 +0100 Subject: [PATCH 5/7] cli: update createDistWorkspace to use dist move utility when possible Signed-off-by: Patrik Oldsberg --- .../src/lib/packager/createDistWorkspace.ts | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/lib/packager/createDistWorkspace.ts b/packages/cli/src/lib/packager/createDistWorkspace.ts index bcbcdb29cc..90daa4bfec 100644 --- a/packages/cli/src/lib/packager/createDistWorkspace.ts +++ b/packages/cli/src/lib/packager/createDistWorkspace.ts @@ -23,6 +23,7 @@ import { } from 'path'; import { tmpdir } from 'os'; import tar, { CreateOptions } from 'tar'; +import partition from 'lodash/partition'; import { paths } from '../paths'; import { run } from '../run'; import { @@ -212,10 +213,33 @@ export async function createDistWorkspace( return targetDir; } +const FAST_PACK_SCRIPTS = [ + undefined, + 'backstage-cli prepack', + 'backstage-cli script prepack', +]; + async function moveToDistWorkspace( workspaceDir: string, localPackages: PackageGraphNode[], ): Promise { + const [fastPackPackages, slowPackPackages] = partition(localPackages, pkg => + FAST_PACK_SCRIPTS.includes(pkg.packageJson.scripts?.prepack), + ); + + // New an improved flow where we avoid calling `yarn pack` + await Promise.all( + fastPackPackages.map(async target => { + console.log(`Moving ${target.name} into dist workspace`); + + const outputDir = relativePath(paths.targetRoot, target.dir); + const absoluteOutputPath = resolvePath(workspaceDir, outputDir); + await copyPackageDist(target.dir, absoluteOutputPath); + }), + ); + + // Old flow is below, which calls `yarn pack` and extracts the tarball + async function pack(target: PackageGraphNode, archive: string) { console.log(`Repacking ${target.name} into dist workspace`); const archivePath = resolvePath(workspaceDir, archive); @@ -260,12 +284,9 @@ async function moveToDistWorkspace( } } - const unsafePackages = localPackages.filter(p => + const [unsafePackages, safePackages] = partition(slowPackPackages, p => UNSAFE_PACKAGES.includes(p.name), ); - const safePackages = localPackages.filter( - p => !UNSAFE_PACKAGES.includes(p.name), - ); // The unsafe package are packed first one by one in order to avoid race conditions // where the CLI is being executed with broken dependencies. From b906f98119eb6e0d9e2fef204fdc3ede0b0c8d83 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 30 Jan 2022 19:36:40 +0100 Subject: [PATCH 6/7] changesets: add changeset for dist workspace improvements Signed-off-by: Patrik Oldsberg --- .changeset/tasty-spoons-beg.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tasty-spoons-beg.md diff --git a/.changeset/tasty-spoons-beg.md b/.changeset/tasty-spoons-beg.md new file mode 100644 index 0000000000..651c70e03c --- /dev/null +++ b/.changeset/tasty-spoons-beg.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Rather than calling `yarn pack`, the `build-workspace` and `backend-bundle` commands now move files directly whenever possible. This cuts out several `yarn` invocations and speeds the packing process up by several orders of magnitude. From 51fe2055df5f2ce2400111aba4fffde9b9683a8c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 7 Feb 2022 18:33:53 +0100 Subject: [PATCH 7/7] cli: fix a spelling Signed-off-by: Patrik Oldsberg --- packages/cli/src/lib/packager/createDistWorkspace.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/lib/packager/createDistWorkspace.ts b/packages/cli/src/lib/packager/createDistWorkspace.ts index 90daa4bfec..15ccda30c2 100644 --- a/packages/cli/src/lib/packager/createDistWorkspace.ts +++ b/packages/cli/src/lib/packager/createDistWorkspace.ts @@ -92,7 +92,7 @@ type Options = { * The target workspace will end up containing dist version of each package and * will be suitable for packaging e.g. into a docker image. * - * This creates a structure that is functionally similar to if the packages where + * This creates a structure that is functionally similar to if the packages were * installed from npm, but uses Yarn workspaces to link to them at runtime. */ export async function createDistWorkspace(