paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #9259 from backstage/rugvip/role-pack

Browse Source 
cli: avoid using `yarn pack` when creating dist workspace
This commit is contained in:
Patrik Oldsberg
2022-02-08 09:25:40 +01:00
committed by GitHub
parent d9ac082bef 51fe2055df
commit b0530b60a0
6 changed files with 422 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/tasty-spoons-beg.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/cli': patch
---
Rather than calling `yarn pack`, the `build-workspace` and `backend-bundle` commands now move files directly whenever possible. This cuts out several `yarn` invocations and speeds the packing process up by several orders of magnitude.
packages/cli/package.json
+2
View File
@@ -86,6 +86,7 @@
"lodash": "^4.17.21",
"minimatch": "3.0.4",
"mini-css-extract-plugin": "^2.4.2",
"npm-packlist": "^3.0.0",
"node-libs-browser": "^2.2.1",
"ora": "^5.3.0",
"postcss": "^8.1.0",
@@ -132,6 +133,7 @@
"@types/minimatch": "^3.0.5",
"@types/mock-fs": "^4.13.0",
"@types/node": "^14.14.32",
"@types/npm-packlist": "^1.1.2",
"@types/recursive-readdir": "^2.2.0",
"@types/rollup-plugin-peer-deps-external": "^2.2.0",
"@types/rollup-plugin-postcss": "^2.0.0",
packages/cli/src/lib/packager/copyPackageDist.ts
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import npmPackList from 'npm-packlist';
import { join as joinPath, resolve as resolvePath } from 'path';
const SKIPPED_KEYS = ['access', 'registry', 'tag', 'alphaTypes', 'betaTypes'];
// Writes e.g. alpha/package.json
async function writeReleaseStageEntrypoint(
pkg: any,
stage: 'alpha' | 'beta',
targetDir: string,
) {
await fs.ensureDir(resolvePath(targetDir, stage));
await fs.writeJson(
resolvePath(targetDir, stage, 'package.json'),
{
name: pkg.name,
version: pkg.version,
main: (pkg.publishConfig.main || pkg.main) && '..',
module: (pkg.publishConfig.module || pkg.module) && '..',
browser: (pkg.publishConfig.browser || pkg.browser) && '..',
types: joinPath('..', pkg.publishConfig[`${stage}Types`]),
},
{ encoding: 'utf8', spaces: 2 },
);
}
export async function copyPackageDist(packageDir: string, targetDir: string) {
const pkgPath = resolvePath(packageDir, 'package.json');
const pkgContent = await fs.readFile(pkgPath, 'utf8');
const pkg = JSON.parse(pkgContent);
const publishConfig = pkg.publishConfig ?? {};
for (const key of Object.keys(publishConfig)) {
if (!SKIPPED_KEYS.includes(key)) {
pkg[key] = publishConfig[key];
}
}
// We remove the dependencies from package.json of packages that are marked
// as bundled, so that yarn doesn't try to install them.
if (pkg.bundled) {
delete pkg.dependencies;
delete pkg.devDependencies;
delete pkg.peerDependencies;
delete pkg.optionalDependencies;
}
// Write the modified package.json so that the file listing is correct
await fs.writeJson(pkgPath, pkg, { encoding: 'utf8', spaces: 2 });
// Lists all dist files, respecting .npmignore, files field in package.json, etc.
const filePaths = await npmPackList({ path: packageDir });
await fs.ensureDir(targetDir);
for (const filePath of filePaths.sort()) {
await fs.copy(
resolvePath(packageDir, filePath),
resolvePath(targetDir, filePath),
);
}
if (publishConfig.alphaTypes) {
await writeReleaseStageEntrypoint(pkg, 'alpha', targetDir);
}
if (publishConfig.betaTypes) {
await writeReleaseStageEntrypoint(pkg, 'beta', targetDir);
}
// Restore package.json
await fs.writeFile(pkgPath, pkgContent, 'utf8');
}
packages/cli/src/lib/packager/createDistWorkspace.ts
+303
View File
@@ -0,0 +1,303 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import chalk from 'chalk';
import fs from 'fs-extra';
import {
join as joinPath,
resolve as resolvePath,
relative as relativePath,
} from 'path';
import { tmpdir } from 'os';
import tar, { CreateOptions } from 'tar';
import partition from 'lodash/partition';
import { paths } from '../paths';
import { run } from '../run';
import {
dependencies as cliDependencies,
devDependencies as cliDevDependencies,
} from '../../../package.json';
import { PackageGraph, PackageGraphNode } from '../monorepo';
import {
BuildOptions,
buildPackages,
getOutputsForRole,
Output,
} from '../builder';
import { copyPackageDist } from './copyPackageDist';
// These packages aren't safe to pack in parallel since the CLI depends on them
const UNSAFE_PACKAGES = [
...Object.keys(cliDependencies),
...Object.keys(cliDevDependencies),
];
type FileEntry =
| string
| {
src: string;
dest: string;
};
type Options = {
/**
* Target directory for the dist workspace, defaults to a temporary directory
*/
targetDir?: string;
/**
* Files to copy into the target workspace.
*
* Defaults to ['yarn.lock', 'package.json'].
*/
files?: FileEntry[];
/**
* If set to true, the target packages are built before they are packaged into the workspace.
*/
buildDependencies?: boolean;
/**
* When `buildDependencies` is set, this list of packages will not be built even if they are dependencies.
*/
buildExcludes?: string[];
/**
* Controls amount of parallelism in some build steps.
*/
parallelism?: number;
/**
* If set, creates a skeleton tarball that contains all package.json files
* with the same structure as the workspace dir.
*/
skeleton?: 'skeleton.tar' | 'skeleton.tar.gz';
};
/**
* Uses `yarn pack` to package local packages and unpacks them into a dist workspace.
* The target workspace will end up containing dist version of each package and
* will be suitable for packaging e.g. into a docker image.
*
* This creates a structure that is functionally similar to if the packages were
* installed from npm, but uses Yarn workspaces to link to them at runtime.
*/
export async function createDistWorkspace(
packageNames: string[],
options: Options = {},
) {
const targetDir =
options.targetDir ??
(await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace')));
const packages = await PackageGraph.listTargetPackages();
const packageGraph = PackageGraph.fromPackages(packages);
const targetNames = packageGraph.collectPackageNames(packageNames, node => {
// Don't include dependencies of packages that are marked as bundled
if (node.packageJson.bundled) {
return undefined;
}
return node.publishedLocalDependencies.keys();
});
const targets = Array.from(targetNames).map(name => packageGraph.get(name)!);
if (options.buildDependencies) {
const exclude = options.buildExcludes ?? [];
const toBuild = new Set(
targets.map(_ => _.name).filter(name => !exclude.includes(name)),
);
const standardBuilds = new Array<BuildOptions>();
const customBuild = new Array<string>();
for (const pkg of packages) {
if (!toBuild.has(pkg.packageJson.name)) {
continue;
}
const role = pkg.packageJson.backstage?.role;
if (!role) {
console.warn(`Ignored ${pkg.packageJson.name} because it has no role`);
customBuild.push(pkg.packageJson.name);
continue;
}
const buildScript = pkg.packageJson.scripts?.build;
if (!buildScript) {
customBuild.push(pkg.packageJson.name);
continue;
}
if (!buildScript.startsWith('backstage-cli script build')) {
console.warn(
`Ignored ${pkg.packageJson.name} because it has a custom build script, '${buildScript}'`,
);
customBuild.push(pkg.packageJson.name);
continue;
}
const outputs = getOutputsForRole(role);
// No need to build and include types in the production runtime
outputs.delete(Output.types);
if (outputs.size > 0) {
standardBuilds.push({
targetDir: pkg.dir,
outputs: outputs,
logPrefix: `${chalk.cyan(relativePath(paths.targetRoot, pkg.dir))}: `,
// No need to detect these for the backend builds, we assume no minification or types
minify: false,
useApiExtractor: false,
});
}
}
await buildPackages(standardBuilds);
if (customBuild.length > 0) {
const scopeArgs = customBuild.flatMap(name => ['--scope', name]);
const lernaArgs =
options.parallelism && Number.isInteger(options.parallelism)
? ['--concurrency', options.parallelism.toString()]
: [];
await run('yarn', ['lerna', ...lernaArgs, 'run', ...scopeArgs, 'build'], {
cwd: paths.targetRoot,
});
}
}
await moveToDistWorkspace(targetDir, targets);
const files: FileEntry[] = options.files ?? ['yarn.lock', 'package.json'];
for (const file of files) {
const src = typeof file === 'string' ? file : file.src;
const dest = typeof file === 'string' ? file : file.dest;
await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest));
}
if (options.skeleton) {
const skeletonFiles = targets.map(target => {
const dir = relativePath(paths.targetRoot, target.dir);
return joinPath(dir, 'package.json');
});
await tar.create(
{
file: resolvePath(targetDir, options.skeleton),
cwd: targetDir,
portable: true,
noMtime: true,
gzip: options.skeleton.endsWith('.gz'),
} as CreateOptions & { noMtime: boolean },
skeletonFiles,
);
}
return targetDir;
}
const FAST_PACK_SCRIPTS = [
undefined,
'backstage-cli prepack',
'backstage-cli script prepack',
];
async function moveToDistWorkspace(
workspaceDir: string,
localPackages: PackageGraphNode[],
): Promise<void> {
const [fastPackPackages, slowPackPackages] = partition(localPackages, pkg =>
FAST_PACK_SCRIPTS.includes(pkg.packageJson.scripts?.prepack),
);
// New an improved flow where we avoid calling `yarn pack`
await Promise.all(
fastPackPackages.map(async target => {
console.log(`Moving ${target.name} into dist workspace`);
const outputDir = relativePath(paths.targetRoot, target.dir);
const absoluteOutputPath = resolvePath(workspaceDir, outputDir);
await copyPackageDist(target.dir, absoluteOutputPath);
}),
);
// Old flow is below, which calls `yarn pack` and extracts the tarball
async function pack(target: PackageGraphNode, archive: string) {
console.log(`Repacking ${target.name} into dist workspace`);
const archivePath = resolvePath(workspaceDir, archive);
await run('yarn', ['pack', '--filename', archivePath], {
cwd: target.dir,
});
// TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed
if (target.packageJson?.scripts?.postpack) {
await run('yarn', ['postpack'], { cwd: target.dir });
}
const outputDir = relativePath(paths.targetRoot, target.dir);
const absoluteOutputPath = resolvePath(workspaceDir, outputDir);
await fs.ensureDir(absoluteOutputPath);
await tar.extract({
file: archivePath,
cwd: absoluteOutputPath,
strip: 1,
});
await fs.remove(archivePath);
// We remove the dependencies from package.json of packages that are marked
// as bundled, so that yarn doesn't try to install them.
if (target.packageJson.bundled) {
const pkgJson = await fs.readJson(
resolvePath(absoluteOutputPath, 'package.json'),
);
delete pkgJson.dependencies;
delete pkgJson.devDependencies;
delete pkgJson.peerDependencies;
delete pkgJson.optionalDependencies;
await fs.writeJson(
resolvePath(absoluteOutputPath, 'package.json'),
pkgJson,
{
spaces: 2,
},
);
}
}
const [unsafePackages, safePackages] = partition(slowPackPackages, p =>
UNSAFE_PACKAGES.includes(p.name),
);
// The unsafe package are packed first one by one in order to avoid race conditions
// where the CLI is being executed with broken dependencies.
for (const target of unsafePackages) {
await pack(target, `temp-package.tgz`);
}
// Repacking in parallel is much faster and safe for all packages outside of the Backstage repo
await Promise.all(
safePackages.map(async (target, index) =>
pack(target, `temp-package-${index}.tgz`),
),
);
}
packages/cli/src/lib/packager/index.ts
+2 -210
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 The Backstage Authors
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,212 +14,4 @@
* limitations under the License.
*/
import fs from 'fs-extra';
import {
join as joinPath,
resolve as resolvePath,
relative as relativePath,
} from 'path';
import { tmpdir } from 'os';
import tar, { CreateOptions } from 'tar';
import { paths } from '../paths';
import { run } from '../run';
import {
dependencies as cliDependencies,
devDependencies as cliDevDependencies,
} from '../../../package.json';
import { getPackages } from '@manypkg/get-packages';
import { PackageGraph, PackageGraphNode } from '../monorepo';
// These packages aren't safe to pack in parallel since the CLI depends on them
const UNSAFE_PACKAGES = [
...Object.keys(cliDependencies),
...Object.keys(cliDevDependencies),
];
type FileEntry =
| string
| {
src: string;
dest: string;
};
type Options = {
/**
* Target directory for the dist workspace, defaults to a temporary directory
*/
targetDir?: string;
/**
* Files to copy into the target workspace.
*
* Defaults to ['yarn.lock', 'package.json'].
*/
files?: FileEntry[];
/**
* If set to true, the target packages are built before they are packaged into the workspace.
*/
buildDependencies?: boolean;
/**
* When `buildDependencies` is set, this list of packages will not be built even if they are dependencies.
*/
buildExcludes?: string[];
/**
* Controls amount of parallelism in some build steps.
*/
parallelism?: number;
/**
* If set, creates a skeleton tarball that contains all package.json files
* with the same structure as the workspace dir.
*/
skeleton?: 'skeleton.tar' | 'skeleton.tar.gz';
};
/**
* Uses `yarn pack` to package local packages and unpacks them into a dist workspace.
* The target workspace will end up containing dist version of each package and
* will be suitable for packaging e.g. into a docker image.
*
* This creates a structure that is functionally similar to if the packages where
* installed from npm, but uses Yarn workspaces to link to them at runtime.
*/
export async function createDistWorkspace(
packageNames: string[],
options: Options = {},
) {
const targetDir =
options.targetDir ??
(await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace')));
const { packages } = await getPackages(paths.targetDir);
const packageGraph = PackageGraph.fromPackages(packages);
const targetNames = packageGraph.collectPackageNames(packageNames, node => {
// Don't include dependencies of packages that are marked as bundled
if (node.packageJson.bundled) {
return undefined;
}
return node.publishedLocalDependencies.keys();
});
const targets = Array.from(targetNames).map(name => packageGraph.get(name)!);
if (options.buildDependencies) {
const exclude = options.buildExcludes ?? [];
const toBuild = targets.filter(target => !exclude.includes(target.name));
if (toBuild.length > 0) {
const scopeArgs = toBuild.flatMap(target => ['--scope', target.name]);
const lernaArgs =
options.parallelism && Number.isInteger(options.parallelism)
? ['--concurrency', options.parallelism.toString()]
: [];
await run('yarn', ['lerna', ...lernaArgs, 'run', ...scopeArgs, 'build'], {
cwd: paths.targetRoot,
});
}
}
await moveToDistWorkspace(targetDir, targets);
const files: FileEntry[] = options.files ?? ['yarn.lock', 'package.json'];
for (const file of files) {
const src = typeof file === 'string' ? file : file.src;
const dest = typeof file === 'string' ? file : file.dest;
await fs.copy(paths.resolveTargetRoot(src), resolvePath(targetDir, dest));
}
if (options.skeleton) {
const skeletonFiles = targets.map(target => {
const dir = relativePath(paths.targetRoot, target.dir);
return joinPath(dir, 'package.json');
});
await tar.create(
{
file: resolvePath(targetDir, options.skeleton),
cwd: targetDir,
portable: true,
noMtime: true,
gzip: options.skeleton.endsWith('.gz'),
} as CreateOptions & { noMtime: boolean },
skeletonFiles,
);
}
return targetDir;
}
async function moveToDistWorkspace(
workspaceDir: string,
localPackages: PackageGraphNode[],
): Promise<void> {
async function pack(target: PackageGraphNode, archive: string) {
console.log(`Repacking ${target.name} into dist workspace`);
const archivePath = resolvePath(workspaceDir, archive);
await run('yarn', ['pack', '--filename', archivePath], {
cwd: target.dir,
});
// TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed
if (target.packageJson?.scripts?.postpack) {
await run('yarn', ['postpack'], { cwd: target.dir });
}
const outputDir = relativePath(paths.targetRoot, target.dir);
const absoluteOutputPath = resolvePath(workspaceDir, outputDir);
await fs.ensureDir(absoluteOutputPath);
await tar.extract({
file: archivePath,
cwd: absoluteOutputPath,
strip: 1,
});
await fs.remove(archivePath);
// We remove the dependencies from package.json of packages that are marked
// as bundled, so that yarn doesn't try to install them.
if (target.packageJson.bundled) {
const pkgJson = await fs.readJson(
resolvePath(absoluteOutputPath, 'package.json'),
);
delete pkgJson.dependencies;
delete pkgJson.devDependencies;
delete pkgJson.peerDependencies;
delete pkgJson.optionalDependencies;
await fs.writeJson(
resolvePath(absoluteOutputPath, 'package.json'),
pkgJson,
{
spaces: 2,
},
);
}
}
const unsafePackages = localPackages.filter(p =>
UNSAFE_PACKAGES.includes(p.name),
);
const safePackages = localPackages.filter(
p => !UNSAFE_PACKAGES.includes(p.name),
);
// The unsafe package are packed first one by one in order to avoid race conditions
// where the CLI is being executed with broken dependencies.
for (const target of unsafePackages) {
await pack(target, `temp-package.tgz`);
}
// Repacking in parallel is much faster and safe for all packages outside of the Backstage repo
await Promise.all(
safePackages.map(async (target, index) =>
pack(target, `temp-package-${index}.tgz`),
),
);
}
export { createDistWorkspace } from './createDistWorkspace';
yarn.lock
+22
View File
@@ -5761,6 +5761,11 @@
resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==
"@types/npm-packlist@^1.1.2":
version "1.1.2"
resolved "https://registry.npmjs.org/@types/npm-packlist/-/npm-packlist-1.1.2.tgz#285978c9023ce68fa0641ca606c7c3b7b0e851c5"
integrity sha512-9NYoEH87t90e6dkaQOuUTY/R1xUE0a67sXzJBuAB+b+/z4FysHFD19g/O154ToGjyWqKYkezVUtuBdtfd4hyfw==
"@types/nunjucks@^3.1.4":
version "3.2.1"
resolved "https://registry.npmjs.org/@types/nunjucks/-/nunjucks-3.2.1.tgz#02a3ade3dc4d3950029c6466a4034565dba7cf8c"
@@ -13545,6 +13550,13 @@ ignore-walk@^3.0.1, ignore-walk@^3.0.3:
dependencies:
minimatch "^3.0.4"
ignore-walk@^4.0.1:
version "4.0.1"
resolved "https://registry.npmjs.org/ignore-walk/-/ignore-walk-4.0.1.tgz#fc840e8346cf88a3a9380c5b17933cd8f4d39fa3"
integrity sha512-rzDQLaW4jQbh2YrOFlJdCtX8qgJTehFRYiUB2r1osqTeDzV/3+Jh8fz1oAPzUThf3iku8Ds4IDqawI5d8mUiQw==
dependencies:
minimatch "^3.0.4"
ignore@^3.3.5:
version "3.3.10"
resolved "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043"
@@ -18012,6 +18024,16 @@ npm-packlist@^2.1.4:
npm-bundled "^1.1.1"
npm-normalize-package-bin "^1.0.1"
npm-packlist@^3.0.0:
version "3.0.0"
resolved "https://registry.npmjs.org/npm-packlist/-/npm-packlist-3.0.0.tgz#0370df5cfc2fcc8f79b8f42b37798dd9ee32c2a9"
integrity sha512-L/cbzmutAwII5glUcf2DBRNY/d0TFd4e/FnaZigJV6JD85RHZXJFGwCndjMWiiViiWSsWt3tiOLpI3ByTnIdFQ==
dependencies:
glob "^7.1.6"
ignore-walk "^4.0.1"
npm-bundled "^1.1.1"
npm-normalize-package-bin "^1.0.1"
npm-pick-manifest@^6.0.0:
version "6.1.0"
resolved "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-6.1.0.tgz#2befed87b0fce956790f62d32afb56d7539c022a"