diff --git a/.changeset/silver-knives-change.md b/.changeset/silver-knives-change.md new file mode 100644 index 0000000000..a2bbaf1e0b --- /dev/null +++ b/.changeset/silver-knives-change.md @@ -0,0 +1,6 @@ +--- +'@backstage/cli': patch +'@backstage/backend-common': patch +--- + +Switched to using `@manypkg/get-packages` to list monorepo packages, which provides better support for different kind of monorepo setups. diff --git a/package.json b/package.json index 7d2b7a1882..fe5876dd97 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ }, "version": "1.0.0", "dependencies": { + "@manypkg/get-packages": "^1.1.3", "@microsoft/api-documenter": "^7.13.77", "@microsoft/api-extractor": "^7.19.2", "@microsoft/api-extractor-model": "^7.15.1", diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 7165992607..04a4971e00 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -36,7 +36,7 @@ "@backstage/integration": "^0.7.0", "@backstage/types": "^0.1.1", "@google-cloud/storage": "^5.8.0", - "@lerna/project": "^4.0.0", + "@manypkg/get-packages": "^1.1.3", "@octokit/rest": "^18.5.3", "@types/cors": "^2.8.6", "@types/dockerode": "^3.3.0", diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 399898cfc4..12cc2e3a9d 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -27,6 +27,7 @@ import { } from '@backstage/config-loader'; import { AppConfig, Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; +import { getPackages } from '@manypkg/get-packages'; import { isValidUrl } from './urls'; @@ -194,11 +195,9 @@ export async function loadBackendConfig(options: { // TODO(hhogg): This is fetching _all_ of the packages of the monorepo // in order to find the secrets for redactions, however we only care about // the backend ones, we need to find a way to exclude the frontend packages. - const { Project } = require('@lerna/project'); - const project = new Project(paths.targetDir); - const packages = await project.getPackages(); + const { packages } = await getPackages(paths.targetDir); const schema = await loadConfigSchema({ - dependencies: packages.map((p: any) => p.name), + dependencies: packages.map(p => p.packageJson.name), }); const config = new ObservableConfigProxy(options.logger); diff --git a/packages/cli/package.json b/packages/cli/package.json index e1adc8f0e2..be09563796 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -34,8 +34,7 @@ "@backstage/errors": "^0.1.5", "@backstage/types": "^0.1.1", "@hot-loader/react-dom": "^16.13.0", - "@lerna/package-graph": "^4.0.0", - "@lerna/project": "^4.0.0", + "@manypkg/get-packages": "^1.1.3", "@octokit/request": "^5.4.12", "@rollup/plugin-commonjs": "^21.0.1", "@rollup/plugin-json": "^4.1.0", diff --git a/packages/cli/src/commands/versions/bump.test.ts b/packages/cli/src/commands/versions/bump.test.ts index 377e0d59d2..2dadd06d1d 100644 --- a/packages/cli/src/commands/versions/bump.test.ts +++ b/packages/cli/src/commands/versions/bump.test.ts @@ -19,7 +19,6 @@ import mockFs from 'mock-fs'; import { Command } from 'commander'; import { resolve as resolvePath } from 'path'; import { paths } from '../../lib/paths'; -import { mapDependencies } from '../../lib/versioning'; import * as runObj from '../../lib/run'; import bump, { bumpBackstageJsonVersion } from './bump'; import { withLogCollector } from '@backstage/test-utils'; @@ -85,13 +84,12 @@ describe('bump', () => { }); it('should bump backstage dependencies', async () => { - // Make sure all modules involved in package discovery are in the module cache before we mock fs - await mapDependencies(paths.targetDir, '@backstage/*'); - mockFs({ '/yarn.lock': lockfileMock, - '/lerna.json': JSON.stringify({ - packages: ['packages/*'], + '/package.json': JSON.stringify({ + workspaces: { + packages: ['packages/*'], + }, }), '/packages/a/package.json': JSON.stringify({ name: 'a', @@ -108,7 +106,6 @@ describe('bump', () => { }), }); - paths.targetDir = '/'; jest .spyOn(paths, 'resolveTargetRoot') .mockImplementation((...path) => resolvePath('/', ...path)); @@ -183,8 +180,6 @@ describe('bump', () => { }); it('should bump backstage dependencies and dependencies matching pattern glob', async () => { - // Make sure all modules involved in package discovery are in the module cache before we mock fs - await mapDependencies(paths.targetDir, '@backstage/*'); const customLockfileMock = `${lockfileMock} "@backstage-extra/custom@^1.1.0": version "1.1.0" @@ -212,8 +207,10 @@ describe('bump', () => { `; mockFs({ '/yarn.lock': customLockfileMock, - '/lerna.json': JSON.stringify({ - packages: ['packages/*'], + '/package.json': JSON.stringify({ + workspaces: { + packages: ['packages/*'], + }, }), '/packages/a/package.json': JSON.stringify({ name: 'a', @@ -234,7 +231,6 @@ describe('bump', () => { }), }); - paths.targetDir = '/'; jest .spyOn(paths, 'resolveTargetRoot') .mockImplementation((...path) => resolvePath('/', ...path)); @@ -319,12 +315,12 @@ describe('bump', () => { }); it('should ignore not found packages', async () => { - // Make sure all modules involved in package discovery are in the module cache before we mock fs - await mapDependencies(paths.targetDir, '@backstage/*'); mockFs({ '/yarn.lock': lockfileMockResult, - '/lerna.json': JSON.stringify({ - packages: ['packages/*'], + '/package.json': JSON.stringify({ + workspaces: { + packages: ['packages/*'], + }, }), '/packages/a/package.json': JSON.stringify({ name: 'a', @@ -341,7 +337,6 @@ describe('bump', () => { }), }); - paths.targetDir = '/'; jest .spyOn(paths, 'resolveTargetRoot') .mockImplementation((...path) => resolvePath('/', ...path)); diff --git a/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.test.ts b/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.test.ts index d22b9b3701..450126e6fa 100644 --- a/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.test.ts +++ b/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.test.ts @@ -26,12 +26,18 @@ describe('LinkedPackageResolvePlugin', () => { path.resolve(root, 'repo/node_modules'), [ { - name: 'a', - location: path.resolve(root, 'external-a'), + dir: path.resolve(root, 'external-a'), + packageJson: { + name: 'a', + version: '1.0.0', + }, }, { - name: '@s/b', - location: path.resolve(root, 'external-b'), + dir: path.resolve(root, 'external-b'), + packageJson: { + name: '@s/b', + version: '1.0.0', + }, }, ], ); diff --git a/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts b/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts index 91615eddee..4811afec6f 100644 --- a/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts +++ b/packages/cli/src/lib/bundler/LinkedPackageResolvePlugin.ts @@ -17,7 +17,7 @@ import { resolve as resolvePath } from 'path'; import { WebpackPluginInstance } from 'webpack'; import { isChildPath } from '@backstage/cli-common'; -import { LernaPackage } from './types'; +import { Package } from '@manypkg/get-packages'; // Enables proper resolution of packages when linking in external packages. // Without this the packages would depend on dependencies in the node_modules @@ -25,7 +25,7 @@ import { LernaPackage } from './types'; export class LinkedPackageResolvePlugin implements WebpackPluginInstance { constructor( private readonly targetModules: string, - private readonly packages: LernaPackage[], + private readonly packages: Package[], ) {} apply(resolver: any) { @@ -41,7 +41,7 @@ export class LinkedPackageResolvePlugin implements WebpackPluginInstance { callback: () => void, ) => { const pkg = this.packages.find( - pkge => data.path && isChildPath(pkge.location, data.path), + pkge => data.path && isChildPath(pkge.dir, data.path), ); if (!pkg) { callback(); @@ -51,14 +51,14 @@ export class LinkedPackageResolvePlugin implements WebpackPluginInstance { // pkg here is an external package. We rewrite the context of any imports to resolve // from the location of the package within the node_modules of the target root rather // than the real location of the external package. - const modulesLocation = resolvePath(this.targetModules, pkg.name); + const modulesLocation = resolvePath( + this.targetModules, + pkg.packageJson.name, + ); const newContext = data.context?.issuer ? { ...data.context, - issuer: data.context.issuer.replace( - pkg.location, - modulesLocation, - ), + issuer: data.context.issuer.replace(pkg.dir, modulesLocation), } : data.context; @@ -70,7 +70,7 @@ export class LinkedPackageResolvePlugin implements WebpackPluginInstance { { ...data, context: newContext, - path: data.path && data.path.replace(pkg.location, modulesLocation), + path: data.path && data.path.replace(pkg.dir, modulesLocation), }, `resolve ${data.request} in ${modulesLocation}`, context, diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 7ac230bb0e..35c5be8bb7 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -24,12 +24,13 @@ import { RunScriptWebpackPlugin } from 'run-script-webpack-plugin'; import webpack, { ProvidePlugin } from 'webpack'; import nodeExternals from 'webpack-node-externals'; import { isChildPath } from '@backstage/cli-common'; +import { getPackages } from '@manypkg/get-packages'; import { optimization } from './optimization'; import { Config } from '@backstage/config'; import { BundlingPaths } from './paths'; import { transforms } from './transforms'; import { LinkedPackageResolvePlugin } from './LinkedPackageResolvePlugin'; -import { BundlingOptions, BackendBundlingOptions, LernaPackage } from './types'; +import { BundlingOptions, BackendBundlingOptions } from './types'; import { version } from '../../lib/version'; import { paths as cliPaths } from '../../lib/paths'; import { runPlain } from '../run'; @@ -74,25 +75,17 @@ async function readBuildInfo() { }; } -async function loadLernaPackages(): Promise { - const { Project } = require('@lerna/project'); - const project = new Project(cliPaths.targetDir); - return project.getPackages(); -} - export async function createConfig( paths: BundlingPaths, options: BundlingOptions, ): Promise { const { checksEnabled, isDev, frontendConfig } = options; - const packages = await loadLernaPackages(); const { plugins, loaders } = transforms(options); // Any package that is part of the monorepo but outside the monorepo root dir need // separate resolution logic. - const externalPkgs = packages.filter( - p => !isChildPath(paths.root, p.location), - ); + const { packages } = await getPackages(cliPaths.targetDir); + const externalPkgs = packages.filter(p => !isChildPath(paths.root, p.dir)); const baseUrl = frontendConfig.getString('app.baseUrl'); const validBaseUrl = new URL(baseUrl); @@ -253,14 +246,11 @@ export async function createBackendConfig( const { checksEnabled, isDev } = options; // Find all local monorepo packages and their node_modules, and mark them as external. - const packages = await await loadLernaPackages(); - const localPackageNames = packages.map((p: any) => p.name); - const moduleDirs = packages.map((p: any) => - resolvePath(p.location, 'node_modules'), - ); - const externalPkgs = packages.filter( - p => !isChildPath(paths.root, p.location), - ); // See frontend config + const { packages } = await getPackages(cliPaths.targetDir); + const localPackageNames = packages.map(p => p.packageJson.name); + const moduleDirs = packages.map(p => resolvePath(p.dir, 'node_modules')); + // See frontend config + const externalPkgs = packages.filter(p => !isChildPath(paths.root, p.dir)); const { loaders } = transforms(options); diff --git a/packages/cli/src/lib/bundler/types.ts b/packages/cli/src/lib/bundler/types.ts index 93865bcf7e..eae0452d8d 100644 --- a/packages/cli/src/lib/bundler/types.ts +++ b/packages/cli/src/lib/bundler/types.ts @@ -55,8 +55,3 @@ export type BackendServeOptions = BundlingPathsOptions & { inspectEnabled: boolean; inspectBrkEnabled: boolean; }; - -export type LernaPackage = { - name: string; - location: string; -}; diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 986e409e58..b43e4798b4 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -22,6 +22,8 @@ import { import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; import { isValidUrl } from './urls'; +import { getPackages } from '@manypkg/get-packages'; +import { PackageGraph } from './monorepo'; type Options = { args: string[]; @@ -40,20 +42,27 @@ export async function loadCliConfig(options: Options) { }); // Consider all packages in the monorepo when loading in config - const { Project } = require('@lerna/project'); - const project = new Project(paths.targetDir); - const packages = await project.getPackages(); + const { packages } = await getPackages(paths.targetDir); let localPackageNames; if (options.fromPackage) { if (packages.length) { - localPackageNames = findPackages(packages, options.fromPackage); + const graph = PackageGraph.fromPackages(packages); + localPackageNames = Array.from( + graph.collectPackageNames([options.fromPackage], node => { + // Workaround for Backstage main repo only, since the CLI has some artificial devDependencies + if (node.name === '@backstage/cli') { + return undefined; + } + return node.localDependencies.keys(); + }), + ); } else { // No packages: it means that it's not a monorepo (e.g. standalone plugin) localPackageNames = [options.fromPackage]; } } else { - localPackageNames = packages.map((p: any) => p.name); + localPackageNames = packages.map(p => p.packageJson.name); } const schema = await loadConfigSchema({ @@ -100,34 +109,3 @@ export async function loadCliConfig(options: Options) { throw error; } } - -function findPackages(packages: any[], fromPackage: string): string[] { - const { PackageGraph } = require('@lerna/package-graph'); - - const graph = new PackageGraph(packages); - - const targets = new Set(); - const searchNames = [fromPackage]; - - while (searchNames.length) { - const name = searchNames.pop()!; - - if (targets.has(name)) { - continue; - } - - const node = graph.get(name); - if (!node) { - throw new Error(`Package '${name}' not found`); - } - - targets.add(name); - - // Workaround for Backstage main repo only, since the CLI has some artificial devDependencies - if (name !== '@backstage/cli') { - searchNames.push(...node.localDependencies.keys()); - } - } - - return Array.from(targets); -} diff --git a/packages/cli/src/lib/monorepo/PackageGraph.test.ts b/packages/cli/src/lib/monorepo/PackageGraph.test.ts new file mode 100644 index 0000000000..e8e531bc93 --- /dev/null +++ b/packages/cli/src/lib/monorepo/PackageGraph.test.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getPackages } from '@manypkg/get-packages'; +import { paths } from '../paths'; +import { PackageGraph } from './PackageGraph'; + +const testPackages = [ + { + dir: '/packages/a', + packageJson: { + name: 'a', + version: '1.0.0', + dependencies: { + b: '1.0.0', + }, + devDependencies: { + c: '1.0.0', + }, + }, + }, + { + dir: '/packages/b', + packageJson: { + name: 'b', + version: '1.0.0', + devDependencies: { + c: '1.0.0', + }, + }, + }, + { + dir: '/packages/c', + packageJson: { + name: 'c', + version: '1.0.0', + }, + }, +]; + +describe('PackageGraph', () => { + it('is able to construct a graph from this repo', async () => { + const { packages } = await getPackages(paths.ownDir); + const graph = PackageGraph.fromPackages(packages); + expect(graph.has('@backstage/cli')).toBe(true); + }); + + it('creates a graph', () => { + const graph = PackageGraph.fromPackages(testPackages); + const a = graph.get('a'); + const b = graph.get('b'); + const c = graph.get('c'); + + expect(a).toMatchObject({ + name: 'a', + dir: '/packages/a', + allLocalDependencies: new Map([ + ['b', b], + ['c', c], + ]), + publishedLocalDependencies: new Map([['b', b]]), + localDependencies: new Map([['b', b]]), + localDevDependencies: new Map([['c', c]]), + localOptionalDependencies: new Map(), + }); + expect(b).toMatchObject({ + name: 'b', + dir: '/packages/b', + allLocalDependencies: new Map([['c', c]]), + publishedLocalDependencies: new Map(), + localDependencies: new Map(), + localDevDependencies: new Map([['c', c]]), + localOptionalDependencies: new Map(), + }); + expect(c).toMatchObject({ + name: 'c', + dir: '/packages/c', + allLocalDependencies: new Map(), + publishedLocalDependencies: new Map(), + localDependencies: new Map(), + localDevDependencies: new Map(), + localOptionalDependencies: new Map(), + }); + }); + + it('collects package names', () => { + const graph = PackageGraph.fromPackages(testPackages); + + expect(graph.collectPackageNames(['a'], () => undefined)).toEqual( + new Set(['a']), + ); + expect( + graph.collectPackageNames(['a'], pkg => pkg.localDependencies.keys()), + ).toEqual(new Set(['a', 'b'])); + expect( + graph.collectPackageNames(['a'], pkg => pkg.localDevDependencies.keys()), + ).toEqual(new Set(['a', 'c'])); + expect( + graph.collectPackageNames(['b', 'a'], pkg => + pkg.localDevDependencies.keys(), + ), + ).toEqual(new Set(['b', 'a', 'c'])); + + // Should not get stuck in cycles + expect(graph.collectPackageNames(['a'], () => ['a', 'b', 'c'])).toEqual( + new Set(['b', 'a', 'c']), + ); + + // Throws on unknown packages + expect(() => graph.collectPackageNames(['a'], () => ['unknown'])).toThrow( + `Package 'unknown' not found`, + ); + }); +}); diff --git a/packages/cli/src/lib/monorepo/PackageGraph.ts b/packages/cli/src/lib/monorepo/PackageGraph.ts new file mode 100644 index 0000000000..9860b92ef6 --- /dev/null +++ b/packages/cli/src/lib/monorepo/PackageGraph.ts @@ -0,0 +1,145 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Package } from '@manypkg/get-packages'; + +type PackageJSON = Package['packageJson']; + +export interface ExtendedPackageJSON extends PackageJSON { + scripts?: { + [key: string]: string; + }; + // The `bundled` field is a field known within Backstage, it means + // that the package bundles all of its dependencies in its build output. + bundled?: boolean; +} + +export type PackageGraphNode = { + /** The name of the package */ + name: string; + /** The directory of the package */ + dir: string; + /** The package data of the package itself */ + packageJson: ExtendedPackageJSON; + /** All direct local dependencies of the package */ + allLocalDependencies: Map; + /** All direct local dependencies that will be present in the published package */ + publishedLocalDependencies: Map; + /** Local dependencies */ + localDependencies: Map; + /** Local devDependencies */ + localDevDependencies: Map; + /** Local optionalDependencies */ + localOptionalDependencies: Map; +}; + +export class PackageGraph extends Map { + static fromPackages(packages: Package[]): PackageGraph { + const graph = new PackageGraph(); + + // Add all local packages to the graph + for (const pkg of packages) { + const name = pkg.packageJson.name; + const existingPkg = graph.get(name); + if (existingPkg) { + throw new Error( + `Duplicate package name '${name}' at ${pkg.dir} and ${existingPkg.dir}`, + ); + } + + graph.set(name, { + name, + dir: pkg.dir, + packageJson: pkg.packageJson as ExtendedPackageJSON, + allLocalDependencies: new Map(), + publishedLocalDependencies: new Map(), + localDependencies: new Map(), + localDevDependencies: new Map(), + localOptionalDependencies: new Map(), + }); + } + + // Populate the local dependency structure + for (const node of graph.values()) { + for (const depName of Object.keys(node.packageJson.dependencies || {})) { + const depPkg = graph.get(depName); + if (depPkg) { + node.allLocalDependencies.set(depName, depPkg); + node.publishedLocalDependencies.set(depName, depPkg); + node.localDependencies.set(depName, depPkg); + } + } + for (const depName of Object.keys( + node.packageJson.devDependencies || {}, + )) { + const depPkg = graph.get(depName); + if (depPkg) { + node.allLocalDependencies.set(depName, depPkg); + node.localDevDependencies.set(depName, depPkg); + } + } + for (const depName of Object.keys( + node.packageJson.optionalDependencies || {}, + )) { + const depPkg = graph.get(depName); + if (depPkg) { + node.allLocalDependencies.set(depName, depPkg); + node.publishedLocalDependencies.set(depName, depPkg); + node.localOptionalDependencies.set(depName, depPkg); + } + } + } + + return graph; + } + + /** + * Traverses the package graph and collects a set of package names. + * + * The traversal starts at the provided list names, and continues + * throughout all the names returned by the `collectFn`, which is + * called once for each seen package. + */ + collectPackageNames( + startingPackageNames: string[], + collectFn: (pkg: PackageGraphNode) => Iterable | undefined, + ): Set { + const targets = new Set(); + const searchNames = startingPackageNames.slice(); + + while (searchNames.length) { + const name = searchNames.pop()!; + + if (targets.has(name)) { + continue; + } + + const node = this.get(name); + if (!node) { + throw new Error(`Package '${name}' not found`); + } + + targets.add(name); + + const collected = collectFn(node); + if (collected) { + searchNames.push(...collected); + } + } + + return targets; + } +} diff --git a/packages/cli/src/lib/monorepo/index.ts b/packages/cli/src/lib/monorepo/index.ts new file mode 100644 index 0000000000..aa12c63abc --- /dev/null +++ b/packages/cli/src/lib/monorepo/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { PackageGraph } from './PackageGraph'; +export type { PackageGraphNode } from './PackageGraph'; diff --git a/packages/cli/src/lib/packager/index.ts b/packages/cli/src/lib/packager/index.ts index 4faf904177..c35e2587b9 100644 --- a/packages/cli/src/lib/packager/index.ts +++ b/packages/cli/src/lib/packager/index.ts @@ -29,6 +29,8 @@ import { dependencies as cliDependencies, devDependencies as cliDevDependencies, } from '../../../package.json'; +import { getPackages } from '@manypkg/get-packages'; +import { PackageGraph, PackageGraphNode } from '../monorepo'; // These packages aren't safe to pack in parallel since the CLI depends on them const UNSAFE_PACKAGES = [ @@ -36,14 +38,6 @@ const UNSAFE_PACKAGES = [ ...Object.keys(cliDevDependencies), ]; -type LernaPackage = { - name: string; - private: boolean; - location: string; - scripts: Record; - get(key: string): any; -}; - type FileEntry = | string | { @@ -102,7 +96,17 @@ export async function createDistWorkspace( options.targetDir ?? (await fs.mkdtemp(resolvePath(tmpdir(), 'dist-workspace'))); - const targets = await findTargetPackages(packageNames); + const { packages } = await getPackages(paths.targetDir); + const packageGraph = PackageGraph.fromPackages(packages); + const targetNames = packageGraph.collectPackageNames(packageNames, node => { + // Don't include dependencies of packages that are marked as bundled + if (node.packageJson.bundled) { + return undefined; + } + + return node.publishedLocalDependencies.keys(); + }); + const targets = Array.from(targetNames).map(name => packageGraph.get(name)!); if (options.buildDependencies) { const exclude = options.buildExcludes ?? []; @@ -133,7 +137,7 @@ export async function createDistWorkspace( if (options.skeleton) { const skeletonFiles = targets.map(target => { - const dir = relativePath(paths.targetRoot, target.location); + const dir = relativePath(paths.targetRoot, target.dir); return joinPath(dir, 'package.json'); }); @@ -154,21 +158,21 @@ export async function createDistWorkspace( async function moveToDistWorkspace( workspaceDir: string, - localPackages: LernaPackage[], + localPackages: PackageGraphNode[], ): Promise { - async function pack(target: LernaPackage, archive: string) { + async function pack(target: PackageGraphNode, archive: string) { console.log(`Repacking ${target.name} into dist workspace`); const archivePath = resolvePath(workspaceDir, archive); await run('yarn', ['pack', '--filename', archivePath], { - cwd: target.location, + cwd: target.dir, }); // TODO(Rugvip): yarn pack doesn't call postpack, once the bug is fixed this can be removed - if (target.scripts.postpack) { - await run('yarn', ['postpack'], { cwd: target.location }); + if (target.packageJson?.scripts?.postpack) { + await run('yarn', ['postpack'], { cwd: target.dir }); } - const outputDir = relativePath(paths.targetRoot, target.location); + const outputDir = relativePath(paths.targetRoot, target.dir); const absoluteOutputPath = resolvePath(workspaceDir, outputDir); await fs.ensureDir(absoluteOutputPath); @@ -181,7 +185,7 @@ async function moveToDistWorkspace( // We remove the dependencies from package.json of packages that are marked // as bundled, so that yarn doesn't try to install them. - if (target.get('bundled')) { + if (target.packageJson.bundled) { const pkgJson = await fs.readJson( resolvePath(absoluteOutputPath, 'package.json'), ); @@ -220,41 +224,3 @@ async function moveToDistWorkspace( ), ); } - -async function findTargetPackages(pkgNames: string[]): Promise { - const { Project } = require('@lerna/project'); - const { PackageGraph } = require('@lerna/package-graph'); - - const project = new Project(paths.targetDir); - const packages = await project.getPackages(); - const graph = new PackageGraph(packages); - - const targets = new Map(); - const searchNames = pkgNames.slice(); - - while (searchNames.length) { - const name = searchNames.pop()!; - - if (targets.has(name)) { - continue; - } - - const node = graph.get(name); - if (!node) { - throw new Error(`Package '${name}' not found`); - } - - // Don't include dependencies of packages that are marked as bundled - if (!node.pkg.get('bundled')) { - const pkgDeps = Object.keys(node.pkg.dependencies ?? {}); - const localDeps: string[] = Array.from(node.localDependencies.keys()); - const filteredDeps = localDeps.filter(dep => pkgDeps.includes(dep)); - - searchNames.push(...filteredDeps); - } - - targets.set(name, node.pkg); - } - - return Array.from(targets.values()); -} diff --git a/packages/cli/src/lib/versioning/packages.test.ts b/packages/cli/src/lib/versioning/packages.test.ts index b7f3614fb7..7b39d4fccb 100644 --- a/packages/cli/src/lib/versioning/packages.test.ts +++ b/packages/cli/src/lib/versioning/packages.test.ts @@ -17,7 +17,6 @@ import mockFs from 'mock-fs'; import path from 'path'; import * as runObj from '../run'; -import { paths } from '../paths'; import { fetchPackageInfo, mapDependencies } from './packages'; import { NotFoundError } from '../errors'; @@ -58,22 +57,19 @@ describe('mapDependencies', () => { }); it('should read dependencies', async () => { - // Make sure all modules involved in package discovery are in the module cache before we mock fs - const { Project } = require('@lerna/project'); - const project = new Project(paths.targetDir); - await project.getPackages(); - mockFs({ - 'lerna.json': JSON.stringify({ - packages: ['pkgs/*'], + '/root/package.json': JSON.stringify({ + workspaces: { + packages: ['pkgs/*'], + }, }), - 'pkgs/a/package.json': JSON.stringify({ + '/root/pkgs/a/package.json': JSON.stringify({ name: 'a', dependencies: { '@backstage/core': '1 || 2', }, }), - 'pkgs/b/package.json': JSON.stringify({ + '/root/pkgs/b/package.json': JSON.stringify({ name: 'b', dependencies: { '@backstage/core': '3', @@ -82,10 +78,7 @@ describe('mapDependencies', () => { }), }); - const dependencyMap = await mapDependencies( - paths.targetDir, - '@backstage/*', - ); + const dependencyMap = await mapDependencies('/root', '@backstage/*'); expect(Array.from(dependencyMap)).toEqual([ [ '@backstage/core', @@ -93,12 +86,12 @@ describe('mapDependencies', () => { { name: 'a', range: '1 || 2', - location: path.resolve('pkgs', 'a'), + location: path.normalize('/root/pkgs/a'), }, { name: 'b', range: '3', - location: path.resolve('pkgs', 'b'), + location: path.normalize('/root/pkgs/b'), }, ], ], @@ -108,7 +101,7 @@ describe('mapDependencies', () => { { name: 'b', range: '^0', - location: path.resolve('pkgs', 'b'), + location: path.normalize('/root/pkgs/b'), }, ], ], diff --git a/packages/cli/src/lib/versioning/packages.ts b/packages/cli/src/lib/versioning/packages.ts index aa402431b9..1dabec6032 100644 --- a/packages/cli/src/lib/versioning/packages.ts +++ b/packages/cli/src/lib/versioning/packages.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import minimatch from 'minimatch'; +import { getPackages } from '@manypkg/get-packages'; import { runPlain } from '../../lib/run'; import { NotFoundError } from '../errors'; @@ -22,7 +23,7 @@ const DEP_TYPES = [ 'devDependencies', 'peerDependencies', 'optionalDependencies', -]; +] as const; // Package data as returned by `yarn info` export type YarnInfoInspectData = { @@ -66,14 +67,12 @@ export async function mapDependencies( targetDir: string, pattern: string, ): Promise> { - const { Project } = require('@lerna/project'); - const project = new Project(targetDir); - const packages = await project.getPackages(); + const { packages } = await getPackages(targetDir); const dependencyMap = new Map(); for (const pkg of packages) { const deps = DEP_TYPES.flatMap( - t => Object.entries(pkg.get(t) ?? {}) as [string, string][], + t => Object.entries(pkg.packageJson[t] ?? {}) as [string, string][], ); for (const [name, range] of deps) { @@ -82,8 +81,8 @@ export async function mapDependencies( name, (dependencyMap.get(name) ?? []).concat({ range, - name: pkg.name, - location: pkg.location, + name: pkg.packageJson.name, + location: pkg.dir, }), ); } diff --git a/scripts/check-type-dependencies.js b/scripts/check-type-dependencies.js index 4db1109145..ae99e5d7f8 100755 --- a/scripts/check-type-dependencies.js +++ b/scripts/check-type-dependencies.js @@ -20,13 +20,10 @@ const { resolve: resolvePath } = require('path'); // Cba polluting root package.json, we'll have this // eslint-disable-next-line import/no-extraneous-dependencies const chalk = require('chalk'); +const { getPackages } = require('@manypkg/get-packages'); async function main() { - // This is from lerna, and cba polluting root package.json - // eslint-disable-next-line import/no-extraneous-dependencies - const { Project } = require('@lerna/project'); - const project = new Project(resolvePath('.')); - const packages = await project.getPackages(); + const { packages } = await getPackages(resolvePath('.')); let hadErrors = false; @@ -38,7 +35,7 @@ async function main() { if (errors.length) { hadErrors = true; console.error( - `Incorrect type dependencies in ${chalk.yellow(pkg.name)}:`, + `Incorrect type dependencies in ${chalk.yellow(pkg.packageJson.name)}:`, ); for (const error of errors) { if (error.name === 'WrongDepError') { @@ -71,8 +68,8 @@ async function main() { function shouldCheckTypes(pkg) { return ( !pkg.private && - pkg.get('types') && - fs.existsSync(resolvePath(pkg.location, 'dist/index.d.ts')) + pkg.packageJson.types && + fs.existsSync(resolvePath(pkg.dir, 'dist/index.d.ts')) ); } @@ -82,7 +79,7 @@ function shouldCheckTypes(pkg) { */ function checkTypes(pkg) { const typeDecl = fs.readFileSync( - resolvePath(pkg.location, 'dist/index.d.ts'), + resolvePath(pkg.dir, 'dist/index.d.ts'), 'utf8', ); const allDeps = (typeDecl.match(/from '.*'/g) || []) @@ -114,21 +111,21 @@ function checkTypes(pkg) { */ function findTypesPackage(dep, pkg) { try { - require.resolve(`@types/${dep}/package.json`, { paths: [pkg.location] }); + require.resolve(`@types/${dep}/package.json`, { paths: [pkg.dir] }); return `@types/${dep}`; } catch { try { - require.resolve(dep, { paths: [pkg.location] }); + require.resolve(dep, { paths: [pkg.dir] }); return undefined; } catch { try { // Some type-only modules don't have a working main field, so try resolving package.json too - require.resolve(`${dep}/package.json`, { paths: [pkg.location] }); + require.resolve(`${dep}/package.json`, { paths: [pkg.dir] }); return undefined; } catch { try { // Finally check if it's just a .d.ts file - require.resolve(`${dep}.d.ts`, { paths: [pkg.location] }); + require.resolve(`${dep}.d.ts`, { paths: [pkg.dir] }); return undefined; } catch { throw mkErr('MissingDepError', `No types for ${dep}`, { dep }); @@ -142,10 +139,10 @@ function findTypesPackage(dep, pkg) { * Figures out what type dependencies are missing, or should be moved between dep types */ function findTypeDepErrors(typeDeps, pkg) { - const devDeps = mkTypeDepSet(pkg.get('devDependencies')); + const devDeps = mkTypeDepSet(pkg.packageJson.devDependencies); const deps = mkTypeDepSet({ - ...pkg.get('dependencies'), - ...pkg.get('peerDependencies'), + ...pkg.packageJson.dependencies, + ...pkg.packageJson.peerDependencies, }); const errors = []; diff --git a/scripts/isolated-release.js b/scripts/isolated-release.js index a7456c41b9..4df900450e 100755 --- a/scripts/isolated-release.js +++ b/scripts/isolated-release.js @@ -17,8 +17,7 @@ const path = require('path'); const childProcess = require('child_process'); -// eslint-disable-next-line import/no-extraneous-dependencies -const { Project } = require('@lerna/project'); +const { getPackages } = require('@manypkg/get-packages'); // Prepare a release of the provided packages, e.g. @backstage/core async function main(args) { @@ -28,11 +27,10 @@ async function main(args) { process.exit(1); } - const project = new Project(__dirname); - const packages = await project.getPackages(); + const { packages } = await getPackages(__dirname); const ignoreArgs = packages - .filter(p => !args.includes(p.name)) - .flatMap(p => ['--ignore', p.name]); + .filter(p => !args.includes(p.packageJson.name)) + .flatMap(p => ['--ignore', p.packageJson.name]); const { status } = childProcess.spawnSync( 'yarn', diff --git a/yarn.lock b/yarn.lock index ada84af89b..9f6bf3d5d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4264,7 +4264,7 @@ tar "^6.1.0" temp-write "^4.0.0" -"@lerna/package-graph@4.0.0", "@lerna/package-graph@^4.0.0": +"@lerna/package-graph@4.0.0": version "4.0.0" resolved "https://registry.npmjs.org/@lerna/package-graph/-/package-graph-4.0.0.tgz#16a00253a8ac810f72041481cb46bcee8d8123dd" integrity sha512-QED2ZCTkfXMKFoTGoccwUzjHtZMSf3UKX14A4/kYyBms9xfFsesCZ6SLI5YeySEgcul8iuIWfQFZqRw+Qrjraw== @@ -4300,7 +4300,7 @@ npmlog "^4.1.2" upath "^2.0.1" -"@lerna/project@4.0.0", "@lerna/project@^4.0.0": +"@lerna/project@4.0.0": version "4.0.0" resolved "https://registry.npmjs.org/@lerna/project/-/project-4.0.0.tgz#ff84893935833533a74deff30c0e64ddb7f0ba6b" integrity sha512-o0MlVbDkD5qRPkFKlBZsXZjoNTWPyuL58564nSfZJ6JYNmgAptnWPB2dQlAc7HWRZkmnC2fCkEdoU+jioPavbg== @@ -4532,6 +4532,18 @@ globby "^11.0.0" read-yaml-file "^1.1.0" +"@manypkg/get-packages@^1.1.3": + version "1.1.3" + resolved "https://registry.npmjs.org/@manypkg/get-packages/-/get-packages-1.1.3.tgz#e184db9bba792fa4693de4658cfb1463ac2c9c47" + integrity sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A== + dependencies: + "@babel/runtime" "^7.5.5" + "@changesets/types" "^4.0.1" + "@manypkg/find-root" "^1.1.0" + fs-extra "^8.1.0" + globby "^11.0.0" + read-yaml-file "^1.1.0" + "@mapbox/node-pre-gyp@^1.0.0": version "1.0.5" resolved "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz#2a0b32fcb416fb3f2250fd24cb2a81421a4f5950"