Merge pull request #3360 from backstage/rugvip/extbuild

cli: enable transformation of workspace packages outside the workspace root

.changeset/cli-links.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+The CLI now detects and transforms linked packages. You can link in external packages by adding them to both the `lerna.json` and `package.json` workspace paths.

.github/styles/vocab.txt

@@ -189,6 +189,7 @@ squidfunk
 src
 stefanalund
 subkey
+subtree
 superfences
 Superfences
 superset

docs/getting-started/create-an-app.md

@@ -38,6 +38,42 @@ app-folder is the name that was provided when prompted.
 Inside that directory, it will generate all the files and folder structure
 needed for you to run your app.
 
+### Linking in local Backstage packages
+
+It can often be useful to try out changes to the packages in the main Backstage
+repo within your own app. For example if you want to make modifications to
+`@backstage/core` and try them out in your app.
+
+To link in external packages, add them to your `package.json` and `lerna.json`
+workspace paths. These can be either relative or absolute paths with or without
+globs. For example:
+
+```json
+"packages": [
+  "packages/*",
+  "plugins/*",
+  "../backstage/packages/core", // New path added to work on @backstage/core
+],
+```
+
+Then reinstall packages to make yarn set up symlinks:
+
+```bash
+yarn install
+```
+
+With this in place you can now modify the `@backstage/core` package within the
+main repo, and have those changes be reflected and tested in your app. Simply
+run your app using `yarn start` as normal.
+
+Note that for backend packages you need to make sure that linked packages are
+not dependencies of any non-linked package. If you for example want to work on
+`@backstage/backend-common`, you need to also link in other backend plugins and
+packages that depend on `@backstage/backend-common`, or temporarily disable
+those plugins in your backend. This is because the transformation of backend
+module tree stops whenever a non-local package is encountered, and from that
+point node will `require` packages directly for that entire module subtree.
+
 ### Troubleshooting
 
 The create app command doesn't always work as expected, this is a collection of

packages/cli/src/lib/bundler/config.ts

@@ -70,13 +70,27 @@ async function readBuildInfo() {
   };
 }
 
+async function loadLernaPackages(): Promise<
+  { name: string; location: string }[]
+> {
+  const LernaProject = require('@lerna/project');
+  const project = new LernaProject(cliPaths.targetDir);
+  return project.getPackages();
+}
+
 export async function createConfig(
   paths: BundlingPaths,
   options: BundlingOptions,
 ): Promise<webpack.Configuration> {
   const { checksEnabled, isDev, frontendConfig } = options;
 
-  const { plugins, loaders } = transforms(options);
+  const packages = await loadLernaPackages();
+  const { plugins, loaders } = transforms({
+    ...options,
+    externalTransforms: packages.map(({ name }) =>
+      cliPaths.resolveTargetRoot('node_modules', name),
+    ),
+  });
 
   const baseUrl = frontendConfig.getString('app.baseUrl');
   const validBaseUrl = new URL(baseUrl);
@@ -159,6 +173,10 @@ export async function createConfig(
       alias: {
         'react-dom': '@hot-loader/react-dom',
       },
+      // Enables proper resolution of packages when linking in external packages.
+      // Without this the packages would depend on dependencies in the node_modules
+      // of the external packages themselves, leading to module duplication
+      symlinks: false,
     },
     module: {
       rules: loaders,
@@ -181,17 +199,20 @@ export async function createBackendConfig(
 ): Promise<webpack.Configuration> {
   const { checksEnabled, isDev } = options;
 
-  const { loaders } = transforms(options);
-
   // Find all local monorepo packages and their node_modules, and mark them as external.
-  const LernaProject = require('@lerna/project');
-  const project = new LernaProject(cliPaths.targetDir);
-  const packages = await project.getPackages();
+  const packages = await await loadLernaPackages();
   const localPackageNames = packages.map((p: any) => p.name);
   const moduleDirs = packages.map((p: any) =>
     resolvePath(p.location, 'node_modules'),
   );
 
+  const { loaders } = transforms({
+    ...options,
+    externalTransforms: packages.map(({ name }) =>
+      cliPaths.resolveTargetRoot('node_modules', name),
+    ),
+  });
+
   return {
     mode: isDev ? 'development' : 'production',
     profile: false,
@@ -240,6 +261,7 @@ export async function createBackendConfig(
       alias: {
         'react-dom': '@hot-loader/react-dom',
       },
+      symlinks: false, // See frontend config, added here for the same reason
     },
     module: {
       rules: loaders,

packages/cli/src/lib/bundler/transforms.ts

@@ -16,7 +16,6 @@
 
 import webpack, { Module, Plugin } from 'webpack';
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
-import { BundlingOptions, BackendBundlingOptions } from './types';
 import { svgrTemplate } from '../svgrTemplate';
 
 type Transforms = {
@@ -24,17 +23,25 @@ type Transforms = {
   plugins: Plugin[];
 };
 
-export const transforms = (
-  options: BundlingOptions | BackendBundlingOptions,
-): Transforms => {
-  const { isDev } = options;
+type TransformOptions = {
+  isDev: boolean;
+  // External paths that should be transformed
+  externalTransforms: string[];
+};
+
+export const transforms = (options: TransformOptions): Transforms => {
+  const { isDev, externalTransforms } = options;
 
   const extraTransforms = isDev ? ['react-hot-loader'] : [];
 
+  const transformExcludeCondition = {
+    and: [/node_modules/, { not: externalTransforms }],
+  };
+
   const loaders = [
     {
       test: /\.(tsx?)$/,
-      exclude: /node_modules/,
+      exclude: transformExcludeCondition,
       loader: require.resolve('@sucrase/webpack-loader'),
       options: {
         transforms: ['typescript', 'jsx', ...extraTransforms],
@@ -43,7 +50,7 @@ export const transforms = (
     },
     {
       test: /\.(jsx?|mjs)$/,
-      exclude: /node_modules/,
+      exclude: transformExcludeCondition,
       loader: require.resolve('@sucrase/webpack-loader'),
       options: {
         transforms: ['jsx', ...extraTransforms],