Merge pull request #25633 from backstage/mtlewis/dynamic-plugins-dev

cli: add support for local development of dynamic plugin containers

.changeset/ninety-icons-smile.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Remove usage of deprecated functionality from @backstage/config-loader

.changeset/rich-mugs-dress.md

@@ -0,0 +1,7 @@
+---
+'@backstage/config-loader': patch
+---
+
+Add boolean `allowMissingDefaultConfig` option to `ConfigSources.default` and
+`ConfigSources.defaultForTargets`, which results in omission of a ConfigSource
+for the default app-config.yaml configuration file if it's not present.

.changeset/strong-otters-compete.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Add support for dynamic plugins via the EXPERIMENTAL_MODULE_FEDERATION environment variable when running `yarn start`.

packages/cli/src/commands/build/buildFrontend.ts

@@ -16,11 +16,9 @@
 
 import fs from 'fs-extra';
 import { resolve as resolvePath } from 'path';
-import { buildBundle } from '../../lib/bundler';
+import { buildBundle, getModuleFederationOptions } from '../../lib/bundler';
 import { getEnvironmentParallelism } from '../../lib/parallel';
 import { loadCliConfig } from '../../lib/config';
-import chalk from 'chalk';
-import { BuildOptions } from '../../lib/bundler/types';
 
 interface BuildAppOptions {
   targetDir: string;
@@ -29,29 +27,6 @@ interface BuildAppOptions {
   isModuleFederationRemote?: true;
 }
 
-function getModuleFederationOptions(
-  name: string,
-  isRemote?: boolean,
-): BuildOptions['moduleFederation'] {
-  if (!isRemote && !process.env.EXPERIMENTAL_MODULE_FEDERATION) {
-    return undefined;
-  }
-
-  console.log(
-    chalk.yellow(
-      `⚠️  WARNING: Module federation is experimental and will receive immediate breaking changes in the future.`,
-    ),
-  );
-
-  return {
-    mode: isRemote ? 'remote' : 'host',
-    // The default output mode requires the name to be a usable as a code
-    // symbol, there might be better options here but for now we need to
-    // sanitize the name.
-    name: name.replaceAll('@', '').replaceAll('/', '__').replaceAll('-', '_'),
-  };
-}
-
 export async function buildFrontend(options: BuildAppOptions) {
   const { targetDir, writeStats, configPaths } = options;
   const { name } = await fs.readJson(resolvePath(targetDir, 'package.json'));

packages/cli/src/commands/start/command.ts

@@ -15,6 +15,7 @@
  */
 
 import { OptionValues } from 'commander';
+import { PackageRole } from '@backstage/cli-node';
 import { findRoleFromCommand } from '../../lib/role';
 import { startBackend, startBackendPlugin } from './startBackend';
 import { startFrontend } from './startFrontend';
@@ -47,6 +48,13 @@ export async function command(opts: OptionValues): Promise<void> {
     case 'frontend-plugin':
     case 'frontend-plugin-module':
       return startFrontend({ entry: 'dev/index', ...options });
+    case 'frontend-dynamic-container' as PackageRole: // experimental
+      return startFrontend({
+        entry: 'src/index',
+        ...options,
+        skipOpenBrowser: true,
+        isModuleFederationRemote: true,
+      });
     default:
       throw new Error(
         `Start command is not supported for package role '${role}'`,

packages/cli/src/commands/start/startFrontend.ts

@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 
-import { serveBundle } from '../../lib/bundler';
+import { readJson } from 'fs-extra';
+import { getModuleFederationOptions, serveBundle } from '../../lib/bundler';
+import { paths } from '../../lib/paths';
 
 interface StartAppOptions {
   verifyVersions?: boolean;
@@ -22,14 +24,23 @@ interface StartAppOptions {
 
   checksEnabled: boolean;
   configPaths: string[];
+  skipOpenBrowser?: boolean;
+  isModuleFederationRemote?: boolean;
 }
 
 export async function startFrontend(options: StartAppOptions) {
+  const { name } = await readJson(paths.resolveTarget('package.json'));
+
   const waitForExit = await serveBundle({
     entry: options.entry,
     checksEnabled: options.checksEnabled,
     configPaths: options.configPaths,
     verifyVersions: options.verifyVersions,
+    skipOpenBrowser: options.skipOpenBrowser,
+    moduleFederation: getModuleFederationOptions(
+      name,
+      options.isModuleFederationRemote,
+    ),
   });
 
   await waitForExit();

packages/cli/src/lib/bundler/config.ts

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-import { BackendBundlingOptions, BundlingOptions } from './types';
+import {
+  BackendBundlingOptions,
+  BundlingOptions,
+  ModuleFederationOptions,
+} from './types';
 import { posix as posixPath, resolve as resolvePath, dirname } from 'path';
 import chalk from 'chalk';
 import webpack, { ProvidePlugin } from 'webpack';
@@ -29,6 +33,7 @@ import { ModuleFederationPlugin } from '@module-federation/enhanced/webpack';
 import { LinkedPackageResolvePlugin } from './LinkedPackageResolvePlugin';
 import ModuleScopePlugin from 'react-dev-utils/ModuleScopePlugin';
 import { RunScriptWebpackPlugin } from 'run-script-webpack-plugin';
+import ReactRefreshPlugin from '@pmmmwh/react-refresh-webpack-plugin';
 import { paths as cliPaths } from '../../lib/paths';
 import fs from 'fs-extra';
 import { getPackages } from '@manypkg/get-packages';
@@ -45,15 +50,42 @@ import { hasReactDomClient } from './hasReactDomClient';
 
 const BUILD_CACHE_ENV_VAR = 'BACKSTAGE_CLI_EXPERIMENTAL_BUILD_CACHE';
 
-export function resolveBaseUrl(config: Config): URL {
+export function resolveBaseUrl(
+  config: Config,
+  moduleFederation?: ModuleFederationOptions,
+): URL {
   const baseUrl = config.getOptionalString('app.baseUrl');
+
+  const defaultBaseUrl =
+    moduleFederation?.mode === 'remote'
+      ? `http://localhost:${process.env.PORT ?? '3000'}`
+      : 'http://localhost:3000';
+
   try {
-    return new URL(baseUrl ?? '/', 'http://localhost:3000');
+    return new URL(baseUrl ?? '/', defaultBaseUrl);
   } catch (error) {
     throw new Error(`Invalid app.baseUrl, ${error}`);
   }
 }
 
+export function resolveEndpoint(
+  config: Config,
+  moduleFederation?: ModuleFederationOptions,
+): {
+  host: string;
+  port: number;
+} {
+  const url = resolveBaseUrl(config, moduleFederation);
+
+  return {
+    host: config.getOptionalString('app.listen.host') ?? url.hostname,
+    port:
+      config.getOptionalNumber('app.listen.port') ??
+      Number(url.port) ??
+      (url.protocol === 'https:' ? 443 : 80),
+  };
+}
+
 async function readBuildInfo() {
   const timestamp = Date.now();
 
@@ -94,7 +126,13 @@ export async function createConfig(
   paths: BundlingPaths,
   options: BundlingOptions,
 ): Promise<webpack.Configuration> {
-  const { checksEnabled, isDev, frontendConfig, publicSubPath = '' } = options;
+  const {
+    checksEnabled,
+    isDev,
+    frontendConfig,
+    moduleFederation,
+    publicSubPath = '',
+  } = options;
 
   const { plugins, loaders } = transforms(options);
   // Any package that is part of the monorepo but outside the monorepo root dir need
@@ -102,12 +140,29 @@ export async function createConfig(
   const { packages } = await getPackages(cliPaths.targetDir);
   const externalPkgs = packages.filter(p => !isChildPath(paths.root, p.dir));
 
-  const validBaseUrl = resolveBaseUrl(frontendConfig);
+  const validBaseUrl = resolveBaseUrl(frontendConfig, moduleFederation);
   let publicPath = validBaseUrl.pathname.replace(/\/$/, '');
   if (publicSubPath) {
     publicPath = `${publicPath}${publicSubPath}`.replace('//', '/');
   }
 
+  if (isDev) {
+    const { host, port } = resolveEndpoint(
+      options.frontendConfig,
+      options.moduleFederation,
+    );
+
+    plugins.push(
+      new ReactRefreshPlugin({
+        overlay: {
+          sockProtocol: 'ws',
+          sockHost: host,
+          sockPort: port,
+        },
+      }),
+    );
+  }
+
   if (checksEnabled) {
     plugins.push(
       new ForkTsCheckerWebpackPlugin({

packages/cli/src/lib/bundler/index.ts

@@ -16,4 +16,5 @@
 
 export { serveBackend } from './backend';
 export { buildBundle } from './bundle';
+export { getModuleFederationOptions } from './moduleFederation';
 export { serveBundle } from './server';

packages/cli/src/lib/bundler/moduleFederation.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import chalk from 'chalk';
+import { ModuleFederationOptions } from './types';
+
+export function getModuleFederationOptions(
+  name: string,
+  isModuleFederationRemote?: boolean,
+): ModuleFederationOptions | undefined {
+  if (
+    !isModuleFederationRemote &&
+    !process.env.EXPERIMENTAL_MODULE_FEDERATION
+  ) {
+    return undefined;
+  }
+
+  console.log(
+    chalk.yellow(
+      `⚠️  WARNING: Module federation is experimental and will receive immediate breaking changes in the future.`,
+    ),
+  );
+
+  return {
+    mode: isModuleFederationRemote ? 'remote' : 'host',
+    // The default output mode requires the name to be a usable as a code
+    // symbol, there might be better options here but for now we need to
+    // sanitize the name.
+    name: name.replaceAll('@', '').replaceAll('/', '__').replaceAll('-', '_'),
+  };
+}

packages/cli/src/lib/bundler/server.ts

@@ -30,7 +30,7 @@ import {
 import { paths as libPaths } from '../../lib/paths';
 import { loadCliConfig } from '../config';
 import { Lockfile } from '../versioning';
-import { createConfig, resolveBaseUrl } from './config';
+import { createConfig, resolveBaseUrl, resolveEndpoint } from './config';
 import { createDetectedModulesEntryPoint } from './packageDetection';
 import { resolveBundlingPaths, resolveOptionalBundlingPaths } from './paths';
 import { ServeOptions } from './types';
@@ -130,14 +130,11 @@ DEPRECATION WARNING: React Router Beta is deprecated and support for it will be
   }
 
   const { frontendConfig, fullConfig } = cliConfig;
-  const url = resolveBaseUrl(frontendConfig);
-
-  const host =
-    frontendConfig.getOptionalString('app.listen.host') || url.hostname;
-  const port =
-    frontendConfig.getOptionalNumber('app.listen.port') ||
-    Number(url.port) ||
-    (url.protocol === 'https:' ? 443 : 80);
+  const url = resolveBaseUrl(frontendConfig, options.moduleFederation);
+  const { host, port } = resolveEndpoint(
+    frontendConfig,
+    options.moduleFederation,
+  );
 
   const detectedModulesEntryPoint = await createDetectedModulesEntryPoint({
     config: fullConfig,
@@ -162,6 +159,7 @@ DEPRECATION WARNING: React Router Beta is deprecated and support for it will be
   const config = await createConfig(paths, {
     ...commonConfigOptions,
     additionalEntryPoints: detectedModulesEntryPoint,
+    moduleFederation: options.moduleFederation,
   });
 
   if (process.env.EXPERIMENTAL_VITE) {
@@ -232,14 +230,17 @@ DEPRECATION WARNING: React Router Beta is deprecated and support for it will be
               directory: paths.targetPublic,
             }
           : undefined,
-        historyApiFallback: {
-          // Paths with dots should still use the history fallback.
-          // See https://github.com/facebookincubator/create-react-app/issues/387.
-          disableDotRule: true,
+        historyApiFallback:
+          options.moduleFederation?.mode === 'remote'
+            ? false
+            : {
+                // Paths with dots should still use the history fallback.
+                // See https://github.com/facebookincubator/create-react-app/issues/387.
+                disableDotRule: true,
 
-          // The index needs to be rewritten relative to the new public path, including subroutes.
-          index: `${config.output?.publicPath}index.html`,
-        },
+                // The index needs to be rewritten relative to the new public path, including subroutes.
+                index: `${config.output?.publicPath}index.html`,
+              },
         server:
           url.protocol === 'https:'
             ? {
@@ -256,7 +257,13 @@ DEPRECATION WARNING: React Router Beta is deprecated and support for it will be
         // When the dev server is behind a proxy, the host and public hostname differ
         allowedHosts: [url.hostname],
         client: {
-          webSocketURL: 'auto://0.0.0.0:0/ws',
+          webSocketURL: { hostname: host, port },
+        },
+        headers: {
+          'Access-Control-Allow-Origin': '*',
+          'Access-Control-Allow-Methods': 'GET, OPTIONS',
+          'Access-Control-Allow-Headers':
+            'X-Requested-With, content-type, Authorization',
         },
       },
       compiler,
@@ -278,7 +285,9 @@ DEPRECATION WARNING: React Router Beta is deprecated and support for it will be
     }
   });
 
-  openBrowser(url.href);
+  if (!options.skipOpenBrowser) {
+    openBrowser(url.href);
+  }
 
   const waitForExit = async () => {
     for (const signal of ['SIGINT', 'SIGTERM'] as const) {

packages/cli/src/lib/bundler/transforms.ts

@@ -17,7 +17,6 @@
 import { ModuleOptions, WebpackPluginInstance } from 'webpack';
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
 import { svgrTemplate } from '../svgrTemplate';
-import ReactRefreshPlugin from '@pmmmwh/react-refresh-webpack-plugin';
 
 type Transforms = {
   loaders: ModuleOptions['rules'];
@@ -193,15 +192,7 @@ export const transforms = (options: TransformOptions): Transforms => {
 
   const plugins = new Array<WebpackPluginInstance>();
 
-  if (isDev) {
-    if (!isBackend) {
-      plugins.push(
-        new ReactRefreshPlugin({
-          overlay: { sockProtocol: 'ws' },
-        }),
-      );
-    }
-  } else {
+  if (!isDev) {
     plugins.push(
       new MiniCssExtractPlugin({
         filename: 'static/[name].[contenthash:8].css',

packages/cli/src/lib/bundler/types.ts

@@ -43,6 +43,8 @@ export type ServeOptions = BundlingPathsOptions & {
   checksEnabled: boolean;
   configPaths: string[];
   verifyVersions?: boolean;
+  skipOpenBrowser?: boolean;
+  moduleFederation?: ModuleFederationOptions;
 };
 
 export type BuildOptions = BundlingPathsOptions & {

packages/cli/src/lib/config.ts

@@ -14,14 +14,9 @@
  * limitations under the License.
  */
 
-import {
-  ConfigTarget,
-  loadConfig,
-  loadConfigSchema,
-} from '@backstage/config-loader';
+import { ConfigSources, loadConfigSchema } from '@backstage/config-loader';
 import { AppConfig, ConfigReader } from '@backstage/config';
 import { paths } from './paths';
-import { isValidUrl } from './urls';
 import { getPackages } from '@manypkg/get-packages';
 import { PackageGraph } from '@backstage/cli-node';
 
@@ -37,13 +32,6 @@ type Options = {
 };
 
 export async function loadCliConfig(options: Options) {
-  const configTargets: ConfigTarget[] = [];
-  options.args.forEach(arg => {
-    if (!isValidUrl(arg)) {
-      configTargets.push({ path: paths.resolveTarget(arg) });
-    }
-  });
-
   // Consider all packages in the monorepo when loading in config
   const { packages } = await getPackages(paths.targetDir);
 
@@ -75,32 +63,62 @@ export async function loadCliConfig(options: Options) {
     noUndeclaredProperties: options.strict,
   });
 
-  const { appConfigs } = await loadConfig({
-    experimentalEnvFunc: options.mockEnv
+  const source = ConfigSources.default({
+    allowMissingDefaultConfig: true,
+    substitutionFunc: options.mockEnv
       ? async name => process.env[name] || 'x'
       : undefined,
-    configRoot: paths.targetRoot,
-    configTargets: configTargets,
-    watch: options.watch && {
-      onChange(newAppConfigs) {
-        const newFrontendAppConfigs = schema.process(newAppConfigs, {
-          visibility: options.fullVisibility
-            ? ['frontend', 'backend', 'secret']
-            : ['frontend'],
-          withFilteredKeys: options.withFilteredKeys,
-          withDeprecatedKeys: options.withDeprecatedKeys,
-          ignoreSchemaErrors: !options.strict,
-        });
-        options.watch?.(newFrontendAppConfigs);
-      },
-    },
+    watch: Boolean(options.watch),
+    rootDir: paths.targetRoot,
+    argv: options.args,
   });
 
+  const appConfigs = await new Promise<AppConfig[]>((resolve, reject) => {
+    async function loadConfigReaderLoop() {
+      let loaded = false;
+
+      try {
+        const abortController = new AbortController();
+        for await (const { configs } of source.readConfigData({
+          signal: abortController.signal,
+        })) {
+          if (loaded) {
+            const newFrontendAppConfigs = schema.process(configs, {
+              visibility: options.fullVisibility
+                ? ['frontend', 'backend', 'secret']
+                : ['frontend'],
+              withFilteredKeys: options.withFilteredKeys,
+              withDeprecatedKeys: options.withDeprecatedKeys,
+              ignoreSchemaErrors: !options.strict,
+            });
+            options.watch?.(newFrontendAppConfigs);
+          } else {
+            resolve(configs);
+            loaded = true;
+
+            if (!options.watch) {
+              abortController.abort();
+            }
+          }
+        }
+      } catch (error) {
+        if (loaded) {
+          console.error(`Failed to reload configuration, ${error}`);
+        } else {
+          reject(error);
+        }
+      }
+    }
+    loadConfigReaderLoop();
+  });
+
+  const configurationLoadedMessage = appConfigs.length
+    ? `Loaded config from ${appConfigs.map(c => c.context).join(', ')}`
+    : `No configuration files found, running without config`;
+
   // printing to stderr to not clobber stdout in case the cli command
   // outputs structured data (e.g. as config:schema does)
-  process.stderr.write(
-    `Loaded config from ${appConfigs.map(c => c.context).join(', ')}\n`,
-  );
+  process.stderr.write(`${configurationLoadedMessage}\n`);
 
   try {
     const frontendAppConfigs = schema.process(appConfigs, {

packages/config-loader/api-report.md

@@ -21,6 +21,7 @@ export type AsyncConfigSourceGenerator = AsyncGenerator<
 
 // @public
 export interface BaseConfigSourcesOptions {
+  allowMissingDefaultConfig?: boolean;
   // (undocumented)
   remote?: Pick<RemoteConfigSourceOptions, 'reloadInterval'>;
   // (undocumented)

packages/config-loader/src/sources/ConfigSources.test.ts

@@ -69,13 +69,18 @@ describe('ConfigSources', () => {
   });
 
   it('should create default sources for targets', () => {
+    const fsSpy = jest.spyOn(fs, 'pathExistsSync').mockImplementation(path => {
+      return path === `${root}app-config.yaml`;
+    });
+
     expect(
       mergeSources(
         ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }),
       ),
     ).toEqual([{ name: 'FileConfigSource', path: `${root}app-config.yaml` }]);
 
-    const fsSpy = jest.spyOn(fs, 'pathExistsSync').mockReturnValue(true);
+    fsSpy.mockReturnValue(true);
+
     expect(
       mergeSources(
         ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }),
@@ -159,6 +164,10 @@ describe('ConfigSources', () => {
   });
 
   it('should create a default source', () => {
+    const fsSpy = jest.spyOn(fs, 'pathExistsSync').mockImplementation(path => {
+      return path === `${root}app-config.yaml`;
+    });
+
     expect(
       mergeSources(
         ConfigSources.default({
@@ -184,6 +193,8 @@ describe('ConfigSources', () => {
       { name: 'FileConfigSource', path: resolvePath('b.yaml') },
       { name: 'EnvConfigSource', env: { HOME: '/' } },
     ]);
+
+    fsSpy.mockRestore();
   });
 
   it('should merge sources', () => {

packages/config-loader/src/sources/ConfigSources.ts

@@ -74,6 +74,11 @@ export interface BaseConfigSourcesOptions {
   watch?: boolean;
   rootDir?: string;
   remote?: Pick<RemoteConfigSourceOptions, 'reloadInterval'>;
+  /**
+   * Allow the default app-config.yaml to be missing, in which case the source
+   * will not be created.
+   */
+  allowMissingDefaultConfig?: boolean;
 
   /**
    * A custom substitution function that overrides the default one.
@@ -177,14 +182,19 @@ export class ConfigSources {
     if (argSources.length === 0) {
       const defaultPath = resolvePath(rootDir, 'app-config.yaml');
       const localPath = resolvePath(rootDir, 'app-config.local.yaml');
+      const alwaysIncludeDefaultConfigSource =
+        !options.allowMissingDefaultConfig;
+
+      if (alwaysIncludeDefaultConfigSource || fs.pathExistsSync(defaultPath)) {
+        argSources.push(
+          FileConfigSource.create({
+            watch: options.watch,
+            path: defaultPath,
+            substitutionFunc: options.substitutionFunc,
+          }),
+        );
+      }
 
-      argSources.push(
-        FileConfigSource.create({
-          watch: options.watch,
-          path: defaultPath,
-          substitutionFunc: options.substitutionFunc,
-        }),
-      );
       if (fs.pathExistsSync(localPath)) {
         argSources.push(
           FileConfigSource.create({