Merge pull request #19787 from backstage/mob/package-discovery

cli,frontend-app-api: add experimental support for frontend package discovery

.changeset/light-drinks-rush.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Add experimental support for frontend package discovery.

packages/app-next-example-plugin/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

packages/app-next-example-plugin/api-report.md

@@ -0,0 +1,16 @@
+## API Report File for "app-next-example-plugin"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { BackstagePlugin } from '@backstage/frontend-plugin-api';
+import { default as React_2 } from 'react';
+
+// @public (undocumented)
+export const examplePlugin: BackstagePlugin;
+
+// @public (undocumented)
+export const ExampleSidebarItem: () => React_2.JSX.Element;
+
+// (No @packageDocumentation comment for this package)
+```

packages/app-next-example-plugin/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "app-next-example-plugin",
+  "description": "Backstage internal example plugin",
+  "version": "0.0.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.esm.js",
+    "types": "dist/index.d.ts"
+  },
+  "private": true,
+  "backstage": {
+    "role": "frontend-plugin"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "packages/app-next-example-plugin"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "license": "Apache-2.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "scripts": {
+    "build": "backstage-cli package build",
+    "start": "backstage-cli package start",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean"
+  },
+  "dependencies": {
+    "@backstage/core-components": "workspace:^",
+    "@backstage/frontend-plugin-api": "workspace:^",
+    "@material-ui/icons": "^4.9.1"
+  },
+  "peerDependencies": {
+    "react": "^16.13.1 || ^17.0.0",
+    "react-dom": "^16.13.1 || ^17.0.0",
+    "react-router-dom": "6.0.0-beta.0 || ^6.3.0"
+  },
+  "devDependencies": {
+    "@backstage/cli": "workspace:^",
+    "cross-fetch": "^3.1.5",
+    "msw": "^1.0.0"
+  },
+  "files": [
+    "dist"
+  ]
+}

packages/app-next-example-plugin/src/Component.tsx

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+
+export const Component = () => (
+  <div>
+    <h1>SHIP IT!</h1>
+  </div>
+);

packages/app-next-example-plugin/src/ExampleSidebarItem.tsx

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { SidebarItem } from '@backstage/core-components';
+import SaveIcon from '@material-ui/icons/Save';
+
+/** @public */
+export const ExampleSidebarItem = () => (
+  <SidebarItem text="Example" to="/example" icon={SaveIcon} />
+);

packages/app-next-example-plugin/src/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { examplePlugin } from './plugin';
+
+// TODO: This should be an extension created & exported in the `plugin.tsx`
+export { ExampleSidebarItem } from './ExampleSidebarItem';

packages/app-next-example-plugin/src/plugin.tsx

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import {
+  createPageExtension,
+  createPlugin,
+} from '@backstage/frontend-plugin-api';
+
+export const ExamplePage = createPageExtension({
+  id: 'example.page',
+  defaultPath: '/example',
+  component: () => import('./Component').then(m => <m.Component />),
+});
+
+/** @public */
+export const examplePlugin = createPlugin({
+  id: 'example',
+  extensions: [ExamplePage],
+});

packages/app-next/app-config.yaml

@@ -1,5 +1,6 @@
 app:
-  packages: 'all' # ✨
+  experimental:
+    packages: 'all' # ✨
 
 
   # scmAuthExtension: >-

packages/app-next/package.json

@@ -86,6 +86,7 @@
     "@roadiehq/backstage-plugin-github-insights": "^2.0.5",
     "@roadiehq/backstage-plugin-github-pull-requests": "^2.2.7",
     "@roadiehq/backstage-plugin-travis-ci": "^2.0.5",
+    "app-next-example-plugin": "workspace:^",
     "history": "^5.0.0",
     "lodash": "^4.17.21",
     "prop-types": "^15.7.2",

packages/cli/src/lib/bundler/bundle.ts

@@ -27,6 +27,7 @@ import { createConfig, resolveBaseUrl } from './config';
 import { BuildOptions } from './types';
 import { resolveBundlingPaths } from './paths';
 import chalk from 'chalk';
+import { createDetectedModulesEntryPoint } from './packageDetection';
 
 // TODO(Rugvip): Limits from CRA, we might want to tweak these though.
 const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024;
@@ -40,12 +41,19 @@ export async function buildBundle(options: BuildOptions) {
   const { statsJsonEnabled, schema: configSchema } = options;
 
   const paths = resolveBundlingPaths(options);
+
+  const detectedModulesEntryPoint = await createDetectedModulesEntryPoint({
+    config: options.fullConfig,
+    targetPath: paths.targetPath,
+  });
+
   const config = await createConfig(paths, {
     ...options,
     checksEnabled: false,
     isDev: false,
     baseUrl: resolveBaseUrl(options.frontendConfig),
     getFrontendAppConfigs: () => options.frontendAppConfigs,
+    additionalEntryPoints: detectedModulesEntryPoint,
   });
 
   const isCi = yn(process.env.CI, { default: false });

packages/cli/src/lib/bundler/config.ts

@@ -161,7 +161,7 @@ export async function createConfig(
     },
     devtool: isDev ? 'eval-cheap-module-source-map' : 'source-map',
     context: paths.targetPath,
-    entry: [paths.targetEntry],
+    entry: [...(options.additionalEntryPoints ?? []), paths.targetEntry],
     resolve: {
       extensions: ['.ts', '.tsx', '.mjs', '.js', '.jsx', '.json', '.wasm'],
       mainFields: ['browser', 'module', 'main'],

packages/cli/src/lib/bundler/packageDetection.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { BackstagePackageJson } from '@backstage/cli-node';
+import { Config } from '@backstage/config';
+import chokidar from 'chokidar';
+import fs from 'fs-extra';
+import { join as joinPath, resolve as resolvePath } from 'path';
+import { paths as cliPaths } from '../paths';
+
+const DETECTED_MODULES_MODULE_NAME = '__backstage-autodetected-plugins__';
+
+interface PackageDetectionConfig {
+  include?: string[];
+  exclude?: string[];
+}
+
+function readPackageDetectionConfig(
+  config: Config,
+): PackageDetectionConfig | undefined {
+  const packages = config.getOptional('app.experimental.packages');
+  if (packages === undefined || packages === null) {
+    return undefined;
+  }
+
+  if (typeof packages === 'string') {
+    if (packages !== 'all') {
+      throw new Error(
+        `Invalid app.experimental.packages mode, got '${packages}', expected 'all'`,
+      );
+    }
+    return {};
+  }
+
+  return {
+    include: config.getOptionalStringArray('app.experimental.packages.include'),
+    exclude: config.getOptionalStringArray('app.experimental.packages.exclude'),
+  };
+}
+
+async function detectPackages(
+  targetPath: string,
+  { include, exclude }: PackageDetectionConfig,
+) {
+  const pkg: BackstagePackageJson = await fs.readJson(
+    resolvePath(targetPath, 'package.json'),
+  );
+
+  return Object.keys(pkg.dependencies ?? {})
+    .flatMap(depName => {
+      const depPackageJson: BackstagePackageJson = require(require.resolve(
+        `${depName}/package.json`,
+        { paths: [targetPath] },
+      ));
+      if (
+        ['frontend-plugin', 'frontend-plugin-module'].includes(
+          depPackageJson.backstage?.role ?? '',
+        )
+      ) {
+        return [depName];
+      }
+      return [];
+    })
+    .filter(name => {
+      if (exclude?.includes(name)) {
+        return false;
+      }
+      if (include && !include.includes(name)) {
+        return false;
+      }
+      return true;
+    });
+}
+
+async function writeDetectedPackagesModule(packageNames: string[]) {
+  const requirePackageScript = packageNames
+    ?.map(pkg => `{name: '${pkg}', module: require('${pkg}')}`)
+    .join(',');
+
+  await fs.writeFile(
+    joinPath(
+      cliPaths.targetRoot,
+      'node_modules',
+      `${DETECTED_MODULES_MODULE_NAME}.js`,
+    ),
+    `window['__@backstage/discovered__'] = { modules: [${requirePackageScript}] };`,
+  );
+}
+
+export async function createDetectedModulesEntryPoint(options: {
+  config: Config;
+  targetPath: string;
+  watch?: () => void;
+}): Promise<string[]> {
+  const { config, watch, targetPath } = options;
+
+  const detectionConfig = readPackageDetectionConfig(config);
+  if (!detectionConfig) {
+    return [];
+  }
+
+  if (watch) {
+    const watcher = chokidar.watch(resolvePath(targetPath, 'package.json'));
+
+    watcher.on('change', async () => {
+      await writeDetectedPackagesModule(
+        await detectPackages(targetPath, detectionConfig),
+      );
+      watch();
+    });
+  }
+
+  await writeDetectedPackagesModule(
+    await detectPackages(targetPath, detectionConfig),
+  );
+
+  return [DETECTED_MODULES_MODULE_NAME];
+}

packages/cli/src/lib/bundler/server.ts

@@ -14,27 +14,31 @@
  * limitations under the License.
  */
 
+import { PackageGraph } from '@backstage/cli-node';
+import { AppConfig } from '@backstage/config';
+import chalk from 'chalk';
 import fs from 'fs-extra';
+import uniq from 'lodash/uniq';
+import openBrowser from 'react-dev-utils/openBrowser';
 import webpack from 'webpack';
 import WebpackDevServer from 'webpack-dev-server';
-import openBrowser from 'react-dev-utils/openBrowser';
-import uniq from 'lodash/uniq';
 
-import { createConfig, resolveBaseUrl } from './config';
-import { ServeOptions } from './types';
-import { resolveBundlingPaths } from './paths';
-import { paths as libPaths } from '../../lib/paths';
-import { loadCliConfig } from '../config';
-import chalk from 'chalk';
-import { AppConfig } from '@backstage/config';
-import { PackageGraph } from '@backstage/cli-node';
-import { Lockfile } from '../versioning';
 import {
   forbiddenDuplicatesFilter,
   includedFilter,
 } from '../../commands/versions/lint';
+import { paths as libPaths } from '../../lib/paths';
+import { loadCliConfig } from '../config';
+import { Lockfile } from '../versioning';
+import { createConfig, resolveBaseUrl } from './config';
+import { createDetectedModulesEntryPoint } from './packageDetection';
+import { resolveBundlingPaths } from './paths';
+import { ServeOptions } from './types';
 
 export async function serveBundle(options: ServeOptions) {
+  const paths = resolveBundlingPaths(options);
+  const targetPkg = await fs.readJson(paths.targetPackageJson);
+
   if (options.verifyVersions) {
     const lockfile = await Lockfile.load(
       libPaths.resolveTargetRoot('yarn.lock'),
@@ -115,10 +119,16 @@ export async function serveBundle(options: ServeOptions) {
     Number(url.port) ||
     (url.protocol === 'https:' ? 443 : 80);
 
-  const paths = resolveBundlingPaths(options);
-  const pkgPath = paths.targetPackageJson;
-  const pkg = await fs.readJson(pkgPath);
+  const detectedModulesEntryPoint = await createDetectedModulesEntryPoint({
+    config: fullConfig,
+    targetPath: paths.targetPath,
+    watch() {
+      server?.invalidate();
+    },
+  });
+
   const config = await createConfig(paths, {
+    ...options,
     checksEnabled: options.checksEnabled,
     isDev: true,
     baseUrl: url,
@@ -126,6 +136,7 @@ export async function serveBundle(options: ServeOptions) {
     getFrontendAppConfigs: () => {
       return latestFrontendAppConfigs;
     },
+    additionalEntryPoints: detectedModulesEntryPoint,
   });
 
   const compiler = webpack(config);
@@ -160,7 +171,7 @@ export async function serveBundle(options: ServeOptions) {
           : false,
       host,
       port,
-      proxy: pkg.proxy,
+      proxy: targetPkg.proxy,
       // When the dev server is behind a proxy, the host and public hostname differ
       allowedHosts: [url.hostname],
       client: {

packages/cli/src/lib/bundler/types.ts

@@ -25,6 +25,7 @@ export type BundlingOptions = {
   getFrontendAppConfigs(): AppConfig[];
   baseUrl: URL;
   parallelism?: number;
+  additionalEntryPoints?: string[];
 };
 
 export type ServeOptions = BundlingPathsOptions & {
@@ -41,6 +42,7 @@ export type BuildOptions = BundlingPathsOptions & {
   schema?: ConfigSchema;
   frontendConfig: Config;
   frontendAppConfigs: AppConfig[];
+  fullConfig: Config;
 };
 
 export type BackendBundlingOptions = {

packages/frontend-app-api/src/createApp.tsx

@@ -32,6 +32,7 @@ import {
 } from './wiring/parameters';
 import { RoutingProvider } from './routing/RoutingContext';
 import { RouteRef } from '@backstage/core-plugin-api';
+import { getAvailablePlugins } from './wiring/discovery';
 
 /** @public */
 export function createApp(options: { plugins: BackstagePlugin[] }): {
@@ -40,11 +41,12 @@ export function createApp(options: { plugins: BackstagePlugin[] }): {
   const appConfig = ConfigReader.fromConfigs(process.env.APP_CONFIG as any);
 
   const builtinExtensions = [CoreRouter];
+  const discoveredPlugins = getAvailablePlugins();
 
   // pull in default extension instance from discovered packages
   // apply config to adjust default extension instances and add more
   const extensionParams = mergeExtensionParameters({
-    sources: options.plugins,
+    sources: [...options.plugins, ...discoveredPlugins],
     builtinExtensions,
     parameters: readAppExtensionParameters(appConfig),
   });

packages/frontend-app-api/src/wiring/discovery.test.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createPlugin } from '@backstage/frontend-plugin-api';
+import { getAvailablePlugins } from './discovery';
+
+const globalSpy = jest.fn();
+Object.defineProperty(global, '__@backstage/discovered__', {
+  get: globalSpy,
+});
+
+describe('getAvailablePlugins', () => {
+  afterEach(jest.resetAllMocks);
+
+  it('should discover nothing with undefined global', () => {
+    expect(getAvailablePlugins()).toEqual([]);
+  });
+
+  it('should discover nothing with empty global', () => {
+    globalSpy.mockReturnValue({
+      modules: [],
+    });
+    expect(getAvailablePlugins()).toEqual([]);
+  });
+
+  it('should discover a plugin', () => {
+    const testPlugin = createPlugin({ id: 'test' });
+    globalSpy.mockReturnValue({
+      modules: [
+        {
+          module: {
+            testPlugin,
+          },
+        },
+      ],
+    });
+    expect(getAvailablePlugins()).toEqual([testPlugin]);
+  });
+
+  it('should ignore garbage', () => {
+    const testPlugin = createPlugin({ id: 'test' });
+    globalSpy.mockReturnValue({
+      modules: [
+        {
+          module: {
+            testPlugin,
+            a: 'a',
+            b: null,
+            c: undefined,
+            d: Symbol('wat'),
+            e: () => {},
+            f: [],
+            g: {},
+            h: class {},
+            i: NaN,
+            j: Infinity,
+            k: -Infinity,
+            l: new Date(),
+            m: new RegExp('wat'),
+            n: new Error('wat'),
+            o: new Map(),
+            p: new Set(),
+            q: new WeakMap(),
+            r: new WeakSet(),
+            s: new ArrayBuffer(1),
+            t: new DataView(new ArrayBuffer(1)),
+            u: false,
+            v: true,
+            w: 0,
+            x: 1,
+            y: -1,
+            z: '',
+          },
+        },
+      ],
+    });
+    expect(getAvailablePlugins()).toEqual([testPlugin]);
+  });
+
+  it('should discover multiple plugins', () => {
+    const test1Plugin = createPlugin({ id: 'test1' });
+    const test2Plugin = createPlugin({ id: 'test2' });
+    const test3Plugin = createPlugin({ id: 'test3' });
+    globalSpy.mockReturnValue({
+      modules: [
+        {
+          module: {
+            test1Plugin,
+            test2Plugin,
+          },
+        },
+        {
+          module: {
+            test3Plugin,
+          },
+        },
+      ],
+    });
+    expect(getAvailablePlugins()).toEqual([
+      test1Plugin,
+      test2Plugin,
+      test3Plugin,
+    ]);
+  });
+});

packages/frontend-app-api/src/wiring/discovery.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { BackstagePlugin } from '@backstage/frontend-plugin-api';
+
+interface DiscoveryGlobal {
+  modules: Array<{ name: string; module: object }>;
+}
+
+/**
+ * @public
+ */
+export function getAvailablePlugins(): BackstagePlugin[] {
+  const discovered = (
+    window as { '__@backstage/discovered__'?: DiscoveryGlobal }
+  )['__@backstage/discovered__'];
+
+  return (
+    discovered?.modules.flatMap(({ module: mod }) =>
+      Object.values(mod).filter(isBackstagePlugin),
+    ) ?? []
+  );
+}
+
+function isBackstagePlugin(obj: unknown): obj is BackstagePlugin {
+  if (obj !== null && typeof obj === 'object' && '$$type' in obj) {
+    return obj.$$type === 'backstage-plugin';
+  }
+  return false;
+}

yarn.lock

@@ -19749,6 +19749,23 @@ __metadata:
   languageName: node
   linkType: hard
 
+"app-next-example-plugin@workspace:^, app-next-example-plugin@workspace:packages/app-next-example-plugin":
+  version: 0.0.0-use.local
+  resolution: "app-next-example-plugin@workspace:packages/app-next-example-plugin"
+  dependencies:
+    "@backstage/cli": "workspace:^"
+    "@backstage/core-components": "workspace:^"
+    "@backstage/frontend-plugin-api": "workspace:^"
+    "@material-ui/icons": ^4.9.1
+    cross-fetch: ^3.1.5
+    msw: ^1.0.0
+  peerDependencies:
+    react: ^16.13.1 || ^17.0.0
+    react-dom: ^16.13.1 || ^17.0.0
+    react-router-dom: 6.0.0-beta.0 || ^6.3.0
+  languageName: unknown
+  linkType: soft
+
 "append-field@npm:^1.0.0":
   version: 1.0.0
   resolution: "append-field@npm:1.0.0"
@@ -25634,6 +25651,7 @@ __metadata:
     "@types/react": "*"
     "@types/react-dom": "*"
     "@types/zen-observable": ^0.8.0
+    app-next-example-plugin: "workspace:^"
     cross-env: ^7.0.0
     cypress: ^10.0.0
     eslint-plugin-cypress: ^2.10.3