cli: switch out some unbounded parallelism to use worker helper

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/cli/src/commands/repo/build.ts

@@ -20,8 +20,9 @@ import { relative as relativePath } from 'path';
 import { buildPackages, getOutputsForRole } from '../../lib/builder';
 import { PackageGraph } from '../../lib/monorepo';
 import { ExtendedPackage } from '../../lib/monorepo/PackageGraph';
+import { runParallelWorkers } from '../../lib/parallel';
 import { paths } from '../../lib/paths';
-import { getRoleInfo } from '../../lib/role';
+import { detectRoleFromPackage } from '../../lib/role';
 import { buildApp } from '../build/buildApp';
 import { buildBackend } from '../build/buildBackend';
 
@@ -76,26 +77,30 @@ function createScriptOptionsParser(anyCmd: Command, commandPath: string[]) {
 
 export async function command(cmd: Command): Promise<void> {
   const packages = await PackageGraph.listTargetPackages();
-  const bundledPackages = new Array<ExtendedPackage>();
+  const apps = new Array<ExtendedPackage>();
+  const backends = new Array<ExtendedPackage>();
 
   const parseBuildScript = createScriptOptionsParser(cmd, ['script', 'build']);
 
   const options = packages.flatMap(pkg => {
-    const role = pkg.packageJson.backstage?.role;
+    const role =
+      pkg.packageJson.backstage?.role ?? detectRoleFromPackage(pkg.packageJson);
     if (!role) {
       console.warn(`Ignored ${pkg.packageJson.name} because it has no role`);
       return [];
     }
 
+    if (role === 'app') {
+      apps.push(pkg);
+      return [];
+    } else if (role === 'backend') {
+      backends.push(pkg);
+      return [];
+    }
+
     const outputs = getOutputsForRole(role);
     if (outputs.size === 0) {
-      if (getRoleInfo(role).output.includes('bundle')) {
-        bundledPackages.push(pkg);
-      } else {
-        console.warn(
-          `Ignored ${pkg.packageJson.name} because it has no output`,
-        );
-      }
+      console.warn(`Ignored ${pkg.packageJson.name} because it has no output`);
       return [];
     }
 
@@ -120,13 +125,11 @@ export async function command(cmd: Command): Promise<void> {
   await buildPackages(options);
 
   if (cmd.all) {
-    const apps = bundledPackages.filter(
-      pkg => pkg.packageJson.backstage?.role === 'app',
-    );
-
     console.log('Building apps');
-    await Promise.all(
-      apps.map(async pkg => {
+    await runParallelWorkers({
+      items: apps,
+      parallelismFactor: 1 / 2,
+      worker: async pkg => {
         const buildOptions = parseBuildScript(pkg.packageJson.scripts?.build);
         if (!buildOptions) {
           console.warn(
@@ -139,15 +142,14 @@ export async function command(cmd: Command): Promise<void> {
           configPaths: (buildOptions.config as string[]) ?? [],
           writeStats: Boolean(buildOptions.stats),
         });
-      }),
-    );
+      },
+    });
 
     console.log('Building backends');
-    const backends = bundledPackages.filter(
-      pkg => pkg.packageJson.backstage?.role === 'backend',
-    );
-    await Promise.all(
-      backends.map(async pkg => {
+    await runParallelWorkers({
+      items: backends,
+      parallelismFactor: 1 / 2,
+      worker: async pkg => {
         const buildOptions = parseBuildScript(pkg.packageJson.scripts?.build);
         if (!buildOptions) {
           console.warn(
@@ -159,7 +161,7 @@ export async function command(cmd: Command): Promise<void> {
           targetDir: pkg.dir,
           skipBuildDependencies: true,
         });
-      }),
-    );
+      },
+    });
   }
 }

packages/cli/src/lib/builder/packager.ts

@@ -23,6 +23,7 @@ import { makeRollupConfigs } from './config';
 import { BuildOptions, Output } from './types';
 import { buildTypeDefinitions } from './buildTypeDefinitions';
 import { getRoleInfo } from '../role';
+import { runParallelWorkers } from '../parallel';
 
 export function formatErrorMessage(error: any) {
   let msg = '';
@@ -128,7 +129,7 @@ export const buildPackages = async (options: BuildOptions[]) => {
     options.map(({ targetDir }) => fs.remove(resolvePath(targetDir!, 'dist'))),
   );
 
-  const buildTasks = rollupConfigs.flat().map(rollupBuild);
+  const buildTasks = rollupConfigs.flat().map(opts => () => rollupBuild(opts));
 
   const typeDefinitionTargetDirs = options
     .filter(
@@ -138,10 +139,14 @@ export const buildPackages = async (options: BuildOptions[]) => {
     .map(_ => _.targetDir!);
 
   if (typeDefinitionTargetDirs.length > 0) {
-    buildTasks.push(buildTypeDefinitions(typeDefinitionTargetDirs));
+    // Make sure this one is started first
+    buildTasks.unshift(() => buildTypeDefinitions(typeDefinitionTargetDirs));
   }
 
-  await Promise.all(buildTasks);
+  await runParallelWorkers({
+    items: buildTasks,
+    worker: async task => task(),
+  });
 };
 
 export function getOutputsForRole(role: string): Set<Output> {

packages/cli/src/lib/parallel.ts

@@ -54,9 +54,11 @@ export function getEnvironmentParallelism() {
 type ParallelWorkerOptions<TItem> = {
   /**
    * Decides the number of parallel workers by multiplying
-   * this with the configured parallelism, which defaults to 4
+   * this with the configured parallelism, which defaults to 4.
+   *
+   * Defaults to 1.
    */
-  parallelismFactor: number;
+  parallelismFactor?: number;
   parallelismSetting?: ParallelismOption;
   items: Iterable<TItem>;
   worker: (item: TItem) => Promise<void>;
@@ -65,7 +67,7 @@ type ParallelWorkerOptions<TItem> = {
 export async function runParallelWorkers<TItem>(
   options: ParallelWorkerOptions<TItem>,
 ) {
-  const { parallelismFactor, parallelismSetting, items, worker } = options;
+  const { parallelismFactor = 1, parallelismSetting, items, worker } = options;
   const parallelism = parallelismSetting
     ? parseParallelismOption(parallelismSetting)
     : getEnvironmentParallelism();