diff --git a/.changeset/cli-node-parallel-helpers.md b/.changeset/cli-node-parallel-helpers.md new file mode 100644 index 0000000000..d4cea5c3c5 --- /dev/null +++ b/.changeset/cli-node-parallel-helpers.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli-node': minor +--- + +Added parallel worker utilities: `runParallelWorkers`, `runWorkerQueueThreads`, `runWorkerThreads`, `parseParallelismOption`, and `getEnvironmentParallelism`. These were moved from the `@backstage/cli` internal code. diff --git a/packages/cli-node/report.api.md b/packages/cli-node/report.api.md index 55d2443643..960bbf9f59 100644 --- a/packages/cli-node/report.api.md +++ b/packages/cli-node/report.api.md @@ -86,6 +86,9 @@ export interface BackstagePackageJson { version: string; } +// @public +export function getEnvironmentParallelism(): number; + // @public export class GitUtils { static listChangedFiles(ref: string): Promise; @@ -200,4 +203,56 @@ export class PackageRoles { static getRoleFromPackage(pkgJson: unknown): PackageRole | undefined; static getRoleInfo(role: string): PackageRoleInfo; } + +// @public +export type ParallelismOption = boolean | string | number | null | undefined; + +// @public +export type ParallelWorkerOptions = { + parallelismFactor?: number; + parallelismSetting?: ParallelismOption; + items: Iterable; + worker: (item: TItem) => Promise; +}; + +// @public +export function parseParallelismOption(parallel: ParallelismOption): number; + +// @public +export function runParallelWorkers( + options: ParallelWorkerOptions, +): Promise; + +// @public +export function runWorkerQueueThreads( + options: WorkerQueueThreadsOptions, +): Promise; + +// @public +export function runWorkerThreads( + options: WorkerThreadsOptions, +): Promise; + +// @public +export type WorkerQueueThreadsOptions = { + items: Iterable; + workerFactory: ( + data: TData, + ) => + | ((item: TItem) => Promise) + | Promise<(item: TItem) => Promise>; + workerData?: TData; + threadCount?: number; +}; + +// @public +export type WorkerThreadsOptions = { + worker: ( + data: TData, + sendMessage: (message: TMessage) => void, + ) => Promise; + workerData?: TData; + threadCount?: number; + onMessage?: (message: TMessage) => void; +}; ``` diff --git a/packages/cli-node/src/index.ts b/packages/cli-node/src/index.ts index 1d3f3ec2ed..6d1c519b9e 100644 --- a/packages/cli-node/src/index.ts +++ b/packages/cli-node/src/index.ts @@ -22,4 +22,5 @@ export * from './git'; export * from './monorepo'; +export * from './parallel'; export * from './roles'; diff --git a/packages/cli-node/src/parallel/index.ts b/packages/cli-node/src/parallel/index.ts new file mode 100644 index 0000000000..60cd2e28b2 --- /dev/null +++ b/packages/cli-node/src/parallel/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type { ParallelismOption, ParallelWorkerOptions } from './parallel'; +export { + parseParallelismOption, + getEnvironmentParallelism, + runParallelWorkers, + runWorkerQueueThreads, + runWorkerThreads, +} from './parallel'; +export type { + WorkerQueueThreadsOptions, + WorkerThreadsOptions, +} from './parallel'; diff --git a/packages/cli/src/lib/parallel.test.ts b/packages/cli-node/src/parallel/parallel.test.ts similarity index 100% rename from packages/cli/src/lib/parallel.test.ts rename to packages/cli-node/src/parallel/parallel.test.ts diff --git a/packages/cli/src/lib/parallel.ts b/packages/cli-node/src/parallel/parallel.ts similarity index 92% rename from packages/cli/src/lib/parallel.ts rename to packages/cli-node/src/parallel/parallel.ts index a4ca5dd13d..26c5803a93 100644 --- a/packages/cli/src/lib/parallel.ts +++ b/packages/cli-node/src/parallel/parallel.ts @@ -22,8 +22,21 @@ const defaultParallelism = Math.ceil(os.cpus().length / 2); const PARALLEL_ENV_VAR = 'BACKSTAGE_CLI_BUILD_PARALLEL'; +/** + * Options for configuring parallelism. Can be a boolean, string, number, null, or undefined. + * - Boolean: true uses default parallelism (half of CPUs), false uses 1 + * - Number: explicit worker count + * - String: parsed as boolean or integer (e.g. "true", "4") + * + * @public + */ export type ParallelismOption = boolean | string | number | null | undefined; +/** + * Parses a parallelism option value into a concrete worker count. + * + * @public + */ export function parseParallelismOption(parallel: ParallelismOption): number { if (parallel === undefined || parallel === null) { return defaultParallelism; @@ -51,11 +64,21 @@ export function parseParallelismOption(parallel: ParallelismOption): number { ); } +/** + * Returns the parallelism value from the BACKSTAGE_CLI_BUILD_PARALLEL environment variable. + * + * @public + */ export function getEnvironmentParallelism() { return parseParallelismOption(process.env[PARALLEL_ENV_VAR]); } -type ParallelWorkerOptions = { +/** + * Options for runParallelWorkers. + * + * @public + */ +export type ParallelWorkerOptions = { /** * Decides the number of parallel workers by multiplying * this with the configured parallelism, which defaults to 4. @@ -68,6 +91,11 @@ type ParallelWorkerOptions = { worker: (item: TItem) => Promise; }; +/** + * Runs items through a worker function in parallel across multiple async workers. + * + * @public + */ export async function runParallelWorkers( options: ParallelWorkerOptions, ) { @@ -119,6 +147,11 @@ type WorkerThreadMessage = message: unknown; }; +/** + * Options for runWorkerQueueThreads. + * + * @public + */ export type WorkerQueueThreadsOptions = { /** The items to process */ items: Iterable; @@ -149,6 +182,8 @@ export type WorkerQueueThreadsOptions = { /** * Spawns one or more worker threads using the `worker_threads` module. * Each thread processes one item at a time from the provided `options.items`. + * + * @public */ export async function runWorkerQueueThreads( options: WorkerQueueThreadsOptions, @@ -250,6 +285,11 @@ function workerQueueThread( ); } +/** + * Options for runWorkerThreads. + * + * @public + */ export type WorkerThreadsOptions = { /** * A function that is called by each worker thread to produce a result. @@ -277,6 +317,8 @@ export type WorkerThreadsOptions = { /** * Spawns one or more worker threads using the `worker_threads` module. + * + * @public */ export async function runWorkerThreads( options: WorkerThreadsOptions, diff --git a/packages/cli/src/modules/build/commands/repo/build.ts b/packages/cli/src/modules/build/commands/repo/build.ts index e64fe9eff0..22c72c91dd 100644 --- a/packages/cli/src/modules/build/commands/repo/build.ts +++ b/packages/cli/src/modules/build/commands/repo/build.ts @@ -23,8 +23,8 @@ import { BackstagePackage, PackageGraph, PackageRoles, + runParallelWorkers, } from '@backstage/cli-node'; -import { runParallelWorkers } from '../../../../lib/parallel'; import { buildFrontend } from '../../lib/buildFrontend'; import { buildBackend } from '../../lib/buildBackend'; import { createScriptOptionsParser } from '../../../../lib/optionsParser'; diff --git a/packages/cli/src/modules/build/lib/buildBackend.ts b/packages/cli/src/modules/build/lib/buildBackend.ts index f377d068bd..ddf9b1f667 100644 --- a/packages/cli/src/modules/build/lib/buildBackend.ts +++ b/packages/cli/src/modules/build/lib/buildBackend.ts @@ -19,9 +19,8 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'node:path'; import * as tar from 'tar'; import { createDistWorkspace } from './packager'; -import { getEnvironmentParallelism } from '../../../lib/parallel'; import { buildPackage, Output } from './builder'; -import { PackageGraph } from '@backstage/cli-node'; +import { PackageGraph, getEnvironmentParallelism } from '@backstage/cli-node'; const BUNDLE_FILE = 'bundle.tar.gz'; const SKELETON_FILE = 'skeleton.tar.gz'; diff --git a/packages/cli/src/modules/build/lib/buildFrontend.ts b/packages/cli/src/modules/build/lib/buildFrontend.ts index 7b0955a519..bb00de7951 100644 --- a/packages/cli/src/modules/build/lib/buildFrontend.ts +++ b/packages/cli/src/modules/build/lib/buildFrontend.ts @@ -17,9 +17,11 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'node:path'; import { buildBundle, getModuleFederationRemoteOptions } from './bundler'; -import { getEnvironmentParallelism } from '../../../lib/parallel'; +import { + BackstagePackageJson, + getEnvironmentParallelism, +} from '@backstage/cli-node'; import { loadCliConfig } from '../../config/lib/config'; -import { BackstagePackageJson } from '@backstage/cli-node'; interface BuildAppOptions { targetDir: string; diff --git a/packages/cli/src/modules/build/lib/builder/packager.ts b/packages/cli/src/modules/build/lib/builder/packager.ts index 12017b694c..b0a34780d7 100644 --- a/packages/cli/src/modules/build/lib/builder/packager.ts +++ b/packages/cli/src/modules/build/lib/builder/packager.ts @@ -21,8 +21,7 @@ import { relative as relativePath, resolve as resolvePath } from 'node:path'; import { paths } from '../../../../lib/paths'; import { makeRollupConfigs } from './config'; import { BuildOptions, Output } from './types'; -import { PackageRoles } from '@backstage/cli-node'; -import { runParallelWorkers } from '../../../../lib/parallel'; +import { PackageRoles, runParallelWorkers } from '@backstage/cli-node'; export function formatErrorMessage(error: any) { let msg = ''; diff --git a/packages/cli/src/modules/build/lib/packager/createDistWorkspace.ts b/packages/cli/src/modules/build/lib/packager/createDistWorkspace.ts index 0f096a3127..b53bca62a6 100644 --- a/packages/cli/src/modules/build/lib/packager/createDistWorkspace.ts +++ b/packages/cli/src/modules/build/lib/packager/createDistWorkspace.ts @@ -41,8 +41,8 @@ import { PackageRoles, PackageGraph, PackageGraphNode, + runParallelWorkers, } from '@backstage/cli-node'; -import { runParallelWorkers } from '../../../../lib/parallel'; import { createTypeDistProject } from '../../../../lib/typeDistProject'; // These packages aren't safe to pack in parallel since the CLI depends on them diff --git a/packages/cli/src/modules/lint/commands/repo/lint.ts b/packages/cli/src/modules/lint/commands/repo/lint.ts index 458ccf1a63..0e17f97dd2 100644 --- a/packages/cli/src/modules/lint/commands/repo/lint.ts +++ b/packages/cli/src/modules/lint/commands/repo/lint.ts @@ -23,9 +23,9 @@ import { PackageGraph, BackstagePackageJson, Lockfile, + runWorkerQueueThreads, } from '@backstage/cli-node'; import { paths } from '../../../../lib/paths'; -import { runWorkerQueueThreads } from '../../../../lib/parallel'; import { createScriptOptionsParser } from '../../../../lib/optionsParser'; import { SuccessCache } from '../../../../lib/cache/SuccessCache'; diff --git a/packages/cli/src/modules/migrate/commands/versions/bump.ts b/packages/cli/src/modules/migrate/commands/versions/bump.ts index ab65f8c604..06658cea5f 100644 --- a/packages/cli/src/modules/migrate/commands/versions/bump.ts +++ b/packages/cli/src/modules/migrate/commands/versions/bump.ts @@ -33,7 +33,7 @@ import { mapDependencies, YarnInfoInspectData, } from '../../../../lib/versioning'; -import { runParallelWorkers } from '../../../../lib/parallel'; +import { runParallelWorkers } from '@backstage/cli-node'; import { getManifestByReleaseLine, getManifestByVersion,