Merge branch 'master' into package-workspaces

Signed-off-by: Gabriel Dugny <gabriel.dugny@believe.com>

# Conflicts:
#	packages/cli-node/src/pacman/yarn/Yarn.test.ts
#	packages/cli-node/src/pacman/yarn/Yarn.ts
This commit is contained in:
Gabriel Dugny
2026-02-25 09:11:36 +01:00
624 changed files with 10070 additions and 1926 deletions
+11
View File
@@ -1,5 +1,16 @@
# @backstage/cli-node
## 0.2.19-next.0
### Patch Changes
- 06c2015: Added `runConcurrentTasks` and `runWorkerQueueThreads` utilities, moved from the `@backstage/cli` internal code.
- 70fc178: Migrated from deprecated `findPaths` to `targetPaths` and `findOwnPaths` from `@backstage/cli-common`.
- Updated dependencies
- @backstage/cli-common@0.2.0-next.0
- @backstage/errors@1.2.7
- @backstage/types@1.2.2
## 0.2.18
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@backstage/cli-node",
"version": "0.2.18",
"version": "0.2.19-next.0",
"description": "Node.js library for Backstage CLIs",
"backstage": {
"role": "node-library"
+30
View File
@@ -86,6 +86,13 @@ export interface BackstagePackageJson {
version: string;
}
// @public
export type ConcurrentTasksOptions<TItem> = {
concurrencyFactor?: number;
items: Iterable<TItem>;
worker: (item: TItem) => Promise<void>;
};
// @public
export class GitUtils {
static listChangedFiles(ref: string): Promise<string[]>;
@@ -200,4 +207,27 @@ export class PackageRoles {
static getRoleFromPackage(pkgJson: unknown): PackageRole | undefined;
static getRoleInfo(role: string): PackageRoleInfo;
}
// @public
export function runConcurrentTasks<TItem>(
options: ConcurrentTasksOptions<TItem>,
): Promise<void>;
// @public
export function runWorkerQueueThreads<TItem, TResult, TContext>(
options: WorkerQueueThreadsOptions<TItem, TResult, TContext>,
): Promise<{
results: TResult[];
}>;
// @public
export type WorkerQueueThreadsOptions<TItem, TResult, TContext> = {
items: Iterable<TItem>;
workerFactory: (
context: TContext,
) =>
| ((item: TItem) => Promise<TResult>)
| Promise<(item: TItem) => Promise<TResult>>;
context?: TContext;
};
```
@@ -0,0 +1,70 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'node:os';
const defaultConcurrency = Math.max(Math.ceil(os.cpus().length / 2), 1);
const CONCURRENCY_ENV_VAR = 'BACKSTAGE_CLI_CONCURRENCY';
const DEPRECATED_CONCURRENCY_ENV_VAR = 'BACKSTAGE_CLI_BUILD_PARALLEL';
type ConcurrencyOption = boolean | string | number | null | undefined;
function parseConcurrencyOption(value: ConcurrencyOption): number {
if (value === undefined || value === null) {
return defaultConcurrency;
} else if (typeof value === 'boolean') {
return value ? defaultConcurrency : 1;
} else if (typeof value === 'number' && Number.isInteger(value)) {
if (value < 1) {
return 1;
}
return value;
} else if (typeof value === 'string') {
if (value === 'true') {
return parseConcurrencyOption(true);
} else if (value === 'false') {
return parseConcurrencyOption(false);
}
const parsed = Number(value);
if (Number.isInteger(parsed)) {
return parseConcurrencyOption(parsed);
}
}
throw Error(
`Concurrency option value '${value}' is not a boolean or integer`,
);
}
let hasWarnedDeprecation = false;
/** @internal */
export function getEnvironmentConcurrency() {
if (process.env[CONCURRENCY_ENV_VAR] !== undefined) {
return parseConcurrencyOption(process.env[CONCURRENCY_ENV_VAR]);
}
if (process.env[DEPRECATED_CONCURRENCY_ENV_VAR] !== undefined) {
if (!hasWarnedDeprecation) {
hasWarnedDeprecation = true;
console.warn(
`The ${DEPRECATED_CONCURRENCY_ENV_VAR} environment variable is deprecated, use ${CONCURRENCY_ENV_VAR} instead`,
);
}
return parseConcurrencyOption(process.env[DEPRECATED_CONCURRENCY_ENV_VAR]);
}
return defaultConcurrency;
}
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { findPaths } from '@backstage/cli-common';
/* eslint-disable-next-line no-restricted-syntax */
export const paths = findPaths(__dirname);
export type { ConcurrentTasksOptions } from './runConcurrentTasks';
export { runConcurrentTasks } from './runConcurrentTasks';
export type { WorkerQueueThreadsOptions } from './runWorkerQueueThreads';
export { runWorkerQueueThreads } from './runWorkerQueueThreads';
@@ -0,0 +1,144 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { runConcurrentTasks } from './runConcurrentTasks';
describe('runConcurrentTasks', () => {
afterEach(() => {
delete process.env.BACKSTAGE_CLI_CONCURRENCY;
});
it('executes work in parallel', async () => {
const started = new Array<number>();
const done = new Array<number>();
const waiting = new Array<() => void>();
process.env.BACKSTAGE_CLI_CONCURRENCY = '4';
const work = runConcurrentTasks({
items: [0, 1, 2, 3, 4],
concurrencyFactor: 0.5, // 2 at a time
worker: async item => {
started.push(item);
await new Promise<void>(resolve => {
waiting[item] = resolve;
});
done.push(item);
},
});
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1]);
expect(done).toEqual([]);
waiting[0]();
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1, 2]);
expect(done).toEqual([0]);
waiting[1]();
waiting[2]();
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1, 2, 3, 4]);
expect(done).toEqual([0, 1, 2]);
waiting[3]();
waiting[4]();
await work;
expect(done).toEqual([0, 1, 2, 3, 4]);
});
it('executes work sequentially', async () => {
const started = new Array<number>();
const done = new Array<number>();
const waiting = new Array<() => void>();
const work = runConcurrentTasks({
items: [0, 1, 2, 3, 4],
concurrencyFactor: 0, // 1 at a time
worker: async item => {
started.push(item);
await new Promise<void>(resolve => {
waiting[item] = resolve;
});
done.push(item);
},
});
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0]);
expect(done).toEqual([]);
waiting[0]();
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1]);
expect(done).toEqual([0]);
waiting[1]();
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1, 2]);
waiting[2]();
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1, 2, 3]);
waiting[3]();
await new Promise(resolve => setTimeout(resolve));
expect(started).toEqual([0, 1, 2, 3, 4]);
waiting[4]();
await work;
expect(done).toEqual([0, 1, 2, 3, 4]);
});
it('returns void', async () => {
const result = await runConcurrentTasks({
items: [1, 2, 3],
worker: async () => {},
});
expect(result).toBeUndefined();
});
it('defaults to environment concurrency', async () => {
const started = new Array<number>();
process.env.BACKSTAGE_CLI_CONCURRENCY = '2';
await runConcurrentTasks({
items: [0, 1],
worker: async item => {
started.push(item);
},
});
expect(started).toEqual([0, 1]);
});
it('supports the deprecated BACKSTAGE_CLI_BUILD_PARALLEL with a warning', async () => {
const warnSpy = jest.spyOn(console, 'warn').mockImplementation();
process.env.BACKSTAGE_CLI_BUILD_PARALLEL = '2';
await runConcurrentTasks({
items: [0, 1],
worker: async () => {},
});
expect(warnSpy).toHaveBeenCalledWith(
'The BACKSTAGE_CLI_BUILD_PARALLEL environment variable is deprecated, use BACKSTAGE_CLI_CONCURRENCY instead',
);
delete process.env.BACKSTAGE_CLI_BUILD_PARALLEL;
warnSpy.mockRestore();
});
});
@@ -0,0 +1,62 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getEnvironmentConcurrency } from './concurrency';
/**
* Options for {@link runConcurrentTasks}.
*
* @public
*/
export type ConcurrentTasksOptions<TItem> = {
/**
* Decides the number of concurrent workers by multiplying
* this with the configured concurrency.
*
* Defaults to 1.
*/
concurrencyFactor?: number;
items: Iterable<TItem>;
worker: (item: TItem) => Promise<void>;
};
/**
* Runs items through a worker function concurrently across multiple async workers.
*
* @public
*/
export async function runConcurrentTasks<TItem>(
options: ConcurrentTasksOptions<TItem>,
): Promise<void> {
const { concurrencyFactor = 1, items, worker } = options;
const concurrency = getEnvironmentConcurrency();
const sharedIterator = items[Symbol.iterator]();
const sharedIterable = {
[Symbol.iterator]: () => sharedIterator,
};
const workerCount = Math.max(Math.floor(concurrencyFactor * concurrency), 1);
await Promise.all(
Array(workerCount)
.fill(0)
.map(async () => {
for (const value of sharedIterable) {
await worker(value);
}
}),
);
}
@@ -0,0 +1,42 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { runWorkerQueueThreads } from './runWorkerQueueThreads';
describe('runWorkerQueueThreads', () => {
it('should execute work in parallel', async () => {
const sharedData = new SharedArrayBuffer(10);
const sharedView = new Uint8Array(sharedData);
const { results } = await runWorkerQueueThreads({
context: sharedData,
items: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
workerFactory: (data: SharedArrayBuffer) => {
const view = new Uint8Array(data);
return async (i: number) => {
view[i] = 10 + i;
return 20 + i;
};
},
});
expect(Array.from(sharedView)).toEqual([
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
]);
expect(results).toEqual([20, 21, 22, 23, 24, 25, 26, 27, 28, 29]);
});
});
@@ -0,0 +1,175 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ErrorLike } from '@backstage/errors';
import { Worker } from 'node:worker_threads';
import { getEnvironmentConcurrency } from './concurrency';
type WorkerThreadMessage =
| {
type: 'done';
}
| {
type: 'item';
index: number;
item: unknown;
}
| {
type: 'start';
}
| {
type: 'result';
index: number;
result: unknown;
}
| {
type: 'error';
error: ErrorLike;
};
/**
* Options for {@link runWorkerQueueThreads}.
*
* @public
*/
export type WorkerQueueThreadsOptions<TItem, TResult, TContext> = {
/** The items to process */
items: Iterable<TItem>;
/**
* A function that will be called within each worker thread at startup,
* which should return the worker function that will be called for each item.
*
* This function must be defined as an arrow function or using the
* function keyword, and must be entirely self contained, not referencing
* any variables outside of its scope. This is because the function source
* is stringified and evaluated in the worker thread.
*
* To pass data to the worker, use the `context` option and `items`, but
* note that they are both copied by value into the worker thread, except for
* types that are explicitly shareable across threads, such as `SharedArrayBuffer`.
*/
workerFactory: (
context: TContext,
) =>
| ((item: TItem) => Promise<TResult>)
| Promise<(item: TItem) => Promise<TResult>>;
/** Context data supplied to each worker factory */
context?: TContext;
};
/**
* Spawns one or more worker threads using the `worker_threads` module.
* Each thread processes one item at a time from the provided `options.items`.
*
* @public
*/
export async function runWorkerQueueThreads<TItem, TResult, TContext>(
options: WorkerQueueThreadsOptions<TItem, TResult, TContext>,
): Promise<{ results: TResult[] }> {
const items = Array.from(options.items);
const workerFactory = options.workerFactory;
const workerData = options.context;
const threadCount = Math.min(getEnvironmentConcurrency(), items.length);
const iterator = items[Symbol.iterator]();
const results = new Array<TResult>();
let itemIndex = 0;
await Promise.all(
Array(threadCount)
.fill(0)
.map(async () => {
const thread = new Worker(`(${workerQueueThread})(${workerFactory})`, {
eval: true,
workerData,
});
return new Promise<void>((resolve, reject) => {
thread.on('message', (message: WorkerThreadMessage) => {
if (message.type === 'start' || message.type === 'result') {
if (message.type === 'result') {
results[message.index] = message.result as TResult;
}
const { value, done } = iterator.next();
if (done) {
thread.postMessage({ type: 'done' });
} else {
thread.postMessage({
type: 'item',
index: itemIndex,
item: value,
});
itemIndex += 1;
}
} else if (message.type === 'error') {
const error = new Error(message.error.message);
error.name = message.error.name;
error.stack = message.error.stack;
reject(error);
}
});
thread.on('error', reject);
thread.on('exit', (code: number) => {
if (code !== 0) {
reject(new Error(`Worker thread exited with code ${code}`));
} else {
resolve();
}
});
});
}),
);
return { results };
}
/* istanbul ignore next */
function workerQueueThread(
workerFuncFactory: (
data: unknown,
) => Promise<(item: unknown) => Promise<unknown>>,
) {
const { parentPort, workerData } = require('node:worker_threads');
Promise.resolve()
.then(() => workerFuncFactory(workerData))
.then(
workerFunc => {
parentPort.on('message', async (message: WorkerThreadMessage) => {
if (message.type === 'done') {
parentPort.close();
return;
}
if (message.type === 'item') {
try {
const result = await workerFunc(message.item);
parentPort.postMessage({
type: 'result',
index: message.index,
result,
});
} catch (error) {
parentPort.postMessage({ type: 'error', error });
}
}
});
parentPort.postMessage({ type: 'start' });
},
error => parentPort.postMessage({ type: 'error', error }),
);
}
+3 -3
View File
@@ -15,7 +15,7 @@
*/
import { assertError, ForwardedError } from '@backstage/errors';
import { paths } from '../paths';
import { targetPaths } from '@backstage/cli-common';
import { runOutput } from '@backstage/cli-common';
/**
@@ -24,7 +24,7 @@ import { runOutput } from '@backstage/cli-common';
export async function runGit(...args: string[]) {
try {
const stdout = await runOutput(['git', ...args], {
cwd: paths.targetRoot,
cwd: targetPaths.rootDir,
});
return stdout.trim().split(/\r\n|\r|\n/);
} catch (error) {
@@ -88,7 +88,7 @@ export class GitUtils {
}
const stdout = await runOutput(['git', 'show', `${showRef}:${path}`], {
cwd: paths.targetRoot,
cwd: targetPaths.rootDir,
});
return stdout;
}
+1
View File
@@ -22,4 +22,5 @@
export * from './git';
export * from './monorepo';
export * from './concurrency';
export * from './roles';
@@ -14,21 +14,16 @@
* limitations under the License.
*/
import { resolve as resolvePath } from 'node:path';
import { getPackages } from '@manypkg/get-packages';
import { PackageGraph } from './PackageGraph';
import { Lockfile } from './Lockfile';
import { GitUtils } from '../git';
import { overrideTargetPaths } from '@backstage/cli-common/testUtils';
const mockListChangedFiles = jest.spyOn(GitUtils, 'listChangedFiles');
const mockReadFileAtRef = jest.spyOn(GitUtils, 'readFileAtRef');
jest.mock('../paths', () => ({
paths: {
targetRoot: '/',
resolveTargetRoot: (...paths: string[]) => resolvePath('/', ...paths),
},
}));
overrideTargetPaths('/');
const testPackages = [
{
@@ -16,7 +16,7 @@
import path from 'node:path';
import { getPackages, Package } from '@manypkg/get-packages';
import { paths } from '../paths';
import { targetPaths } from '@backstage/cli-common';
import { PackageRole } from '../roles';
import { GitUtils } from '../git';
import { Lockfile } from './Lockfile';
@@ -192,7 +192,7 @@ export class PackageGraph extends Map<string, PackageGraphNode> {
* Lists all local packages in a monorepo.
*/
static async listTargetPackages(): Promise<BackstagePackage[]> {
const { packages } = await getPackages(paths.targetDir);
const { packages } = await getPackages(targetPaths.dir);
return packages as BackstagePackage[];
}
@@ -332,7 +332,7 @@ export class PackageGraph extends Map<string, PackageGraphNode> {
Array.from(this.values()).map(pkg => [
// relative from root, convert to posix, and add a / at the end
path
.relative(paths.targetRoot, pkg.dir)
.relative(targetPaths.rootDir, pkg.dir)
.split(path.sep)
.join(path.posix.sep) + path.posix.sep,
pkg,
@@ -374,7 +374,7 @@ export class PackageGraph extends Map<string, PackageGraphNode> {
let otherLockfile: Lockfile;
try {
thisLockfile = await Lockfile.load(
paths.resolveTargetRoot('yarn.lock'),
targetPaths.resolveRoot('yarn.lock'),
);
otherLockfile = Lockfile.parse(
await GitUtils.readFileAtRef('yarn.lock', options.ref),
+2 -2
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { paths } from '../paths';
import { targetPaths } from '@backstage/cli-common';
import fs from 'fs-extra';
/**
@@ -26,7 +26,7 @@ import fs from 'fs-extra';
* @public
*/
export async function isMonoRepo(): Promise<boolean> {
const rootPackageJsonPath = paths.resolveTargetRoot('package.json');
const rootPackageJsonPath = targetPaths.resolveRoot('package.json');
try {
const pkg = await fs.readJson(rootPackageJsonPath);
return Boolean(pkg?.workspaces);
@@ -16,12 +16,10 @@
import { isMonoRepo } from './isMonoRepo';
import { createMockDirectory } from '@backstage/backend-test-utils';
import { overrideTargetPaths } from '@backstage/cli-common/testUtils';
const mockDir = createMockDirectory();
jest.mock('../paths', () => ({
paths: { resolveTargetRoot: (...args: string[]) => mockDir.resolve(...args) },
}));
overrideTargetPaths(mockDir.path);
describe('isMonoRepo', () => {
it('should detect a monorepo', async () => {
@@ -15,16 +15,13 @@
*/
import { createMockDirectory } from '@backstage/backend-test-utils';
import { overrideTargetPaths } from '@backstage/cli-common/testUtils';
import { detectPackageManager } from './PackageManager';
import { Yarn } from './yarn';
import { withLogCollector } from '@backstage/test-utils';
const mockDir = createMockDirectory();
jest.mock('../paths', () => ({
...jest.requireActual('../paths'),
paths: { resolveTargetRoot: (...args: string[]) => mockDir.resolve(...args) },
}));
overrideTargetPaths(mockDir.path);
const mockYarnCreate = jest.spyOn(Yarn, 'create');
@@ -16,7 +16,7 @@
import { Yarn } from './yarn';
import { Lockfile } from './Lockfile';
import { paths } from '../paths';
import { targetPaths } from '@backstage/cli-common';
import { RunOptions } from '@backstage/cli-common';
import fs from 'fs-extra';
@@ -85,7 +85,7 @@ export interface PackageManager {
*/
export async function detectPackageManager(): Promise<PackageManager> {
const hasYarnLockfile = await fileExists(
paths.resolveTargetRoot('yarn.lock'),
targetPaths.resolveRoot('yarn.lock'),
);
if (hasYarnLockfile) {
return await Yarn.create();
@@ -93,7 +93,7 @@ export async function detectPackageManager(): Promise<PackageManager> {
try {
const packageJson = await fs.readJson(
paths.resolveTargetRoot('package.json'),
targetPaths.resolveRoot('package.json'),
);
if (packageJson.workspaces) {
// technically this could be NPM as well