Merge pull request #9500 from backstage/rugvip/parallint

cli: introduce new `repo lint` command

.changeset/angry-bottles-mix.md

@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Added a new experimental and hidden `backstage-cli repo lint` command that can be used to lint all packages in the project, similar to `lerna run lint`.

packages/cli/src/commands/index.ts

@@ -42,6 +42,21 @@ export function registerRepoCommand(program: CommanderStatic) {
       'Build all packages, including bundled app and backend packages.',
     )
     .action(lazy(() => import('./repo/build').then(m => m.command)));
+
+  command
+    .command('lint')
+    .description('Lint all packages in the project')
+    .option(
+      '--format <format>',
+      'Lint report output format',
+      'eslint-formatter-friendly',
+    )
+    .option(
+      '--since <ref>',
+      'Only lint packages that changed since the specified ref',
+    )
+    .option('--fix', 'Attempt to automatically fix violations')
+    .action(lazy(() => import('./repo/lint').then(m => m.command)));
 }
 
 export function registerScriptCommand(program: CommanderStatic) {

packages/cli/src/commands/repo/lint.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import chalk from 'chalk';
+import { Command } from 'commander';
+import { relative as relativePath } from 'path';
+import { PackageGraph, ExtendedPackageJSON } from '../../lib/monorepo';
+import { runWorkerQueueThreads } from '../../lib/parallel';
+import { paths } from '../../lib/paths';
+
+function depCount(pkg: ExtendedPackageJSON) {
+  const deps = pkg.dependencies ? Object.keys(pkg.dependencies).length : 0;
+  const devDeps = pkg.devDependencies
+    ? Object.keys(pkg.devDependencies).length
+    : 0;
+  return deps + devDeps;
+}
+
+export async function command(cmd: Command): Promise<void> {
+  let packages = await PackageGraph.listTargetPackages();
+
+  if (cmd.since) {
+    const graph = PackageGraph.fromPackages(packages);
+    packages = await graph.listChangedPackages({ ref: cmd.since });
+  }
+
+  // Packages are ordered from most to least number of dependencies, as a
+  // very cheap way of guessing which packages are more likely to be larger.
+  packages.sort((a, b) => depCount(b.packageJson) - depCount(a.packageJson));
+
+  // This formatter uses the cwd to format file paths, so let's have that happen from the root instead
+  if (cmd.format === 'eslint-formatter-friendly') {
+    process.chdir(paths.targetRoot);
+  }
+
+  // Make sure lint output is colored unless the user explicitly disabled it
+  if (!process.env.FORCE_COLOR) {
+    process.env.FORCE_COLOR = '1';
+  }
+
+  const resultsList = await runWorkerQueueThreads({
+    items: packages.map(pkg => ({
+      fullDir: pkg.dir,
+      relativeDir: relativePath(paths.targetRoot, pkg.dir),
+    })),
+    workerData: {
+      fix: Boolean(cmd.fix),
+      format: cmd.format as string | undefined,
+    },
+    workerFactory: async ({ fix, format }) => {
+      const { ESLint } = require('eslint');
+
+      return async ({
+        fullDir,
+        relativeDir,
+      }): Promise<{ relativeDir: string; resultText: string }> => {
+        // Bit of a hack to make file resolutions happen from the correct directory
+        // since some lint rules don't respect the cwd of ESLint
+        process.cwd = () => fullDir;
+
+        const start = Date.now();
+        const eslint = new ESLint({
+          cwd: fullDir,
+          fix,
+          extensions: ['js', 'jsx', 'ts', 'tsx', 'mjs', 'cjs'],
+        });
+        const formatter = await eslint.loadFormatter(format);
+
+        const results = await eslint.lintFiles(['.']);
+
+        const count = String(results.length).padStart(3);
+        const time = ((Date.now() - start) / 1000).toFixed(2);
+        console.log(`Checked ${count} files in ${relativeDir} ${time}s`);
+
+        if (fix) {
+          await ESLint.outputFixes(results);
+        }
+
+        const resultText = formatter.format(results);
+
+        return { relativeDir, resultText };
+      };
+    },
+  });
+
+  let failed = false;
+  for (const { relativeDir, resultText } of resultsList) {
+    if (resultText) {
+      console.log();
+      console.log(chalk.red(`Lint failed in ${relativeDir}:`));
+      console.log(resultText.trimLeft());
+
+      failed = true;
+    }
+  }
+
+  if (failed) {
+    process.exit(1);
+  }
+}

packages/cli/src/lib/git.test.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { runGit, listChangedFiles } from './git';
+
+describe('runGit', () => {
+  it('runs a git command', async () => {
+    await expect(runGit('log', 'HEAD..HEAD')).resolves.toEqual(['']);
+  });
+
+  it('fails for unknown commands', async () => {
+    await expect(runGit('ryckbegäran')).rejects.toThrow(
+      /^git ryckbegäran failed, git: 'ryckbegäran' is not a git command/,
+    );
+  });
+
+  it('forwards failures', async () => {
+    await expect(
+      runGit(
+        'show',
+        '--quiet',
+        '--pretty=format:%s',
+        '0000000000000000000000000000000000000000',
+      ),
+    ).rejects.toThrow(
+      'git show failed, fatal: bad object 0000000000000000000000000000000000000000',
+    );
+  });
+});
+
+describe('listChangedFiles', () => {
+  it('requires a ref', async () => {
+    await expect(listChangedFiles('')).rejects.toThrow('ref is required');
+  });
+
+  it('should return something', async () => {
+    await expect(listChangedFiles('HEAD')).resolves.toEqual(expect.any(Array));
+  });
+});

packages/cli/src/lib/git.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { assertError, ForwardedError } from '@backstage/errors';
+import { execFile as execFileCb } from 'child_process';
+import { promisify } from 'util';
+import { paths } from './paths';
+
+const execFile = promisify(execFileCb);
+
+/**
+ * Run a git command, trimming the output splitting it into lines.
+ */
+export async function runGit(...args: string[]) {
+  try {
+    const { stdout } = await execFile('git', args, {
+      shell: true,
+      cwd: paths.targetRoot,
+    });
+    return stdout.trim().split(/\r\n|\r|\n/);
+  } catch (error) {
+    assertError(error);
+    if (error.stderr || typeof error.code === 'number') {
+      const stderr = (error.stderr as undefined | Buffer)?.toString('utf8');
+      const msg = stderr?.trim() ?? `with exit code ${error.code}`;
+      throw new Error(`git ${args[0]} failed, ${msg}`);
+    }
+    throw new ForwardedError('Unknown execution error', error);
+  }
+}
+
+/**
+ * Returns a sorted list of all files that have changed since the merge base
+ * of the provided `ref` and HEAD, as well as all files that are not tracked by git.
+ */
+export async function listChangedFiles(ref: string) {
+  if (!ref) {
+    throw new Error('ref is required');
+  }
+  const [base] = await runGit('merge-base', 'HEAD', ref);
+
+  const tracked = await runGit('diff', '--name-only', base);
+  const untracked = await runGit('ls-files', '--others', '--exclude-standard');
+
+  return Array.from(new Set([...tracked, ...untracked]));
+}

packages/cli/src/lib/monorepo/PackageGraph.test.ts

@@ -15,8 +15,20 @@
  */
 
 import { getPackages } from '@manypkg/get-packages';
-import { paths } from '../paths';
 import { PackageGraph } from './PackageGraph';
+import { listChangedFiles } from '../git';
+
+jest.mock('../git');
+
+const mockListChangedFiles = listChangedFiles as jest.MockedFunction<
+  typeof listChangedFiles
+>;
+
+jest.mock('../paths', () => ({
+  paths: {
+    targetRoot: '/',
+  },
+}));
 
 const testPackages = [
   {
@@ -53,7 +65,7 @@ const testPackages = [
 
 describe('PackageGraph', () => {
   it('is able to construct a graph from this repo', async () => {
-    const { packages } = await getPackages(paths.ownDir);
+    const { packages } = await getPackages(__dirname);
     const graph = PackageGraph.fromPackages(packages);
     expect(graph.has('@backstage/cli')).toBe(true);
   });
@@ -124,4 +136,24 @@ describe('PackageGraph', () => {
       `Package 'unknown' not found`,
     );
   });
+
+  it('lists changed packages', async () => {
+    const graph = PackageGraph.fromPackages(testPackages);
+
+    mockListChangedFiles.mockResolvedValueOnce(
+      [
+        'README.md',
+        'packages/a/src/foo.ts',
+        'packages/a/src/bar.ts',
+        'packages/b/package.json',
+        'packages/f/src/foo.ts',
+        'packages/f/README.md',
+        'plugins/foo/src/index.ts',
+      ].sort(),
+    );
+
+    await expect(
+      graph.listChangedPackages({ ref: 'origin/master' }),
+    ).resolves.toEqual([graph.get('a'), graph.get('b')]);
+  });
 });

packages/cli/src/lib/monorepo/PackageGraph.ts

@@ -14,9 +14,11 @@
  * limitations under the License.
  */
 
+import { relative as relativePath, posix as posixPath } from 'path';
 import { getPackages, Package } from '@manypkg/get-packages';
 import { paths } from '../paths';
 import { PackageRole } from '../role';
+import { listChangedFiles } from '../git';
 
 type PackageJSON = Package['packageJson'];
 
@@ -158,4 +160,47 @@ export class PackageGraph extends Map<string, PackageGraphNode> {
 
     return targets;
   }
+
+  async listChangedPackages(options: { ref: string }) {
+    const changedFiles = await listChangedFiles(options.ref);
+
+    const dirMap = new Map(
+      Array.from(this.values()).map(pkg => [
+        posixPath.normalize(relativePath(paths.targetRoot, pkg.dir)) +
+          posixPath.sep,
+        pkg,
+      ]),
+    );
+    const packageDirs = Array.from(dirMap.keys());
+
+    const result = new Array<PackageGraphNode>();
+    let searchIndex = 0;
+
+    changedFiles.sort();
+    packageDirs.sort();
+
+    for (const packageDir of packageDirs) {
+      // Skip through changes that appear before our package dir
+      while (
+        searchIndex < changedFiles.length &&
+        changedFiles[searchIndex] < packageDir
+      ) {
+        searchIndex += 1;
+      }
+
+      // Check if we arrived at a match, otherwise we move on to the next package dir
+      if (changedFiles[searchIndex]?.startsWith(packageDir)) {
+        searchIndex += 1;
+
+        result.push(dirMap.get(packageDir)!);
+
+        // Skip through the rest of the changed files for the same package
+        while (changedFiles[searchIndex]?.startsWith(packageDir)) {
+          searchIndex += 1;
+        }
+      }
+    }
+
+    return result;
+  }
 }

packages/cli/src/lib/monorepo/index.ts

@@ -15,4 +15,8 @@
  */
 
 export { PackageGraph } from './PackageGraph';
-export type { PackageGraphNode } from './PackageGraph';
+export type {
+  PackageGraphNode,
+  ExtendedPackage,
+  ExtendedPackageJSON,
+} from './PackageGraph';

packages/cli/src/lib/parallel.test.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import os from 'os';
 import {
   parseParallelismOption,
   getEnvironmentParallelism,
@@ -22,6 +23,8 @@ import {
   runWorkerThreads,
 } from './parallel';
 
+const defaultParallelism = Math.ceil(os.cpus().length / 2);
+
 describe('parseParallelismOption', () => {
   it('coerces false no parallelism', () => {
     expect(parseParallelismOption(false)).toBe(1);
@@ -29,10 +32,10 @@ describe('parseParallelismOption', () => {
   });
 
   it('coerces true or undefined to default parallelism', () => {
-    expect(parseParallelismOption(true)).toBe(4);
-    expect(parseParallelismOption('true')).toBe(4);
-    expect(parseParallelismOption(undefined)).toBe(4);
-    expect(parseParallelismOption(null)).toBe(4);
+    expect(parseParallelismOption(true)).toBe(defaultParallelism);
+    expect(parseParallelismOption('true')).toBe(defaultParallelism);
+    expect(parseParallelismOption(undefined)).toBe(defaultParallelism);
+    expect(parseParallelismOption(null)).toBe(defaultParallelism);
   });
 
   it('coerces number string to number', () => {
@@ -52,13 +55,13 @@ describe('getEnvironmentParallelism', () => {
     expect(getEnvironmentParallelism()).toBe(2);
 
     process.env.BACKSTAGE_CLI_BUILD_PARALLEL = 'true';
-    expect(getEnvironmentParallelism()).toBe(4);
+    expect(getEnvironmentParallelism()).toBe(defaultParallelism);
 
     process.env.BACKSTAGE_CLI_BUILD_PARALLEL = 'false';
     expect(getEnvironmentParallelism()).toBe(1);
 
     delete process.env.BACKSTAGE_CLI_BUILD_PARALLEL;
-    expect(getEnvironmentParallelism()).toBe(4);
+    expect(getEnvironmentParallelism()).toBe(defaultParallelism);
   });
 });
 
@@ -70,6 +73,7 @@ describe('runParallelWorkers', () => {
 
     const work = runParallelWorkers({
       items: [0, 1, 2, 3, 4],
+      parallelismSetting: 4,
       parallelismFactor: 0.5, // 2 at a time
       worker: async item => {
         started.push(item);

packages/cli/src/lib/parallel.ts

@@ -14,20 +14,21 @@
  * limitations under the License.
  */
 
+import os from 'os';
 import { ErrorLike } from '@backstage/errors';
 import { Worker } from 'worker_threads';
 
-export const DEFAULT_PARALLELISM = 4;
+const defaultParallelism = Math.ceil(os.cpus().length / 2);
 
-export const PARALLEL_ENV_VAR = 'BACKSTAGE_CLI_BUILD_PARALLEL';
+const PARALLEL_ENV_VAR = 'BACKSTAGE_CLI_BUILD_PARALLEL';
 
 export type ParallelismOption = boolean | string | number | null | undefined;
 
 export function parseParallelismOption(parallel: ParallelismOption): number {
   if (parallel === undefined || parallel === null) {
-    return DEFAULT_PARALLELISM;
+    return defaultParallelism;
   } else if (typeof parallel === 'boolean') {
-    return parallel ? DEFAULT_PARALLELISM : 1;
+    return parallel ? defaultParallelism : 1;
   } else if (typeof parallel === 'number' && Number.isInteger(parallel)) {
     if (parallel < 1) {
       return 1;
@@ -134,10 +135,14 @@ export type WorkerQueueThreadsOptions<TItem, TResult, TData> = {
    * note that they are both copied by value into the worker thread, except for
    * types that are explicitly shareable across threads, such as `SharedArrayBuffer`.
    */
-  workerFactory: (data: TData) => (item: TItem) => Promise<TResult>;
+  workerFactory: (
+    data: TData,
+  ) =>
+    | ((item: TItem) => Promise<TResult>)
+    | Promise<(item: TItem) => Promise<TResult>>;
   /** Data supplied to each worker factory */
   workerData?: TData;
-  /** Number of threads, defaults to the environment parallelism */
+  /** Number of threads, defaults to half of the number of available CPUs */
   threadCount?: number;
 };
 
@@ -148,13 +153,14 @@ export type WorkerQueueThreadsOptions<TItem, TResult, TData> = {
 export async function runWorkerQueueThreads<TItem, TResult, TData>(
   options: WorkerQueueThreadsOptions<TItem, TResult, TData>,
 ): Promise<TResult[]> {
+  const items = Array.from(options.items);
   const {
     workerFactory,
     workerData,
-    threadCount = getEnvironmentParallelism(),
+    threadCount = Math.min(getEnvironmentParallelism(), items.length),
   } = options;
 
-  const iterator = options.items[Symbol.iterator]();
+  const iterator = items[Symbol.iterator]();
   const results = new Array<TResult>();
   let itemIndex = 0;
 
@@ -208,31 +214,39 @@ export async function runWorkerQueueThreads<TItem, TResult, TData>(
 }
 
 function workerQueueThread(
-  workerFuncFactory: (data: unknown) => (item: unknown) => Promise<unknown>,
+  workerFuncFactory: (
+    data: unknown,
+  ) => Promise<(item: unknown) => Promise<unknown>>,
 ) {
   const { parentPort, workerData } = require('worker_threads');
-  const workerFunc = workerFuncFactory(workerData);
 
-  parentPort.on('message', async (message: WorkerThreadMessage) => {
-    if (message.type === 'done') {
-      parentPort.close();
-      return;
-    }
-    if (message.type === 'item') {
-      try {
-        const result = await workerFunc(message.item);
-        parentPort.postMessage({
-          type: 'result',
-          index: message.index,
-          result,
+  Promise.resolve()
+    .then(() => workerFuncFactory(workerData))
+    .then(
+      workerFunc => {
+        parentPort.on('message', async (message: WorkerThreadMessage) => {
+          if (message.type === 'done') {
+            parentPort.close();
+            return;
+          }
+          if (message.type === 'item') {
+            try {
+              const result = await workerFunc(message.item);
+              parentPort.postMessage({
+                type: 'result',
+                index: message.index,
+                result,
+              });
+            } catch (error) {
+              parentPort.postMessage({ type: 'error', error });
+            }
+          }
         });
-      } catch (error) {
-        parentPort.postMessage({ type: 'error', error });
-      }
-    }
-  });
 
-  parentPort.postMessage({ type: 'start' });
+        parentPort.postMessage({ type: 'start' });
+      },
+      error => parentPort.postMessage({ type: 'error', error }),
+    );
 }
 
 export type WorkerThreadsOptions<TResult, TData, TMessage> = {