cli: add --successCache option for repo test

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>

packages/cli/cli-report.md

@@ -469,6 +469,8 @@ Usage: backstage-cli repo test [options]
 
 Options:
   --since <ref>
+  --successCache
+  --successCacheDir <path>
   --jest-help
   -h, --help
 ```

packages/cli/config/jest.js

@@ -306,7 +306,7 @@ async function getRootConfig() {
     ),
   ).then(_ => _.flat());
 
-  const configs = await Promise.all(
+  let configs = await Promise.all(
     projectPaths.flat().map(async projectPath => {
       const packagePath = path.resolve(projectPath, 'package.json');
       if (!(await fs.pathExists(packagePath))) {
@@ -331,9 +331,15 @@ async function getRootConfig() {
     }),
   ).then(cs => cs.filter(Boolean));
 
+  const cache = global.__backstageCli_jestSuccessCache;
+  if (cache) {
+    configs = await cache.filterConfigs(configs, globalRootConfig);
+  }
+
   return {
     rootDir: paths.targetRoot,
     projects: configs,
+    testResultsProcessor: require.resolve('./jestCacheResultProcessor.cjs'),
     ...globalRootConfig,
   };
 }

packages/cli/config/jestCacheResultProcessor.cjs

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = async results => {
+  const cache = global.__backstageCli_jestSuccessCache;
+  if (!cache) {
+    return results;
+  }
+
+  const successful = new Set();
+  const failed = new Set();
+  for (const testResult of results.testResults) {
+    const projectName = testResult.displayName.name;
+    if (testResult.numFailingTests > 0) {
+      failed.add(projectName);
+      successful.delete(projectName);
+    } else if (!failed.has(projectName)) {
+      successful.add(projectName);
+    }
+  }
+
+  await cache.reportResults({
+    successful: successful,
+  });
+
+  return results;
+};

packages/cli/package.json

@@ -113,6 +113,7 @@
     "html-webpack-plugin": "^5.3.1",
     "inquirer": "^8.2.0",
     "jest": "^29.7.0",
+    "jest-cli": "^29.7.0",
     "jest-css-modules": "^2.1.0",
     "jest-environment-jsdom": "^29.0.2",
     "jest-runtime": "^29.0.2",
@@ -152,6 +153,7 @@
     "webpack-dev-server": "^5.0.0",
     "webpack-node-externals": "^3.0.0",
     "yaml": "^2.0.0",
+    "yargs": "^16.2.0",
     "yml-loader": "^2.1.0",
     "yn": "^4.0.0",
     "zod": "^3.22.4"

packages/cli/src/commands/index.ts

@@ -105,6 +105,14 @@ export function registerRepoCommand(program: Command) {
       '--since <ref>',
       'Only test packages that changed since the specified ref',
     )
+    .option(
+      '--successCache',
+      'Enable success caching, which skips running tests for unchanged packages that were successful in the previous run',
+    )
+    .option(
+      '--successCacheDir <path>',
+      'Set the success cache location, (default: node_modules/.cache/backstage-cli)',
+    )
     .option(
       '--jest-help',
       'Show help for Jest CLI options, which are passed through',

packages/cli/src/commands/repo/test.ts

@@ -15,10 +15,76 @@
  */
 
 import os from 'os';
+import crypto from 'node:crypto';
+import fs from 'fs-extra';
+import yargs from 'yargs';
+import { resolve as resolvePath, relative as relativePath } from 'path';
 import { Command, OptionValues } from 'commander';
-import { PackageGraph } from '@backstage/cli-node';
+import { Lockfile, PackageGraph } from '@backstage/cli-node';
 import { paths } from '../../lib/paths';
-import { runCheck } from '../../lib/run';
+import { runCheck, runPlain } from '../../lib/run';
+
+type JestProject = {
+  displayName: string;
+};
+
+interface GlobalWithCache extends Global {
+  __backstageCli_jestSuccessCache?: {
+    filterConfigs(
+      projectConfigs: JestProject[],
+      globalConfig: unknown,
+    ): Promise<JestProject[]>;
+    reportResults(options: { successful: Set<string> }): Promise<void>;
+  };
+}
+
+const CACHE_FILE_NAME = 'test-cache.json';
+
+type Cache = string[];
+
+async function readCache(dir: string): Promise<Cache | undefined> {
+  try {
+    const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME));
+    if (!Array.isArray(data)) {
+      return undefined;
+    }
+    if (data.some(x => typeof x !== 'string')) {
+      return undefined;
+    }
+    return data as Cache;
+  } catch {
+    return undefined;
+  }
+}
+
+function writeCache(dir: string, cache: Cache) {
+  fs.mkdirpSync(dir);
+  fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 });
+}
+
+/**
+ * Use git to get the HEAD tree hashes of each package in the project.
+ */
+async function getPackageTreeHashes(graph: PackageGraph) {
+  const pkgs = Array.from(graph.values());
+  const output = await runPlain(
+    'git',
+    'ls-tree',
+    '--object-only',
+    'HEAD',
+    '--',
+    ...pkgs.map(pkg => relativePath(paths.targetRoot, pkg.dir)),
+  );
+
+  const treeShaList = output.trim().split(/\r?\n/);
+  if (treeShaList.length !== pkgs.length) {
+    throw new Error(
+      `Error listing project git tree hashes, output length does not equal input length`,
+    );
+  }
+
+  return new Map(pkgs.map((pkg, i) => [pkg.packageJson.name, treeShaList[i]]));
+}
 
 export function createFlagFinder(args: string[]) {
   const flags = new Set<string>();
@@ -120,9 +186,18 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
     removeOptionArg(args, '--since');
   }
 
-  if (opts.since && !hasFlags('--selectProjects')) {
+  let packageGraph: PackageGraph | undefined;
+  async function getPackageGraph() {
+    if (packageGraph) {
+      return packageGraph;
+    }
     const packages = await PackageGraph.listTargetPackages();
-    const graph = PackageGraph.fromPackages(packages);
+    packageGraph = PackageGraph.fromPackages(packages);
+    return packageGraph;
+  }
+
+  if (opts.since && !hasFlags('--selectProjects')) {
+    const graph = await getPackageGraph();
     const changedPackages = await graph.listChangedPackages({
       ref: opts.since,
       analyzeLockfile: true,
@@ -163,5 +238,106 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
     (process.stdout as any)._handle.setBlocking(true);
   }
 
-  await require('jest').run(args);
+  const jestCli = require('jest-cli');
+
+  // This code path is enabled by the --successCache flag, which is specific to
+  // the `repo test` command in the Backstage CLI.
+  if (opts.successCache) {
+    removeOptionArg(args, '--successCache');
+    removeOptionArg(args, '--successCacheDir');
+
+    const cacheDir = resolvePath(
+      opts.successCacheDir ?? 'node_modules/.cache/backstage-cli',
+    );
+
+    // Parse the args to ensure that no file filters are provided, in which case we refuse to run
+    const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions)
+      .argv;
+    if (parsedArgs.length > 0) {
+      throw new Error(
+        `The --successCache flag can not be combined with the following arguments: ${parsedArgs.join(
+          ', ',
+        )}`,
+      );
+    }
+    // Likewise, it's not possible to combine sharding and the success cache
+    if (args.includes('--shard')) {
+      throw new Error(
+        `The --successCache flag can not be combined with the --shard flag`,
+      );
+    }
+
+    // Shared state for the bridge
+    const projectHashes = new Map<string, string>();
+    const outputSuccessCache = new Array<string>();
+
+    // Set up a bridge with the @backstage/cli/config/jest configuration file. These methods
+    // are picked up by the config script itself, as well as the custom result processor.
+    const globalWithCache = global as GlobalWithCache;
+    globalWithCache.__backstageCli_jestSuccessCache = {
+      async filterConfigs(projectConfigs, globalRootConfig) {
+        const graph = await getPackageGraph();
+        const cache = await readCache(cacheDir);
+        const lockfile = await Lockfile.load(
+          paths.resolveTargetRoot('yarn.lock'),
+        );
+        const packageTreeHashes = await getPackageTreeHashes(graph);
+
+        // Base hash shared by all projects
+        const baseHash = crypto.createHash('sha1');
+        baseHash.update('v1'); // The version of this implementation
+        baseHash.update('\0');
+        baseHash.update(process.version); // Node.js version
+        baseHash.update('\0');
+        baseHash.update(JSON.stringify(globalRootConfig)); // Variable global jest config
+        const baseSha = baseHash.digest('hex');
+
+        return projectConfigs.filter(project => {
+          const packageName = project.displayName;
+          const pkg = graph.get(packageName);
+          if (!pkg) {
+            throw new Error(
+              `Package ${packageName} not found in package graph`,
+            );
+          }
+
+          const hash = crypto.createHash('sha1');
+
+          const packageTreeSha = packageTreeHashes.get(packageName);
+          if (!packageTreeSha) {
+            throw new Error(`Tree sha not found for ${packageName}`);
+          }
+          hash.update(baseSha);
+          hash.update(packageTreeSha);
+          // The project ID is a hash of the transform configuration, which helps
+          // us bust the cache when any changes are made to the transform implementation.
+          hash.update(JSON.stringify(project));
+          hash.update(lockfile.getDependencyTreeHash(packageName));
+
+          const sha = hash.digest('hex');
+
+          projectHashes.set(packageName, sha);
+
+          if (cache?.includes(sha)) {
+            console.log(`Skipped ${packageName} due to cache hit`);
+            outputSuccessCache.push(sha);
+            return undefined;
+          }
+
+          return project;
+        });
+      },
+      async reportResults(options) {
+        for (const packageName of options.successful) {
+          const sha = projectHashes.get(packageName);
+          if (sha) {
+            outputSuccessCache.push(sha);
+          }
+        }
+        writeCache(cacheDir, outputSuccessCache);
+      },
+    };
+  }
+
+  await jestCli.run(args);
 }