diff --git a/.changeset/small-donkeys-attack.md b/.changeset/small-donkeys-attack.md index 116cc611b5..ecfc969452 100644 --- a/.changeset/small-donkeys-attack.md +++ b/.changeset/small-donkeys-attack.md @@ -2,6 +2,6 @@ '@backstage/cli': patch --- -Added a new `--successCache` option to the `backstage-cli repo lint` command. The cache keeps track of successful lint runs and avoids re-running linting of individual packages if they haven't changed. This option is primarily intended to be used in CI. +Added a new `--successCache` option to the `backstage-cli repo test` and `backstage-cli repo lint` commands. The cache keeps track of successful runs and avoids re-running for individual packages if they haven't changed. This option is intended only to be used in CI. -In addition a `--successCacheDir` option has also been added to be able to override the default cache directory. +In addition a `--successCacheDir ` option has also been added to be able to override the default cache directory. diff --git a/docs/tooling/cli/02-build-system.md b/docs/tooling/cli/02-build-system.md index 52410001f7..dfd95ae6eb 100644 --- a/docs/tooling/cli/02-build-system.md +++ b/docs/tooling/cli/02-build-system.md @@ -564,6 +564,7 @@ The overrides in a single `package.json` may for example look like this: Caching is used sparingly throughout the Backstage build system. It is always used as a way to squeeze out a little bit of extra performance, rather than requirement to keep things fast. The following is a list of places where optional caching is available: - **TypeScript** - The default `tsconfig.json` used by Backstage projects has `incremental` set to `true`, which enables local caching of type checking results. It is however generally not recommended in CI, where `yarn tsc:full` is preferred, which sets `--incremental false`. +- **Testing** - The `backstage-cli repo test` command has a `--successCache` flag that enables caching of successful test results. This is done at the package level, meaning that if a package has not been changed since the last test run and it was successful, the testing will be skipped. This is recommended to be used in CI, but not during local development. - **Linting** - The `backstage-cli repo lint` command has a `--successCache` flag that enables caching of successful linting results. This is done at the package level, meaning that if a package has not been changed since the last lint run and it was successful, the linting will be skipped. This is recommended to be used in CI, but not during local development. - **Webpack** - It is possible to enable experimental caching of frontend package builds using the `BACKSTAGE_CLI_EXPERIMENTAL_BUILD_CACHE` environment variable. This will enable the Webpack filesystem cache. diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index 72cf7c4505..f164e5e1fc 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -469,6 +469,8 @@ Usage: backstage-cli repo test [options] Options: --since + --successCache + --successCacheDir --jest-help -h, --help ``` diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index d99db11656..94164b09ab 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -306,7 +306,7 @@ async function getRootConfig() { ), ).then(_ => _.flat()); - const configs = await Promise.all( + let configs = await Promise.all( projectPaths.flat().map(async projectPath => { const packagePath = path.resolve(projectPath, 'package.json'); if (!(await fs.pathExists(packagePath))) { @@ -331,9 +331,15 @@ async function getRootConfig() { }), ).then(cs => cs.filter(Boolean)); + const cache = global.__backstageCli_jestSuccessCache; + if (cache) { + configs = await cache.filterConfigs(configs, globalRootConfig); + } + return { rootDir: paths.targetRoot, projects: configs, + testResultsProcessor: require.resolve('./jestCacheResultProcessor.cjs'), ...globalRootConfig, }; } diff --git a/packages/cli/config/jestCacheResultProcessor.cjs b/packages/cli/config/jestCacheResultProcessor.cjs new file mode 100644 index 0000000000..ae7de0ba3c --- /dev/null +++ b/packages/cli/config/jestCacheResultProcessor.cjs @@ -0,0 +1,40 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = async results => { + const cache = global.__backstageCli_jestSuccessCache; + if (!cache) { + return results; + } + + const successful = new Set(); + const failed = new Set(); + for (const testResult of results.testResults) { + const projectName = testResult.displayName.name; + if (testResult.numFailingTests > 0) { + failed.add(projectName); + successful.delete(projectName); + } else if (!failed.has(projectName)) { + successful.add(projectName); + } + } + + await cache.reportResults({ + successful: successful, + }); + + return results; +}; diff --git a/packages/cli/package.json b/packages/cli/package.json index 67f69c988c..f438cae444 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -113,6 +113,7 @@ "html-webpack-plugin": "^5.3.1", "inquirer": "^8.2.0", "jest": "^29.7.0", + "jest-cli": "^29.7.0", "jest-css-modules": "^2.1.0", "jest-environment-jsdom": "^29.0.2", "jest-runtime": "^29.0.2", @@ -152,6 +153,7 @@ "webpack-dev-server": "^5.0.0", "webpack-node-externals": "^3.0.0", "yaml": "^2.0.0", + "yargs": "^16.2.0", "yml-loader": "^2.1.0", "yn": "^4.0.0", "zod": "^3.22.4" diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 0d72400bc4..a3060f2d67 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -105,6 +105,14 @@ export function registerRepoCommand(program: Command) { '--since ', 'Only test packages that changed since the specified ref', ) + .option( + '--successCache', + 'Enable success caching, which skips running tests for unchanged packages that were successful in the previous run', + ) + .option( + '--successCacheDir ', + 'Set the success cache location, (default: node_modules/.cache/backstage-cli)', + ) .option( '--jest-help', 'Show help for Jest CLI options, which are passed through', diff --git a/packages/cli/src/commands/repo/test.ts b/packages/cli/src/commands/repo/test.ts index 1ca0da5495..72c9f3aada 100644 --- a/packages/cli/src/commands/repo/test.ts +++ b/packages/cli/src/commands/repo/test.ts @@ -15,10 +15,76 @@ */ import os from 'os'; +import crypto from 'node:crypto'; +import fs from 'fs-extra'; +import yargs from 'yargs'; +import { resolve as resolvePath, relative as relativePath } from 'path'; import { Command, OptionValues } from 'commander'; -import { PackageGraph } from '@backstage/cli-node'; +import { Lockfile, PackageGraph } from '@backstage/cli-node'; import { paths } from '../../lib/paths'; -import { runCheck } from '../../lib/run'; +import { runCheck, runPlain } from '../../lib/run'; + +type JestProject = { + displayName: string; +}; + +interface GlobalWithCache extends Global { + __backstageCli_jestSuccessCache?: { + filterConfigs( + projectConfigs: JestProject[], + globalConfig: unknown, + ): Promise; + reportResults(options: { successful: Set }): Promise; + }; +} + +const CACHE_FILE_NAME = 'test-cache.json'; + +type Cache = string[]; + +async function readCache(dir: string): Promise { + try { + const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); + if (!Array.isArray(data)) { + return undefined; + } + if (data.some(x => typeof x !== 'string')) { + return undefined; + } + return data as Cache; + } catch { + return undefined; + } +} + +function writeCache(dir: string, cache: Cache) { + fs.mkdirpSync(dir); + fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); +} + +/** + * Use git to get the HEAD tree hashes of each package in the project. + */ +async function getPackageTreeHashes(graph: PackageGraph) { + const pkgs = Array.from(graph.values()); + const output = await runPlain( + 'git', + 'ls-tree', + '--object-only', + 'HEAD', + '--', + ...pkgs.map(pkg => relativePath(paths.targetRoot, pkg.dir)), + ); + + const treeShaList = output.trim().split(/\r?\n/); + if (treeShaList.length !== pkgs.length) { + throw new Error( + `Error listing project git tree hashes, output length does not equal input length`, + ); + } + + return new Map(pkgs.map((pkg, i) => [pkg.packageJson.name, treeShaList[i]])); +} export function createFlagFinder(args: string[]) { const flags = new Set(); @@ -120,9 +186,18 @@ export async function command(opts: OptionValues, cmd: Command): Promise { removeOptionArg(args, '--since'); } - if (opts.since && !hasFlags('--selectProjects')) { + let packageGraph: PackageGraph | undefined; + async function getPackageGraph() { + if (packageGraph) { + return packageGraph; + } const packages = await PackageGraph.listTargetPackages(); - const graph = PackageGraph.fromPackages(packages); + packageGraph = PackageGraph.fromPackages(packages); + return packageGraph; + } + + if (opts.since && !hasFlags('--selectProjects')) { + const graph = await getPackageGraph(); const changedPackages = await graph.listChangedPackages({ ref: opts.since, analyzeLockfile: true, @@ -163,5 +238,106 @@ export async function command(opts: OptionValues, cmd: Command): Promise { (process.stdout as any)._handle.setBlocking(true); } - await require('jest').run(args); + const jestCli = require('jest-cli'); + + // This code path is enabled by the --successCache flag, which is specific to + // the `repo test` command in the Backstage CLI. + if (opts.successCache) { + removeOptionArg(args, '--successCache'); + removeOptionArg(args, '--successCacheDir'); + + const cacheDir = resolvePath( + opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', + ); + + // Parse the args to ensure that no file filters are provided, in which case we refuse to run + const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions) + .argv; + if (parsedArgs.length > 0) { + throw new Error( + `The --successCache flag can not be combined with the following arguments: ${parsedArgs.join( + ', ', + )}`, + ); + } + // Likewise, it's not possible to combine sharding and the success cache + if (args.includes('--shard')) { + throw new Error( + `The --successCache flag can not be combined with the --shard flag`, + ); + } + + // Shared state for the bridge + const projectHashes = new Map(); + const outputSuccessCache = new Array(); + + // Set up a bridge with the @backstage/cli/config/jest configuration file. These methods + // are picked up by the config script itself, as well as the custom result processor. + const globalWithCache = global as GlobalWithCache; + globalWithCache.__backstageCli_jestSuccessCache = { + async filterConfigs(projectConfigs, globalRootConfig) { + const graph = await getPackageGraph(); + const cache = await readCache(cacheDir); + const lockfile = await Lockfile.load( + paths.resolveTargetRoot('yarn.lock'), + ); + const packageTreeHashes = await getPackageTreeHashes(graph); + + // Base hash shared by all projects + const baseHash = crypto.createHash('sha1'); + baseHash.update('v1'); // The version of this implementation + baseHash.update('\0'); + baseHash.update(process.version); // Node.js version + baseHash.update('\0'); + baseHash.update(JSON.stringify(globalRootConfig)); // Variable global jest config + const baseSha = baseHash.digest('hex'); + + return projectConfigs.filter(project => { + const packageName = project.displayName; + const pkg = graph.get(packageName); + if (!pkg) { + throw new Error( + `Package ${packageName} not found in package graph`, + ); + } + + const hash = crypto.createHash('sha1'); + + const packageTreeSha = packageTreeHashes.get(packageName); + if (!packageTreeSha) { + throw new Error(`Tree sha not found for ${packageName}`); + } + hash.update(baseSha); + hash.update(packageTreeSha); + // The project ID is a hash of the transform configuration, which helps + // us bust the cache when any changes are made to the transform implementation. + hash.update(JSON.stringify(project)); + hash.update(lockfile.getDependencyTreeHash(packageName)); + + const sha = hash.digest('hex'); + + projectHashes.set(packageName, sha); + + if (cache?.includes(sha)) { + console.log(`Skipped ${packageName} due to cache hit`); + outputSuccessCache.push(sha); + return undefined; + } + + return project; + }); + }, + async reportResults(options) { + for (const packageName of options.successful) { + const sha = projectHashes.get(packageName); + if (sha) { + outputSuccessCache.push(sha); + } + } + writeCache(cacheDir, outputSuccessCache); + }, + }; + } + + await jestCli.run(args); } diff --git a/yarn.lock b/yarn.lock index 2d1cc17f58..4029c9aef2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3988,6 +3988,7 @@ __metadata: html-webpack-plugin: ^5.3.1 inquirer: ^8.2.0 jest: ^29.7.0 + jest-cli: ^29.7.0 jest-css-modules: ^2.1.0 jest-environment-jsdom: ^29.0.2 jest-runtime: ^29.0.2 @@ -4032,6 +4033,7 @@ __metadata: webpack-dev-server: ^5.0.0 webpack-node-externals: ^3.0.0 yaml: ^2.0.0 + yargs: ^16.2.0 yml-loader: ^2.1.0 yn: ^4.0.0 zod: ^3.22.4