From 80a1f413341a7b574e2635c26a6e88b343df63d7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 11:37:03 +0200 Subject: [PATCH 1/9] cli/templates: .eslintrc.js -> .eslintrc.js.hbs Signed-off-by: Patrik Oldsberg --- packages/cli/src/lib/new/factories/backendModule.test.ts | 2 +- packages/cli/src/lib/new/factories/backendPlugin.test.ts | 2 +- packages/cli/src/lib/new/factories/frontendPlugin.test.ts | 2 +- packages/cli/src/lib/new/factories/nodeLibraryPackage.test.ts | 2 +- packages/cli/src/lib/new/factories/pluginCommon.test.ts | 2 +- packages/cli/src/lib/new/factories/pluginNode.test.ts | 2 +- packages/cli/src/lib/new/factories/pluginWeb.test.ts | 2 +- packages/cli/src/lib/new/factories/scaffolderModule.test.ts | 2 +- packages/cli/src/lib/new/factories/webLibraryPackage.test.ts | 2 +- .../default-backend-module/{.eslintrc.js => .eslintrc.js.hbs} | 0 .../default-backend-plugin/{.eslintrc.js => .eslintrc.js.hbs} | 0 .../{.eslintrc.js => .eslintrc.js.hbs} | 0 .../{.eslintrc.js => .eslintrc.js.hbs} | 0 .../templates/default-plugin/{.eslintrc.js => .eslintrc.js.hbs} | 0 .../{.eslintrc.js => .eslintrc.js.hbs} | 0 .../node-library-package/{.eslintrc.js => .eslintrc.js.hbs} | 0 .../scaffolder-module/{.eslintrc.js => .eslintrc.js.hbs} | 0 .../web-library-package/{.eslintrc.js => .eslintrc.js.hbs} | 0 18 files changed, 9 insertions(+), 9 deletions(-) rename packages/cli/templates/default-backend-module/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/default-backend-plugin/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/default-common-plugin-package/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/default-node-plugin-package/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/default-plugin/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/default-react-plugin-package/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/node-library-package/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/scaffolder-module/{.eslintrc.js => .eslintrc.js.hbs} (100%) rename packages/cli/templates/web-library-package/{.eslintrc.js => .eslintrc.js.hbs} (100%) diff --git a/packages/cli/src/lib/new/factories/backendModule.test.ts b/packages/cli/src/lib/new/factories/backendModule.test.ts index 59b5403239..36ed9655a1 100644 --- a/packages/cli/src/lib/new/factories/backendModule.test.ts +++ b/packages/cli/src/lib/new/factories/backendModule.test.ts @@ -90,7 +90,7 @@ describe('backendModule factory', () => { `availability plugins${sep}test-backend-module-tester-two`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/src/lib/new/factories/backendPlugin.test.ts b/packages/cli/src/lib/new/factories/backendPlugin.test.ts index 3b90e2b0dd..c5dc237517 100644 --- a/packages/cli/src/lib/new/factories/backendPlugin.test.ts +++ b/packages/cli/src/lib/new/factories/backendPlugin.test.ts @@ -89,7 +89,7 @@ describe('backendPlugin factory', () => { `availability plugins${sep}test-backend`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating index.ts.hbs', 'templating package.json.hbs', diff --git a/packages/cli/src/lib/new/factories/frontendPlugin.test.ts b/packages/cli/src/lib/new/factories/frontendPlugin.test.ts index 55e569a15f..a4a89c3245 100644 --- a/packages/cli/src/lib/new/factories/frontendPlugin.test.ts +++ b/packages/cli/src/lib/new/factories/frontendPlugin.test.ts @@ -91,7 +91,7 @@ describe('frontendPlugin factory', () => { `availability plugins${sep}test`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.tsx.hbs', diff --git a/packages/cli/src/lib/new/factories/nodeLibraryPackage.test.ts b/packages/cli/src/lib/new/factories/nodeLibraryPackage.test.ts index 69999761e9..929182d84e 100644 --- a/packages/cli/src/lib/new/factories/nodeLibraryPackage.test.ts +++ b/packages/cli/src/lib/new/factories/nodeLibraryPackage.test.ts @@ -75,7 +75,7 @@ describe('nodeLibraryPackage factory', () => { `availability ${joinPath('packages', expectedNodeLibraryPackageName)}`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/src/lib/new/factories/pluginCommon.test.ts b/packages/cli/src/lib/new/factories/pluginCommon.test.ts index 8f01401435..6dfcc83536 100644 --- a/packages/cli/src/lib/new/factories/pluginCommon.test.ts +++ b/packages/cli/src/lib/new/factories/pluginCommon.test.ts @@ -73,7 +73,7 @@ describe('pluginCommon factory', () => { `availability plugins${sep}test-common`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/src/lib/new/factories/pluginNode.test.ts b/packages/cli/src/lib/new/factories/pluginNode.test.ts index b6aad0a129..e6ff093e22 100644 --- a/packages/cli/src/lib/new/factories/pluginNode.test.ts +++ b/packages/cli/src/lib/new/factories/pluginNode.test.ts @@ -73,7 +73,7 @@ describe('pluginNode factory', () => { `availability plugins${sep}test-node`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/src/lib/new/factories/pluginWeb.test.ts b/packages/cli/src/lib/new/factories/pluginWeb.test.ts index 20fbbbbdb7..ddc4920fb8 100644 --- a/packages/cli/src/lib/new/factories/pluginWeb.test.ts +++ b/packages/cli/src/lib/new/factories/pluginWeb.test.ts @@ -73,7 +73,7 @@ describe('pluginWeb factory', () => { `availability plugins${sep}test-react`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/src/lib/new/factories/scaffolderModule.test.ts b/packages/cli/src/lib/new/factories/scaffolderModule.test.ts index 60dade618a..bdcd8b6bdc 100644 --- a/packages/cli/src/lib/new/factories/scaffolderModule.test.ts +++ b/packages/cli/src/lib/new/factories/scaffolderModule.test.ts @@ -89,7 +89,7 @@ describe('scaffolderModule factory', () => { `availability plugins${sep}scaffolder-backend-module-test`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/src/lib/new/factories/webLibraryPackage.test.ts b/packages/cli/src/lib/new/factories/webLibraryPackage.test.ts index 691f0b9f1c..942b66f350 100644 --- a/packages/cli/src/lib/new/factories/webLibraryPackage.test.ts +++ b/packages/cli/src/lib/new/factories/webLibraryPackage.test.ts @@ -75,7 +75,7 @@ describe('webLibraryPackage factory', () => { `availability ${joinPath('packages', expectedwebLibraryPackageName)}`, 'creating temp dir', 'Executing Template:', - 'copying .eslintrc.js', + 'templating .eslintrc.js.hbs', 'templating README.md.hbs', 'templating package.json.hbs', 'templating index.ts.hbs', diff --git a/packages/cli/templates/default-backend-module/.eslintrc.js b/packages/cli/templates/default-backend-module/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/default-backend-module/.eslintrc.js rename to packages/cli/templates/default-backend-module/.eslintrc.js.hbs diff --git a/packages/cli/templates/default-backend-plugin/.eslintrc.js b/packages/cli/templates/default-backend-plugin/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/default-backend-plugin/.eslintrc.js rename to packages/cli/templates/default-backend-plugin/.eslintrc.js.hbs diff --git a/packages/cli/templates/default-common-plugin-package/.eslintrc.js b/packages/cli/templates/default-common-plugin-package/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/default-common-plugin-package/.eslintrc.js rename to packages/cli/templates/default-common-plugin-package/.eslintrc.js.hbs diff --git a/packages/cli/templates/default-node-plugin-package/.eslintrc.js b/packages/cli/templates/default-node-plugin-package/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/default-node-plugin-package/.eslintrc.js rename to packages/cli/templates/default-node-plugin-package/.eslintrc.js.hbs diff --git a/packages/cli/templates/default-plugin/.eslintrc.js b/packages/cli/templates/default-plugin/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/default-plugin/.eslintrc.js rename to packages/cli/templates/default-plugin/.eslintrc.js.hbs diff --git a/packages/cli/templates/default-react-plugin-package/.eslintrc.js b/packages/cli/templates/default-react-plugin-package/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/default-react-plugin-package/.eslintrc.js rename to packages/cli/templates/default-react-plugin-package/.eslintrc.js.hbs diff --git a/packages/cli/templates/node-library-package/.eslintrc.js b/packages/cli/templates/node-library-package/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/node-library-package/.eslintrc.js rename to packages/cli/templates/node-library-package/.eslintrc.js.hbs diff --git a/packages/cli/templates/scaffolder-module/.eslintrc.js b/packages/cli/templates/scaffolder-module/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/scaffolder-module/.eslintrc.js rename to packages/cli/templates/scaffolder-module/.eslintrc.js.hbs diff --git a/packages/cli/templates/web-library-package/.eslintrc.js b/packages/cli/templates/web-library-package/.eslintrc.js.hbs similarity index 100% rename from packages/cli/templates/web-library-package/.eslintrc.js rename to packages/cli/templates/web-library-package/.eslintrc.js.hbs From fec7278938e07ffd583be358f15f87285238322c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 11:39:01 +0200 Subject: [PATCH 2/9] cli-node: add utility for generating package dependency hash Signed-off-by: Patrik Oldsberg --- .changeset/calm-owls-move.md | 5 ++ packages/cli-node/report.api.md | 2 + packages/cli-node/src/monorepo/Lockfile.ts | 8 +++ .../cli-node/src/monorepo/PackageGraph.ts | 51 +++++++++++++++++-- 4 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 .changeset/calm-owls-move.md diff --git a/.changeset/calm-owls-move.md b/.changeset/calm-owls-move.md new file mode 100644 index 0000000000..a4dcaeeff8 --- /dev/null +++ b/.changeset/calm-owls-move.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli-node': patch +--- + +Added new `packageGraph.getDependencyHash(name)` utility. diff --git a/packages/cli-node/report.api.md b/packages/cli-node/report.api.md index 97dab9684d..6c2294012d 100644 --- a/packages/cli-node/report.api.md +++ b/packages/cli-node/report.api.md @@ -92,6 +92,7 @@ export function isMonoRepo(): Promise; export class Lockfile { createSimplifiedDependencyGraph(): Map>; diff(otherLockfile: Lockfile): LockfileDiff; + getVersions(name: string): string[]; static load(path: string): Promise; static parse(content: string): Lockfile; } @@ -116,6 +117,7 @@ export class PackageGraph extends Map { collectFn: (pkg: PackageGraphNode) => Iterable | undefined, ): Set; static fromPackages(packages: Package[]): PackageGraph; + getDependencyHash(name: string): Promise; listChangedPackages(options: { ref: string; analyzeLockfile?: boolean; diff --git a/packages/cli-node/src/monorepo/Lockfile.ts b/packages/cli-node/src/monorepo/Lockfile.ts index 8a481c77ad..d7336cf970 100644 --- a/packages/cli-node/src/monorepo/Lockfile.ts +++ b/packages/cli-node/src/monorepo/Lockfile.ts @@ -133,6 +133,14 @@ export class Lockfile { private readonly data: LockfileData, ) {} + /** + * Returns all versions of a package in the lockfile. + */ + getVersions(name: string): string[] { + const queries = this.packages.get(name); + return queries ? queries.map(q => q.version) : []; + } + /** * Creates a simplified dependency graph from the lockfile data, where each * key is a package, and the value is a set of all packages that it depends on diff --git a/packages/cli-node/src/monorepo/PackageGraph.ts b/packages/cli-node/src/monorepo/PackageGraph.ts index 69ec99c2dd..91c5312bc1 100644 --- a/packages/cli-node/src/monorepo/PackageGraph.ts +++ b/packages/cli-node/src/monorepo/PackageGraph.ts @@ -15,6 +15,7 @@ */ import path from 'path'; +import crypto from 'node:crypto'; import { getPackages, Package } from '@manypkg/get-packages'; import { paths } from '../paths'; import { PackageRole } from '../roles'; @@ -283,6 +284,43 @@ export class PackageGraph extends Map { return targets; } + /** + * Generates a sha1 hex hash of the dependency graph for a package. + */ + async getDependencyHash(name: string): Promise { + const pkg = this.get(name); + if (!pkg) { + throw new Error(`Package '${name}' not found`); + } + + const lockfile = await this.#getLockfile(); + const depGraph = lockfile.createSimplifiedDependencyGraph(); + + const seen = new Set(); + const queue = [name]; + + while (queue.length > 0) { + const deps = depGraph.get(queue.pop()!); + if (deps) { + for (const dep of deps) { + if (!seen.has(dep)) { + seen.add(dep); + queue.push(dep); + } + } + } + } + + const hash = crypto.createHash('sha1'); + for (const dep of Array.from(seen).sort()) { + hash.update(dep); + hash.update('\0'); + hash.update(lockfile.getVersions(dep).join(' ')); + hash.update('\0'); + } + return hash.digest('hex'); + } + /** * Lists all packages that have changed since a given git ref. * @@ -342,9 +380,7 @@ export class PackageGraph extends Map { let thisLockfile: Lockfile; let otherLockfile: Lockfile; try { - thisLockfile = await Lockfile.load( - paths.resolveTargetRoot('yarn.lock'), - ); + thisLockfile = await this.#getLockfile(); otherLockfile = Lockfile.parse( await GitUtils.readFileAtRef('yarn.lock', options.ref), ); @@ -410,4 +446,13 @@ export class PackageGraph extends Map { return result; } + + #lockfilePromise?: Promise; + #getLockfile(): Promise { + if (this.#lockfilePromise) { + return this.#lockfilePromise; + } + this.#lockfilePromise = Lockfile.load(paths.resolveTargetRoot('yarn.lock')); + return this.#lockfilePromise; + } } From 8fe740dc95d0f12b9aa0a21f67b0a43fe4b11afd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 11:51:28 +0200 Subject: [PATCH 3/9] cli: added option to cache successful lint runs with repo lint Signed-off-by: Patrik Oldsberg --- .changeset/small-donkeys-attack.md | 5 + packages/cli/cli-report.md | 1 + packages/cli/src/commands/index.ts | 4 + packages/cli/src/commands/repo/lint.ts | 131 +++++++++++++++++++++++-- 4 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 .changeset/small-donkeys-attack.md diff --git a/.changeset/small-donkeys-attack.md b/.changeset/small-donkeys-attack.md new file mode 100644 index 0000000000..97a65cbfd9 --- /dev/null +++ b/.changeset/small-donkeys-attack.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +Added a new `--cache [path]` option to the `backstage-cli repo lint` command. The cache keeps track of successful lint runs and avoids re-running linting of individual packages if they haven't changed. This option is primarily intended to be used in CI. diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index 70dc2614cc..64348a872e 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -446,6 +446,7 @@ Usage: backstage-cli repo lint [options] Options: --format --since + --cache [path] --fix -h, --help ``` diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index b54536cd55..9ae52a4972 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -61,6 +61,10 @@ export function registerRepoCommand(program: Command) { '--since ', 'Only lint packages that changed since the specified ref', ) + .option( + '--cache [path]', + 'Enable caching, storing it in node_modules/.cache/backstage-cli by default, or at the provided directory', + ) .option('--fix', 'Attempt to automatically fix violations') .action(lazy(() => import('./repo/lint').then(m => m.command))); diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index d0d6ee8f06..a6381d3520 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -16,7 +16,9 @@ import chalk from 'chalk'; import { Command, OptionValues } from 'commander'; -import { relative as relativePath } from 'path'; +import fs from 'fs-extra'; +import { createHash } from 'crypto'; +import { relative as relativePath, resolve as resolvePath } from 'path'; import { PackageGraph, BackstagePackageJson } from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; @@ -30,11 +32,42 @@ function depCount(pkg: BackstagePackageJson) { return deps + devDeps; } +const CACHE_FILE_NAME = 'lint-cache.json'; + +type Cache = string[]; + +async function readCache(dir: string): Promise { + try { + const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); + if (!Array.isArray(data)) { + return undefined; + } + if (data.some(x => typeof x !== 'string')) { + return undefined; + } + return data as Cache; + } catch { + return undefined; + } +} + +async function writeCache(dir: string, cache: Cache) { + await fs.mkdirp(dir); + await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); +} + export async function command(opts: OptionValues, cmd: Command): Promise { let packages = await PackageGraph.listTargetPackages(); + const cacheDir = + opts.cache === true + ? paths.resolveTargetRoot('node_modules/.cache/backstage-cli') + : opts.cache; + const cache = cacheDir ? await readCache(cacheDir) : undefined; + + const graph = PackageGraph.fromPackages(packages); + if (opts.since) { - const graph = PackageGraph.fromPackages(packages); packages = await graph.listChangedPackages({ ref: opts.since, analyzeLockfile: true, @@ -57,26 +90,61 @@ export async function command(opts: OptionValues, cmd: Command): Promise { const parseLintScript = createScriptOptionsParser(cmd, ['package', 'lint']); + const items = await Promise.all( + packages.map(async pkg => { + const base = { + fullDir: pkg.dir, + relativeDir: relativePath(paths.targetRoot, pkg.dir), + lintOptions: parseLintScript(pkg.packageJson.scripts?.lint), + parentHash: undefined, + }; + + if (!cacheDir) { + return base; + } + + const hash = createHash('sha1'); + + hash.update(await graph.getDependencyHash(pkg.packageJson.name)); + hash.update('\0'); + hash.update(process.version); // Node.js version + hash.update('\0'); + hash.update('v1'); // The version of this implementation + + return { + ...base, + parentHash: hash.digest('hex'), + }; + }), + ); + const resultsList = await runWorkerQueueThreads({ - items: packages.map(pkg => ({ - fullDir: pkg.dir, - relativeDir: relativePath(paths.targetRoot, pkg.dir), - lintOptions: parseLintScript(pkg.packageJson.scripts?.lint), - })), + items, workerData: { fix: Boolean(opts.fix), format: opts.format as string | undefined, + shouldCache: Boolean(cacheDir), + successCache: cache, }, - workerFactory: async ({ fix, format }) => { + workerFactory: async ({ fix, format, shouldCache, successCache }) => { const { ESLint } = require('eslint') as typeof import('eslint'); + const crypto = require('crypto') as typeof import('crypto'); + const recursiveReadDir = + require('recursive-readdir') as typeof import('recursive-readdir'); + const { readFile } = + require('fs/promises') as typeof import('fs/promises'); + const { relative: workerRelativePath } = + require('path') as typeof import('path'); return async ({ fullDir, relativeDir, lintOptions, + parentHash, }): Promise<{ relativeDir: string; - resultText: string; + sha?: string; + resultText?: string; failed: boolean; }> => { // Bit of a hack to make file resolutions happen from the correct directory @@ -89,6 +157,35 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix, extensions: ['js', 'jsx', 'ts', 'tsx', 'mjs', 'cjs'], }); + + let sha: string | undefined = undefined; + if (shouldCache) { + const result = await recursiveReadDir(fullDir); + + const hash = crypto.createHash('sha1'); + hash.update(parentHash!); + hash.update('\0'); + + for (const path of result.sort()) { + if (await eslint.isPathIgnored(path)) { + continue; + } + hash.update(workerRelativePath(fullDir, path)); + hash.update('\0'); + hash.update(await readFile(path)); + hash.update('\0'); + hash.update( + JSON.stringify(await eslint.calculateConfigForFile(path)), + ); + hash.update('\0'); + } + sha = await hash.digest('hex'); + if (successCache?.includes(sha)) { + console.log(`Skipped ${relativeDir} due to cache hit`); + return { relativeDir, sha, failed: false }; + } + } + const formatter = await eslint.loadFormatter(format); const results = await eslint.lintFiles(['.']); @@ -112,13 +209,21 @@ export async function command(opts: OptionValues, cmd: Command): Promise { relativeDir, resultText, failed, + sha, }; }; }, }); + const outputSuccessCache = []; + let failed = false; - for (const { relativeDir, resultText, failed: runFailed } of resultsList) { + for (const { + relativeDir, + resultText, + failed: runFailed, + sha, + } of resultsList) { if (runFailed) { console.log(chalk.red(`Lint failed in ${relativeDir}`)); failed = true; @@ -129,9 +234,15 @@ export async function command(opts: OptionValues, cmd: Command): Promise { console.log(); console.log(resultText.trimStart()); } + } else if (sha) { + outputSuccessCache.push(sha); } } + if (cacheDir) { + await writeCache(cacheDir, outputSuccessCache); + } + if (failed) { process.exit(1); } From d4ae3b2570c09d6ac22fd0598e1b65f7c7d1402d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 12:04:01 +0200 Subject: [PATCH 4/9] docs/tooling: document caching options Signed-off-by: Patrik Oldsberg --- docs/tooling/cli/02-build-system.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/tooling/cli/02-build-system.md b/docs/tooling/cli/02-build-system.md index ab7dc07802..c922134a7c 100644 --- a/docs/tooling/cli/02-build-system.md +++ b/docs/tooling/cli/02-build-system.md @@ -559,6 +559,14 @@ The overrides in a single `package.json` may for example look like this: }, ``` +## Caching + +Caching is used sparingly throughout the Backstage build system. It is always used as a way to squeeze out a little bit of extra performance, rather than requirement to keep things fast. The following is a list of places where optional caching is available: + +- **TypeScript** - The default `tsconfig.json` used by Backstage projects has `incremental` set to `true`, which enables local caching of type checking results. It is however generally not recommended in CI, where `yarn tsc:full` is preferred, which sets `--incremental false`. +- **Linting** - The `backstage-cli repo lint` command has a `--cache` flag that enables caching of successful linting results. This is done at the package level, meaning that if a package has not been changed since the last lint run and it was successful, the linting will be skipped. This is recommended to be used in CI, but not during local development. +- **Webpack** - It is possible to enable experimental caching of frontend package builds using the `BACKSTAGE_CLI_EXPERIMENTAL_BUILD_CACHE` environment variable. This will enable the Webpack filesystem cache. + ### Debugging Jest Tests For your productivity working with unit tests it's quite essential to have your debugging configured in IDE. It will help you to identify the root cause of the issue faster. From de8732596a978adab4e66f23dcc1a3cd65c1b8cf Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 12:24:09 +0200 Subject: [PATCH 5/9] .github/workflows: enable repo lint cache Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 2 +- .github/workflows/deploy_packages.yml | 2 +- .github/workflows/verify_windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 886abafcd7..28c53db776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,7 +234,7 @@ jobs: run: node scripts/verify-release.js - name: lint changed packages - run: yarn backstage-cli repo lint --since origin/master + run: yarn backstage-cli repo lint --since origin/master --cache - name: test changed packages run: yarn backstage-cli repo test --maxWorkers=3 --workerIdleMemoryLimit=1300M --since origin/master diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index 92210e90be..b09f419fed 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -94,7 +94,7 @@ jobs: run: yarn backstage-cli config:check --lax - name: lint - run: yarn backstage-cli repo lint + run: yarn backstage-cli repo lint --cache - name: type checking and declarations run: yarn tsc:full diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index 9846ff28d2..113c76d497 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -46,7 +46,7 @@ jobs: run: yarn install --immutable - name: lint - run: yarn backstage-cli repo lint + run: yarn backstage-cli repo lint --cache - name: type checking and declarations run: yarn tsc:full From fede18031dbd30ef954e7aa0524ad3bfe7df11b2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 12:30:47 +0200 Subject: [PATCH 6/9] cli: include package lint options in repo lint cache key Signed-off-by: Patrik Oldsberg --- packages/cli/src/commands/repo/lint.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index a6381d3520..b5a89471a7 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -92,10 +92,11 @@ export async function command(opts: OptionValues, cmd: Command): Promise { const items = await Promise.all( packages.map(async pkg => { + const lintOptions = parseLintScript(pkg.packageJson.scripts?.lint); const base = { fullDir: pkg.dir, relativeDir: relativePath(paths.targetRoot, pkg.dir), - lintOptions: parseLintScript(pkg.packageJson.scripts?.lint), + lintOptions, parentHash: undefined, }; @@ -107,6 +108,8 @@ export async function command(opts: OptionValues, cmd: Command): Promise { hash.update(await graph.getDependencyHash(pkg.packageJson.name)); hash.update('\0'); + hash.update(JSON.stringify(lintOptions)); + hash.update('\0'); hash.update(process.version); // Node.js version hash.update('\0'); hash.update('v1'); // The version of this implementation From e1bc9cc97dad94d45b3ca8a4b64d5ecc8a593b24 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 13:36:31 +0200 Subject: [PATCH 7/9] cli-node: refactor to move getDependencyHash to Lockfile Signed-off-by: Patrik Oldsberg --- .changeset/calm-owls-move.md | 2 +- packages/cli-node/report.api.md | 3 +- packages/cli-node/src/monorepo/Lockfile.ts | 68 ++++++++++++++++--- .../cli-node/src/monorepo/PackageGraph.ts | 51 +------------- packages/cli/src/commands/repo/lint.ts | 24 +++++-- 5 files changed, 82 insertions(+), 66 deletions(-) diff --git a/.changeset/calm-owls-move.md b/.changeset/calm-owls-move.md index a4dcaeeff8..bd7fb72508 100644 --- a/.changeset/calm-owls-move.md +++ b/.changeset/calm-owls-move.md @@ -2,4 +2,4 @@ '@backstage/cli-node': patch --- -Added new `packageGraph.getDependencyHash(name)` utility. +Added new `lockfile.getDependencyTreeHash(name)` utility. diff --git a/packages/cli-node/report.api.md b/packages/cli-node/report.api.md index 6c2294012d..eb18e2e9f8 100644 --- a/packages/cli-node/report.api.md +++ b/packages/cli-node/report.api.md @@ -92,7 +92,7 @@ export function isMonoRepo(): Promise; export class Lockfile { createSimplifiedDependencyGraph(): Map>; diff(otherLockfile: Lockfile): LockfileDiff; - getVersions(name: string): string[]; + getDependencyTreeHash(startName: string): string; static load(path: string): Promise; static parse(content: string): Lockfile; } @@ -117,7 +117,6 @@ export class PackageGraph extends Map { collectFn: (pkg: PackageGraphNode) => Iterable | undefined, ): Set; static fromPackages(packages: Package[]): PackageGraph; - getDependencyHash(name: string): Promise; listChangedPackages(options: { ref: string; analyzeLockfile?: boolean; diff --git a/packages/cli-node/src/monorepo/Lockfile.ts b/packages/cli-node/src/monorepo/Lockfile.ts index d7336cf970..7c4dc04f33 100644 --- a/packages/cli-node/src/monorepo/Lockfile.ts +++ b/packages/cli-node/src/monorepo/Lockfile.ts @@ -15,6 +15,7 @@ */ import { parseSyml } from '@yarnpkg/parsers'; +import crypto from 'node:crypto'; import fs from 'fs-extra'; const ENTRY_PATTERN = /^((?:@[^/]+\/)?[^@/]+)@(.+)$/; @@ -133,14 +134,6 @@ export class Lockfile { private readonly data: LockfileData, ) {} - /** - * Returns all versions of a package in the lockfile. - */ - getVersions(name: string): string[] { - const queries = this.packages.get(name); - return queries ? queries.map(q => q.version) : []; - } - /** * Creates a simplified dependency graph from the lockfile data, where each * key is a package, and the value is a set of all packages that it depends on @@ -223,4 +216,63 @@ export class Lockfile { return diff; } + + /** + * Generates a sha1 hex hash of the dependency graph for a package. + */ + getDependencyTreeHash(startName: string): string { + if (!this.packages.has(startName)) { + throw new Error(`Package '${startName}' not found in lockfile`); + } + + const hash = crypto.createHash('sha1'); + + const queue = [startName]; + const seen = new Set(); + + while (queue.length > 0) { + const name = queue.pop()!; + + if (seen.has(name)) { + continue; + } + seen.add(name); + + const entries = this.packages.get(name); + if (!entries) { + continue; // In case of missing optional peer dependencies + } + + hash.update(`pkg:${name}`); + hash.update('\0'); + + // TODO(Rugvip): This uses the same simplified lookup as createSimplifiedDependencyGraph() + // we could match version queries to make the resulting tree a bit smaller. + const deps = new Array(); + for (const entry of entries) { + // We're not being particular about stable ordering here. If the lockfile ordering changes, so will likely hash. + hash.update(entry.version); + + const data = this.data[entry.dataKey]; + if (!data) { + continue; + } + + const checksum = data.checksum || data.integrity; + if (checksum) { + hash.update('#'); + hash.update(checksum); + } + + hash.update(' '); + + deps.push(...Object.keys(data.dependencies ?? {})); + deps.push(...Object.keys(data.peerDependencies ?? {})); + } + + queue.push(...new Set(deps)); + } + + return hash.digest('hex'); + } } diff --git a/packages/cli-node/src/monorepo/PackageGraph.ts b/packages/cli-node/src/monorepo/PackageGraph.ts index 91c5312bc1..69ec99c2dd 100644 --- a/packages/cli-node/src/monorepo/PackageGraph.ts +++ b/packages/cli-node/src/monorepo/PackageGraph.ts @@ -15,7 +15,6 @@ */ import path from 'path'; -import crypto from 'node:crypto'; import { getPackages, Package } from '@manypkg/get-packages'; import { paths } from '../paths'; import { PackageRole } from '../roles'; @@ -284,43 +283,6 @@ export class PackageGraph extends Map { return targets; } - /** - * Generates a sha1 hex hash of the dependency graph for a package. - */ - async getDependencyHash(name: string): Promise { - const pkg = this.get(name); - if (!pkg) { - throw new Error(`Package '${name}' not found`); - } - - const lockfile = await this.#getLockfile(); - const depGraph = lockfile.createSimplifiedDependencyGraph(); - - const seen = new Set(); - const queue = [name]; - - while (queue.length > 0) { - const deps = depGraph.get(queue.pop()!); - if (deps) { - for (const dep of deps) { - if (!seen.has(dep)) { - seen.add(dep); - queue.push(dep); - } - } - } - } - - const hash = crypto.createHash('sha1'); - for (const dep of Array.from(seen).sort()) { - hash.update(dep); - hash.update('\0'); - hash.update(lockfile.getVersions(dep).join(' ')); - hash.update('\0'); - } - return hash.digest('hex'); - } - /** * Lists all packages that have changed since a given git ref. * @@ -380,7 +342,9 @@ export class PackageGraph extends Map { let thisLockfile: Lockfile; let otherLockfile: Lockfile; try { - thisLockfile = await this.#getLockfile(); + thisLockfile = await Lockfile.load( + paths.resolveTargetRoot('yarn.lock'), + ); otherLockfile = Lockfile.parse( await GitUtils.readFileAtRef('yarn.lock', options.ref), ); @@ -446,13 +410,4 @@ export class PackageGraph extends Map { return result; } - - #lockfilePromise?: Promise; - #getLockfile(): Promise { - if (this.#lockfilePromise) { - return this.#lockfilePromise; - } - this.#lockfilePromise = Lockfile.load(paths.resolveTargetRoot('yarn.lock')); - return this.#lockfilePromise; - } } diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index b5a89471a7..655e524d19 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -19,7 +19,11 @@ import { Command, OptionValues } from 'commander'; import fs from 'fs-extra'; import { createHash } from 'crypto'; import { relative as relativePath, resolve as resolvePath } from 'path'; -import { PackageGraph, BackstagePackageJson } from '@backstage/cli-node'; +import { + PackageGraph, + BackstagePackageJson, + Lockfile, +} from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; import { createScriptOptionsParser } from './optionsParser'; @@ -63,11 +67,15 @@ export async function command(opts: OptionValues, cmd: Command): Promise { opts.cache === true ? paths.resolveTargetRoot('node_modules/.cache/backstage-cli') : opts.cache; - const cache = cacheDir ? await readCache(cacheDir) : undefined; - - const graph = PackageGraph.fromPackages(packages); + const cacheContext = cacheDir + ? { + cache: await readCache(cacheDir), + lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')), + } + : undefined; if (opts.since) { + const graph = PackageGraph.fromPackages(packages); packages = await graph.listChangedPackages({ ref: opts.since, analyzeLockfile: true, @@ -100,13 +108,15 @@ export async function command(opts: OptionValues, cmd: Command): Promise { parentHash: undefined, }; - if (!cacheDir) { + if (!cacheContext) { return base; } const hash = createHash('sha1'); - hash.update(await graph.getDependencyHash(pkg.packageJson.name)); + hash.update( + cacheContext.lockfile.getDependencyTreeHash(pkg.packageJson.name), + ); hash.update('\0'); hash.update(JSON.stringify(lintOptions)); hash.update('\0'); @@ -127,7 +137,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix: Boolean(opts.fix), format: opts.format as string | undefined, shouldCache: Boolean(cacheDir), - successCache: cache, + successCache: cacheContext?.cache, }, workerFactory: async ({ fix, format, shouldCache, successCache }) => { const { ESLint } = require('eslint') as typeof import('eslint'); From beaa723f1e084501725829c1709630d6f31ec521 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 14:00:46 +0200 Subject: [PATCH 8/9] cli-node: added tests for lockfile.getDependencyTreeHash Signed-off-by: Patrik Oldsberg --- .../cli-node/src/monorepo/Lockfile.test.ts | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/packages/cli-node/src/monorepo/Lockfile.test.ts b/packages/cli-node/src/monorepo/Lockfile.test.ts index 4bc7a6d96e..03b50fea90 100644 --- a/packages/cli-node/src/monorepo/Lockfile.test.ts +++ b/packages/cli-node/src/monorepo/Lockfile.test.ts @@ -493,4 +493,95 @@ d@^1: ); }); }); + + describe('getDependencyTreeHash', () => { + const content = `${MODERN_HEADER} +"a@npm:^1": + version: "1.0.0" + checksum: sha512-a-1 + dependencies: + b: "^2" + +"b@npm:2.0.x, b@npm:^2": + version: "2.0.0" + checksum: sha512-b-1 + +"b@npm:4": + version: "3.0.0" + checksum: sha512-b-2 + +"c@npm:^1": + version: "4.0.0" + checksum: sha512-c-1 +`; + const lockfile = Lockfile.parse(content); + + const hashA = lockfile.getDependencyTreeHash('a'); + const hashB = lockfile.getDependencyTreeHash('b'); + const hashC = lockfile.getDependencyTreeHash('c'); + + it('should generate stable dependency hashes', () => { + expect(hashA).toMatchInlineSnapshot( + `"2d1d4c1c577c291e815e87779c72fc78a78e56cc"`, + ); + expect(hashB).toMatchInlineSnapshot( + `"7e46d0c7337540179b442c87a7c5555543798f15"`, + ); + expect(hashC).toMatchInlineSnapshot( + `"e65103abd217954bad40e2f834b990a5e6fa4054"`, + ); + }); + + it('should generate different hashes for different versions', () => { + const lockfileNewA = Lockfile.parse(content.replace('1.0.0', '1.0.1')); + expect(lockfileNewA.getDependencyTreeHash('a')).not.toBe(hashA); + expect(lockfileNewA.getDependencyTreeHash('b')).toBe(hashB); + expect(lockfileNewA.getDependencyTreeHash('c')).toBe(hashC); + + const lockfileNewB1 = Lockfile.parse(content.replace('2.0.0', '2.0.1')); + expect(lockfileNewB1.getDependencyTreeHash('a')).not.toBe(hashA); + expect(lockfileNewB1.getDependencyTreeHash('b')).not.toBe(hashB); + expect(lockfileNewB1.getDependencyTreeHash('c')).toBe(hashC); + + const lockfileNewB2 = Lockfile.parse(content.replace('3.0.0', '3.0.1')); + expect(lockfileNewB2.getDependencyTreeHash('a')).not.toBe(hashA); + expect(lockfileNewB2.getDependencyTreeHash('b')).not.toBe(hashB); + expect(lockfileNewB2.getDependencyTreeHash('c')).toBe(hashC); + + const lockfileNewC = Lockfile.parse(content.replace('4.0.0', '4.0.1')); + expect(lockfileNewC.getDependencyTreeHash('a')).toBe(hashA); + expect(lockfileNewC.getDependencyTreeHash('b')).toBe(hashB); + expect(lockfileNewC.getDependencyTreeHash('c')).not.toBe(hashC); + }); + + it('should generate different hashes for different checksums', () => { + const lockfileNewA = Lockfile.parse( + content.replace('sha512-a-1', 'sha512-a-1-new'), + ); + expect(lockfileNewA.getDependencyTreeHash('a')).not.toBe(hashA); + expect(lockfileNewA.getDependencyTreeHash('b')).toBe(hashB); + expect(lockfileNewA.getDependencyTreeHash('c')).toBe(hashC); + + const lockfileNewB1 = Lockfile.parse( + content.replace('sha512-b-1', 'sha512-b-1-new'), + ); + expect(lockfileNewB1.getDependencyTreeHash('a')).not.toBe(hashA); + expect(lockfileNewB1.getDependencyTreeHash('b')).not.toBe(hashB); + expect(lockfileNewB1.getDependencyTreeHash('c')).toBe(hashC); + + const lockfileNewB2 = Lockfile.parse( + content.replace('sha512-b-2', 'sha512-b-2-new'), + ); + expect(lockfileNewB2.getDependencyTreeHash('a')).not.toBe(hashA); + expect(lockfileNewB2.getDependencyTreeHash('b')).not.toBe(hashB); + expect(lockfileNewB2.getDependencyTreeHash('c')).toBe(hashC); + + const lockfileNewC = Lockfile.parse( + content.replace('sha512-c-1', 'sha512-c-1-new'), + ); + expect(lockfileNewC.getDependencyTreeHash('a')).toBe(hashA); + expect(lockfileNewC.getDependencyTreeHash('b')).toBe(hashB); + expect(lockfileNewC.getDependencyTreeHash('c')).not.toBe(hashC); + }); + }); }); From 50cc7351cf6ce9f42b1d248e6af62758a603d802 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 7 Oct 2024 16:37:03 +0200 Subject: [PATCH 9/9] cli: switch repo lint --cache option to --successCache and --successCacheDir Signed-off-by: Patrik Oldsberg --- .changeset/small-donkeys-attack.md | 4 +++- .github/workflows/ci.yml | 2 +- .github/workflows/deploy_packages.yml | 2 +- .github/workflows/verify_windows.yml | 2 +- docs/tooling/cli/02-build-system.md | 2 +- packages/cli/cli-report.md | 3 ++- packages/cli/src/commands/index.ts | 8 ++++++-- packages/cli/src/commands/repo/lint.ts | 13 ++++++------- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.changeset/small-donkeys-attack.md b/.changeset/small-donkeys-attack.md index 97a65cbfd9..116cc611b5 100644 --- a/.changeset/small-donkeys-attack.md +++ b/.changeset/small-donkeys-attack.md @@ -2,4 +2,6 @@ '@backstage/cli': patch --- -Added a new `--cache [path]` option to the `backstage-cli repo lint` command. The cache keeps track of successful lint runs and avoids re-running linting of individual packages if they haven't changed. This option is primarily intended to be used in CI. +Added a new `--successCache` option to the `backstage-cli repo lint` command. The cache keeps track of successful lint runs and avoids re-running linting of individual packages if they haven't changed. This option is primarily intended to be used in CI. + +In addition a `--successCacheDir` option has also been added to be able to override the default cache directory. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28c53db776..abe36f40de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,7 +234,7 @@ jobs: run: node scripts/verify-release.js - name: lint changed packages - run: yarn backstage-cli repo lint --since origin/master --cache + run: yarn backstage-cli repo lint --since origin/master --successCache - name: test changed packages run: yarn backstage-cli repo test --maxWorkers=3 --workerIdleMemoryLimit=1300M --since origin/master diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index b09f419fed..073f210384 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -94,7 +94,7 @@ jobs: run: yarn backstage-cli config:check --lax - name: lint - run: yarn backstage-cli repo lint --cache + run: yarn backstage-cli repo lint --successCache - name: type checking and declarations run: yarn tsc:full diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index 113c76d497..80db28c2e1 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -46,7 +46,7 @@ jobs: run: yarn install --immutable - name: lint - run: yarn backstage-cli repo lint --cache + run: yarn backstage-cli repo lint --successCache - name: type checking and declarations run: yarn tsc:full diff --git a/docs/tooling/cli/02-build-system.md b/docs/tooling/cli/02-build-system.md index c922134a7c..52410001f7 100644 --- a/docs/tooling/cli/02-build-system.md +++ b/docs/tooling/cli/02-build-system.md @@ -564,7 +564,7 @@ The overrides in a single `package.json` may for example look like this: Caching is used sparingly throughout the Backstage build system. It is always used as a way to squeeze out a little bit of extra performance, rather than requirement to keep things fast. The following is a list of places where optional caching is available: - **TypeScript** - The default `tsconfig.json` used by Backstage projects has `incremental` set to `true`, which enables local caching of type checking results. It is however generally not recommended in CI, where `yarn tsc:full` is preferred, which sets `--incremental false`. -- **Linting** - The `backstage-cli repo lint` command has a `--cache` flag that enables caching of successful linting results. This is done at the package level, meaning that if a package has not been changed since the last lint run and it was successful, the linting will be skipped. This is recommended to be used in CI, but not during local development. +- **Linting** - The `backstage-cli repo lint` command has a `--successCache` flag that enables caching of successful linting results. This is done at the package level, meaning that if a package has not been changed since the last lint run and it was successful, the linting will be skipped. This is recommended to be used in CI, but not during local development. - **Webpack** - It is possible to enable experimental caching of frontend package builds using the `BACKSTAGE_CLI_EXPERIMENTAL_BUILD_CACHE` environment variable. This will enable the Webpack filesystem cache. ### Debugging Jest Tests diff --git a/packages/cli/cli-report.md b/packages/cli/cli-report.md index 64348a872e..72cf7c4505 100644 --- a/packages/cli/cli-report.md +++ b/packages/cli/cli-report.md @@ -446,7 +446,8 @@ Usage: backstage-cli repo lint [options] Options: --format --since - --cache [path] + --successCache + --successCacheDir --fix -h, --help ``` diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 9ae52a4972..0d72400bc4 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -62,8 +62,12 @@ export function registerRepoCommand(program: Command) { 'Only lint packages that changed since the specified ref', ) .option( - '--cache [path]', - 'Enable caching, storing it in node_modules/.cache/backstage-cli by default, or at the provided directory', + '--successCache', + 'Enable success caching, which skips running tests for unchanged packages that were successful in the previous run', + ) + .option( + '--successCacheDir ', + 'Set the success cache location, (default: node_modules/.cache/backstage-cli)', ) .option('--fix', 'Attempt to automatically fix violations') .action(lazy(() => import('./repo/lint').then(m => m.command))); diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index 655e524d19..8e1f6d52d7 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -63,11 +63,10 @@ async function writeCache(dir: string, cache: Cache) { export async function command(opts: OptionValues, cmd: Command): Promise { let packages = await PackageGraph.listTargetPackages(); - const cacheDir = - opts.cache === true - ? paths.resolveTargetRoot('node_modules/.cache/backstage-cli') - : opts.cache; - const cacheContext = cacheDir + const cacheDir = resolvePath( + opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', + ); + const cacheContext = opts.successCache ? { cache: await readCache(cacheDir), lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')), @@ -136,7 +135,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { workerData: { fix: Boolean(opts.fix), format: opts.format as string | undefined, - shouldCache: Boolean(cacheDir), + shouldCache: Boolean(cacheContext), successCache: cacheContext?.cache, }, workerFactory: async ({ fix, format, shouldCache, successCache }) => { @@ -252,7 +251,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } } - if (cacheDir) { + if (cacheContext) { await writeCache(cacheDir, outputSuccessCache); }