From 04297a0c119e259f96a7e6c95773e44678f23840 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 17 Oct 2024 09:55:20 +0200 Subject: [PATCH] cli: refactor success cache to be additive Signed-off-by: Patrik Oldsberg --- .changeset/loud-trainers-yawn.md | 5 ++ packages/cli/src/commands/repo/lint.ts | 40 ++-------- packages/cli/src/commands/repo/test.ts | 39 ++-------- packages/cli/src/lib/cache/SuccessCache.ts | 89 ++++++++++++++++++++++ 4 files changed, 107 insertions(+), 66 deletions(-) create mode 100644 .changeset/loud-trainers-yawn.md create mode 100644 packages/cli/src/lib/cache/SuccessCache.ts diff --git a/.changeset/loud-trainers-yawn.md b/.changeset/loud-trainers-yawn.md new file mode 100644 index 0000000000..b48f773caa --- /dev/null +++ b/.changeset/loud-trainers-yawn.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The `--successCache` option for the `repo test` and `repo lint` commands now use an additive store that keeps old entries around for a week before they are cleaned up automatically. diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index 1895597e96..9b1ce4ee3e 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -16,9 +16,8 @@ import chalk from 'chalk'; import { Command, OptionValues } from 'commander'; -import fs from 'fs-extra'; import { createHash } from 'crypto'; -import { relative as relativePath, resolve as resolvePath } from 'path'; +import { relative as relativePath } from 'path'; import { PackageGraph, BackstagePackageJson, @@ -27,6 +26,7 @@ import { import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; import { createScriptOptionsParser } from './optionsParser'; +import { SuccessCache } from '../../lib/cache/SuccessCache'; function depCount(pkg: BackstagePackageJson) { const deps = pkg.dependencies ? Object.keys(pkg.dependencies).length : 0; @@ -36,39 +36,13 @@ function depCount(pkg: BackstagePackageJson) { return deps + devDeps; } -const CACHE_FILE_NAME = 'lint-cache.json'; - -type Cache = string[]; - -async function readCache(dir: string): Promise { - try { - const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); - if (!Array.isArray(data)) { - return undefined; - } - if (data.some(x => typeof x !== 'string')) { - return undefined; - } - return data as Cache; - } catch { - return undefined; - } -} - -async function writeCache(dir: string, cache: Cache) { - await fs.mkdirp(dir); - await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); -} - export async function command(opts: OptionValues, cmd: Command): Promise { let packages = await PackageGraph.listTargetPackages(); - const cacheDir = resolvePath( - opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', - ); + const cache = new SuccessCache('lint', opts.successCacheDir); const cacheContext = opts.successCache ? { - cache: await readCache(cacheDir), + entries: await cache.read(), lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')), } : undefined; @@ -136,7 +110,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix: Boolean(opts.fix), format: opts.format as string | undefined, shouldCache: Boolean(cacheContext), - successCache: cacheContext?.cache, + successCache: cacheContext?.entries, rootDir: paths.targetRoot, }, workerFactory: async ({ @@ -202,7 +176,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { hash.update('\0'); } sha = await hash.digest('hex'); - if (successCache?.includes(sha)) { + if (successCache?.has(sha)) { console.log(`Skipped ${relativeDir} due to cache hit`); return { relativeDir, sha, failed: false }; } @@ -262,7 +236,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } if (cacheContext) { - await writeCache(cacheDir, outputSuccessCache); + await cache.write(outputSuccessCache); } if (failed) { diff --git a/packages/cli/src/commands/repo/test.ts b/packages/cli/src/commands/repo/test.ts index 1d59ba3d63..1035354eb2 100644 --- a/packages/cli/src/commands/repo/test.ts +++ b/packages/cli/src/commands/repo/test.ts @@ -16,14 +16,14 @@ import os from 'os'; import crypto from 'node:crypto'; -import fs from 'fs-extra'; import yargs from 'yargs'; -import { resolve as resolvePath, relative as relativePath } from 'path'; +import { relative as relativePath } from 'path'; import { Command, OptionValues } from 'commander'; import { Lockfile, PackageGraph } from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runCheck, runPlain } from '../../lib/run'; import { isChildPath } from '@backstage/cli-common'; +import { SuccessCache } from '../../lib/cache/SuccessCache'; type JestProject = { displayName: string; @@ -50,30 +50,6 @@ interface GlobalWithCache extends Global { }; } -const CACHE_FILE_NAME = 'test-cache.json'; - -type Cache = string[]; - -async function readCache(dir: string): Promise { - try { - const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); - if (!Array.isArray(data)) { - return undefined; - } - if (data.some(x => typeof x !== 'string')) { - return undefined; - } - return data as Cache; - } catch { - return undefined; - } -} - -function writeCache(dir: string, cache: Cache) { - fs.mkdirpSync(dir); - fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); -} - /** * Use git to get the HEAD tree hashes of each package in the project. */ @@ -272,10 +248,6 @@ export async function command(opts: OptionValues, cmd: Command): Promise { removeOptionArg(args, '--successCache', 1); removeOptionArg(args, '--successCacheDir'); - const cacheDir = resolvePath( - opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', - ); - // Parse the args to ensure that no file filters are provided, in which case we refuse to run const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions) .argv; @@ -293,6 +265,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { ); } + const cache = new SuccessCache('test', opts.successCacheDir); const graph = await getPackageGraph(); // Shared state for the bridge @@ -305,7 +278,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { globalWithCache.__backstageCli_jestSuccessCache = { // This is called by `config/jest.js` after the project configs have been gathered async filterConfigs(projectConfigs, globalRootConfig) { - const cache = await readCache(cacheDir); + const cacheEntries = await cache.read(); const lockfile = await Lockfile.load( paths.resolveTargetRoot('yarn.lock'), ); @@ -350,7 +323,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { projectHashes.set(packageName, sha); - if (cache?.includes(sha)) { + if (cacheEntries.has(sha)) { if (!selectedProjects || selectedProjects.includes(packageName)) { console.log(`Skipped ${packageName} due to cache hit`); } @@ -391,7 +364,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } } - await writeCache(cacheDir, outputSuccessCache); + await cache.write(outputSuccessCache); }, }; } diff --git a/packages/cli/src/lib/cache/SuccessCache.ts b/packages/cli/src/lib/cache/SuccessCache.ts new file mode 100644 index 0000000000..29f4301eea --- /dev/null +++ b/packages/cli/src/lib/cache/SuccessCache.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'node:path'; + +const DEFAULT_CACHE_BASE_PATH = 'node_modules/.cache/backstage-cli'; + +const CACHE_MAX_AGE_MS = 7 * 24 * 3600_000; + +export class SuccessCache { + readonly #path: string; + + constructor(name: string, basePath?: string) { + this.#path = resolvePath(basePath ?? DEFAULT_CACHE_BASE_PATH, name); + } + + async read(): Promise> { + const state = await fs.stat(this.#path).catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + if (!state || !state.isDirectory()) { + return new Set(); + } + const items = await fs.readdir(this.#path); + + const returned = new Set(); + const removed = new Set(); + + const now = Date.now(); + + for (const item of items) { + const split = item.split('_'); + if (split.length !== 2) { + removed.add(item); + continue; + } + const createdAt = parseInt(split[0], 10); + if (Number.isNaN(createdAt) || now - createdAt > CACHE_MAX_AGE_MS) { + removed.add(item); + } else { + returned.add(split[1]); + } + } + + for (const item of removed) { + await fs.unlink(resolvePath(this.#path, item)); + } + + return returned; + } + + async write(newEntries: Iterable): Promise { + const now = Date.now(); + + await fs.ensureDir(this.#path); + + const existingItems = await fs.readdir(this.#path); + + const empty = Buffer.alloc(0); + for (const key of newEntries) { + // Remove any existing items with the key we're about to add + const trimmedItems = existingItems.filter(item => + item.endsWith(`_${key}`), + ); + for (const trimmedItem of trimmedItems) { + await fs.unlink(resolvePath(this.#path, trimmedItem)); + } + + await fs.writeFile(resolvePath(this.#path, `${now}_${key}`), empty); + } + } +}