paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cli: refactor success cache to be additive

Browse Source 
Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
2024-10-17 09:55:20 +02:00
parent 232a9600f0
commit 04297a0c11
4 changed files with 107 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/loud-trainers-yawn.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/cli': patch
---
The `--successCache` option for the `repo test` and `repo lint` commands now use an additive store that keeps old entries around for a week before they are cleaned up automatically.
packages/cli/src/commands/repo/lint.ts
+7 -33
View File
@@ -16,9 +16,8 @@
import chalk from 'chalk';
import { Command, OptionValues } from 'commander';
import fs from 'fs-extra';
import { createHash } from 'crypto';
import { relative as relativePath, resolve as resolvePath } from 'path';
import { relative as relativePath } from 'path';
import {
PackageGraph,
BackstagePackageJson,
@@ -27,6 +26,7 @@ import {
import { paths } from '../../lib/paths';
import { runWorkerQueueThreads } from '../../lib/parallel';
import { createScriptOptionsParser } from './optionsParser';
import { SuccessCache } from '../../lib/cache/SuccessCache';
function depCount(pkg: BackstagePackageJson) {
const deps = pkg.dependencies ? Object.keys(pkg.dependencies).length : 0;
@@ -36,39 +36,13 @@ function depCount(pkg: BackstagePackageJson) {
return deps + devDeps;
}
const CACHE_FILE_NAME = 'lint-cache.json';
type Cache = string[];
async function readCache(dir: string): Promise<Cache | undefined> {
try {
const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME));
if (!Array.isArray(data)) {
return undefined;
}
if (data.some(x => typeof x !== 'string')) {
return undefined;
}
return data as Cache;
} catch {
return undefined;
}
}
async function writeCache(dir: string, cache: Cache) {
await fs.mkdirp(dir);
await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 });
}
export async function command(opts: OptionValues, cmd: Command): Promise<void> {
let packages = await PackageGraph.listTargetPackages();
const cacheDir = resolvePath(
opts.successCacheDir ?? 'node_modules/.cache/backstage-cli',
);
const cache = new SuccessCache('lint', opts.successCacheDir);
const cacheContext = opts.successCache
? {
cache: await readCache(cacheDir),
entries: await cache.read(),
lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')),
}
: undefined;
@@ -136,7 +110,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
fix: Boolean(opts.fix),
format: opts.format as string | undefined,
shouldCache: Boolean(cacheContext),
successCache: cacheContext?.cache,
successCache: cacheContext?.entries,
rootDir: paths.targetRoot,
},
workerFactory: async ({
@@ -202,7 +176,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
hash.update('\0');
}
sha = await hash.digest('hex');
if (successCache?.includes(sha)) {
if (successCache?.has(sha)) {
console.log(`Skipped ${relativeDir} due to cache hit`);
return { relativeDir, sha, failed: false };
}
@@ -262,7 +236,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
}
if (cacheContext) {
await writeCache(cacheDir, outputSuccessCache);
await cache.write(outputSuccessCache);
}
if (failed) {
packages/cli/src/commands/repo/test.ts
+6 -33
View File
@@ -16,14 +16,14 @@
import os from 'os';
import crypto from 'node:crypto';
import fs from 'fs-extra';
import yargs from 'yargs';
import { resolve as resolvePath, relative as relativePath } from 'path';
import { relative as relativePath } from 'path';
import { Command, OptionValues } from 'commander';
import { Lockfile, PackageGraph } from '@backstage/cli-node';
import { paths } from '../../lib/paths';
import { runCheck, runPlain } from '../../lib/run';
import { isChildPath } from '@backstage/cli-common';
import { SuccessCache } from '../../lib/cache/SuccessCache';
type JestProject = {
displayName: string;
@@ -50,30 +50,6 @@ interface GlobalWithCache extends Global {
};
}
const CACHE_FILE_NAME = 'test-cache.json';
type Cache = string[];
async function readCache(dir: string): Promise<Cache | undefined> {
try {
const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME));
if (!Array.isArray(data)) {
return undefined;
}
if (data.some(x => typeof x !== 'string')) {
return undefined;
}
return data as Cache;
} catch {
return undefined;
}
}
function writeCache(dir: string, cache: Cache) {
fs.mkdirpSync(dir);
fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 });
}
/**
* Use git to get the HEAD tree hashes of each package in the project.
*/
@@ -272,10 +248,6 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
removeOptionArg(args, '--successCache', 1);
removeOptionArg(args, '--successCacheDir');
const cacheDir = resolvePath(
opts.successCacheDir ?? 'node_modules/.cache/backstage-cli',
);
// Parse the args to ensure that no file filters are provided, in which case we refuse to run
const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions)
.argv;
@@ -293,6 +265,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
);
}
const cache = new SuccessCache('test', opts.successCacheDir);
const graph = await getPackageGraph();
// Shared state for the bridge
@@ -305,7 +278,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
globalWithCache.__backstageCli_jestSuccessCache = {
// This is called by `config/jest.js` after the project configs have been gathered
async filterConfigs(projectConfigs, globalRootConfig) {
const cache = await readCache(cacheDir);
const cacheEntries = await cache.read();
const lockfile = await Lockfile.load(
paths.resolveTargetRoot('yarn.lock'),
);
@@ -350,7 +323,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
projectHashes.set(packageName, sha);
if (cache?.includes(sha)) {
if (cacheEntries.has(sha)) {
if (!selectedProjects || selectedProjects.includes(packageName)) {
console.log(`Skipped ${packageName} due to cache hit`);
}
@@ -391,7 +364,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise<void> {
}
}
await writeCache(cacheDir, outputSuccessCache);
await cache.write(outputSuccessCache);
},
};
}
packages/cli/src/lib/cache/SuccessCache.ts
Vendored
+89
View File
@@ -0,0 +1,89 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { resolve as resolvePath } from 'node:path';
const DEFAULT_CACHE_BASE_PATH = 'node_modules/.cache/backstage-cli';
const CACHE_MAX_AGE_MS = 7 * 24 * 3600_000;
export class SuccessCache {
readonly #path: string;
constructor(name: string, basePath?: string) {
this.#path = resolvePath(basePath ?? DEFAULT_CACHE_BASE_PATH, name);
}
async read(): Promise<Set<string>> {
const state = await fs.stat(this.#path).catch(error => {
if (error.code === 'ENOENT') {
return undefined;
}
throw error;
});
if (!state || !state.isDirectory()) {
return new Set();
}
const items = await fs.readdir(this.#path);
const returned = new Set<string>();
const removed = new Set<string>();
const now = Date.now();
for (const item of items) {
const split = item.split('_');
if (split.length !== 2) {
removed.add(item);
continue;
}
const createdAt = parseInt(split[0], 10);
if (Number.isNaN(createdAt) || now - createdAt > CACHE_MAX_AGE_MS) {
removed.add(item);
} else {
returned.add(split[1]);
}
}
for (const item of removed) {
await fs.unlink(resolvePath(this.#path, item));
}
return returned;
}
async write(newEntries: Iterable<string>): Promise<void> {
const now = Date.now();
await fs.ensureDir(this.#path);
const existingItems = await fs.readdir(this.#path);
const empty = Buffer.alloc(0);
for (const key of newEntries) {
// Remove any existing items with the key we're about to add
const trimmedItems = existingItems.filter(item =>
item.endsWith(`_${key}`),
);
for (const trimmedItem of trimmedItems) {
await fs.unlink(resolvePath(this.#path, trimmedItem));
}
await fs.writeFile(resolvePath(this.#path, `${now}_${key}`), empty);
}
}
}