From 04297a0c119e259f96a7e6c95773e44678f23840 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 17 Oct 2024 09:55:20 +0200 Subject: [PATCH 1/3] cli: refactor success cache to be additive Signed-off-by: Patrik Oldsberg --- .changeset/loud-trainers-yawn.md | 5 ++ packages/cli/src/commands/repo/lint.ts | 40 ++-------- packages/cli/src/commands/repo/test.ts | 39 ++-------- packages/cli/src/lib/cache/SuccessCache.ts | 89 ++++++++++++++++++++++ 4 files changed, 107 insertions(+), 66 deletions(-) create mode 100644 .changeset/loud-trainers-yawn.md create mode 100644 packages/cli/src/lib/cache/SuccessCache.ts diff --git a/.changeset/loud-trainers-yawn.md b/.changeset/loud-trainers-yawn.md new file mode 100644 index 0000000000..b48f773caa --- /dev/null +++ b/.changeset/loud-trainers-yawn.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The `--successCache` option for the `repo test` and `repo lint` commands now use an additive store that keeps old entries around for a week before they are cleaned up automatically. diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index 1895597e96..9b1ce4ee3e 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -16,9 +16,8 @@ import chalk from 'chalk'; import { Command, OptionValues } from 'commander'; -import fs from 'fs-extra'; import { createHash } from 'crypto'; -import { relative as relativePath, resolve as resolvePath } from 'path'; +import { relative as relativePath } from 'path'; import { PackageGraph, BackstagePackageJson, @@ -27,6 +26,7 @@ import { import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; import { createScriptOptionsParser } from './optionsParser'; +import { SuccessCache } from '../../lib/cache/SuccessCache'; function depCount(pkg: BackstagePackageJson) { const deps = pkg.dependencies ? Object.keys(pkg.dependencies).length : 0; @@ -36,39 +36,13 @@ function depCount(pkg: BackstagePackageJson) { return deps + devDeps; } -const CACHE_FILE_NAME = 'lint-cache.json'; - -type Cache = string[]; - -async function readCache(dir: string): Promise { - try { - const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); - if (!Array.isArray(data)) { - return undefined; - } - if (data.some(x => typeof x !== 'string')) { - return undefined; - } - return data as Cache; - } catch { - return undefined; - } -} - -async function writeCache(dir: string, cache: Cache) { - await fs.mkdirp(dir); - await fs.writeJson(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); -} - export async function command(opts: OptionValues, cmd: Command): Promise { let packages = await PackageGraph.listTargetPackages(); - const cacheDir = resolvePath( - opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', - ); + const cache = new SuccessCache('lint', opts.successCacheDir); const cacheContext = opts.successCache ? { - cache: await readCache(cacheDir), + entries: await cache.read(), lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')), } : undefined; @@ -136,7 +110,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix: Boolean(opts.fix), format: opts.format as string | undefined, shouldCache: Boolean(cacheContext), - successCache: cacheContext?.cache, + successCache: cacheContext?.entries, rootDir: paths.targetRoot, }, workerFactory: async ({ @@ -202,7 +176,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { hash.update('\0'); } sha = await hash.digest('hex'); - if (successCache?.includes(sha)) { + if (successCache?.has(sha)) { console.log(`Skipped ${relativeDir} due to cache hit`); return { relativeDir, sha, failed: false }; } @@ -262,7 +236,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } if (cacheContext) { - await writeCache(cacheDir, outputSuccessCache); + await cache.write(outputSuccessCache); } if (failed) { diff --git a/packages/cli/src/commands/repo/test.ts b/packages/cli/src/commands/repo/test.ts index 1d59ba3d63..1035354eb2 100644 --- a/packages/cli/src/commands/repo/test.ts +++ b/packages/cli/src/commands/repo/test.ts @@ -16,14 +16,14 @@ import os from 'os'; import crypto from 'node:crypto'; -import fs from 'fs-extra'; import yargs from 'yargs'; -import { resolve as resolvePath, relative as relativePath } from 'path'; +import { relative as relativePath } from 'path'; import { Command, OptionValues } from 'commander'; import { Lockfile, PackageGraph } from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runCheck, runPlain } from '../../lib/run'; import { isChildPath } from '@backstage/cli-common'; +import { SuccessCache } from '../../lib/cache/SuccessCache'; type JestProject = { displayName: string; @@ -50,30 +50,6 @@ interface GlobalWithCache extends Global { }; } -const CACHE_FILE_NAME = 'test-cache.json'; - -type Cache = string[]; - -async function readCache(dir: string): Promise { - try { - const data = await fs.readJson(resolvePath(dir, CACHE_FILE_NAME)); - if (!Array.isArray(data)) { - return undefined; - } - if (data.some(x => typeof x !== 'string')) { - return undefined; - } - return data as Cache; - } catch { - return undefined; - } -} - -function writeCache(dir: string, cache: Cache) { - fs.mkdirpSync(dir); - fs.writeJsonSync(resolvePath(dir, CACHE_FILE_NAME), cache, { spaces: 2 }); -} - /** * Use git to get the HEAD tree hashes of each package in the project. */ @@ -272,10 +248,6 @@ export async function command(opts: OptionValues, cmd: Command): Promise { removeOptionArg(args, '--successCache', 1); removeOptionArg(args, '--successCacheDir'); - const cacheDir = resolvePath( - opts.successCacheDir ?? 'node_modules/.cache/backstage-cli', - ); - // Parse the args to ensure that no file filters are provided, in which case we refuse to run const { _: parsedArgs } = await yargs(args).options(jestCli.yargsOptions) .argv; @@ -293,6 +265,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { ); } + const cache = new SuccessCache('test', opts.successCacheDir); const graph = await getPackageGraph(); // Shared state for the bridge @@ -305,7 +278,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { globalWithCache.__backstageCli_jestSuccessCache = { // This is called by `config/jest.js` after the project configs have been gathered async filterConfigs(projectConfigs, globalRootConfig) { - const cache = await readCache(cacheDir); + const cacheEntries = await cache.read(); const lockfile = await Lockfile.load( paths.resolveTargetRoot('yarn.lock'), ); @@ -350,7 +323,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { projectHashes.set(packageName, sha); - if (cache?.includes(sha)) { + if (cacheEntries.has(sha)) { if (!selectedProjects || selectedProjects.includes(packageName)) { console.log(`Skipped ${packageName} due to cache hit`); } @@ -391,7 +364,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { } } - await writeCache(cacheDir, outputSuccessCache); + await cache.write(outputSuccessCache); }, }; } diff --git a/packages/cli/src/lib/cache/SuccessCache.ts b/packages/cli/src/lib/cache/SuccessCache.ts new file mode 100644 index 0000000000..29f4301eea --- /dev/null +++ b/packages/cli/src/lib/cache/SuccessCache.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'node:path'; + +const DEFAULT_CACHE_BASE_PATH = 'node_modules/.cache/backstage-cli'; + +const CACHE_MAX_AGE_MS = 7 * 24 * 3600_000; + +export class SuccessCache { + readonly #path: string; + + constructor(name: string, basePath?: string) { + this.#path = resolvePath(basePath ?? DEFAULT_CACHE_BASE_PATH, name); + } + + async read(): Promise> { + const state = await fs.stat(this.#path).catch(error => { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + }); + if (!state || !state.isDirectory()) { + return new Set(); + } + const items = await fs.readdir(this.#path); + + const returned = new Set(); + const removed = new Set(); + + const now = Date.now(); + + for (const item of items) { + const split = item.split('_'); + if (split.length !== 2) { + removed.add(item); + continue; + } + const createdAt = parseInt(split[0], 10); + if (Number.isNaN(createdAt) || now - createdAt > CACHE_MAX_AGE_MS) { + removed.add(item); + } else { + returned.add(split[1]); + } + } + + for (const item of removed) { + await fs.unlink(resolvePath(this.#path, item)); + } + + return returned; + } + + async write(newEntries: Iterable): Promise { + const now = Date.now(); + + await fs.ensureDir(this.#path); + + const existingItems = await fs.readdir(this.#path); + + const empty = Buffer.alloc(0); + for (const key of newEntries) { + // Remove any existing items with the key we're about to add + const trimmedItems = existingItems.filter(item => + item.endsWith(`_${key}`), + ); + for (const trimmedItem of trimmedItems) { + await fs.unlink(resolvePath(this.#path, trimmedItem)); + } + + await fs.writeFile(resolvePath(this.#path, `${now}_${key}`), empty); + } + } +} From e0aafa51ce9c452333a5277f34260fa5c1e8f5bc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 17 Oct 2024 20:53:03 +0200 Subject: [PATCH 2/3] cli: gracefully handle file in place of success cache dir Signed-off-by: Patrik Oldsberg --- packages/cli/src/lib/cache/SuccessCache.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/lib/cache/SuccessCache.ts b/packages/cli/src/lib/cache/SuccessCache.ts index 29f4301eea..5370958537 100644 --- a/packages/cli/src/lib/cache/SuccessCache.ts +++ b/packages/cli/src/lib/cache/SuccessCache.ts @@ -29,15 +29,19 @@ export class SuccessCache { } async read(): Promise> { - const state = await fs.stat(this.#path).catch(error => { + try { + const stat = await fs.stat(this.#path); + if (!stat.isDirectory()) { + await fs.rm(this.#path); + return new Set(); + } + } catch (error) { if (error.code === 'ENOENT') { - return undefined; + return new Set(); } throw error; - }); - if (!state || !state.isDirectory()) { - return new Set(); } + const items = await fs.readdir(this.#path); const returned = new Set(); From 1ff8ca30282a69e5bf29af64a2fbbeec84e86422 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 18 Oct 2024 15:02:22 +0200 Subject: [PATCH 3/3] catalog-client: fix case sensitive filters in mock api Signed-off-by: Patrik Oldsberg --- .changeset/two-donuts-float.md | 5 ++++ .../testUtils/InMemoryCatalogClient.test.ts | 24 +++++++++++++++++++ .../src/testUtils/InMemoryCatalogClient.ts | 19 +++++++++++---- 3 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 .changeset/two-donuts-float.md diff --git a/.changeset/two-donuts-float.md b/.changeset/two-donuts-float.md new file mode 100644 index 0000000000..d9df082910 --- /dev/null +++ b/.changeset/two-donuts-float.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +Fix for certain filter fields in the `catalogApiMock` being case sensitive. diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts index 3f3eaed477..b77fcd0676 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { CATALOG_FILTER_EXISTS } from '../types'; import { InMemoryCatalogClient } from './InMemoryCatalogClient'; import { Entity } from '@backstage/catalog-model'; @@ -25,6 +26,7 @@ const entity1: Entity = { name: 'e1', uid: 'u1', }, + relations: [{ type: 'relatedTo', targetRef: 'customkind:default/e2' }], }; const entity2: Entity = { @@ -42,10 +44,32 @@ const entities = [entity1, entity2]; describe('InMemoryCatalogClient', () => { it('getEntities', async () => { const client = new InMemoryCatalogClient({ entities }); + await expect(client.getEntities()).resolves.toEqual({ items: entities }); + + await expect( + client.getEntities({ filter: { 'metadata.name': 'E1' } }), + ).resolves.toEqual({ items: [entity1] }); + await expect( client.getEntities({ filter: { 'metadata.uid': 'u2' } }), ).resolves.toEqual({ items: [entity2] }); + + await expect( + client.getEntities({ filter: { 'metadata.uid': 'U2' } }), + ).resolves.toEqual({ items: [entity2] }); + + await expect( + client.getEntities({ + filter: { 'relations.relatedto': CATALOG_FILTER_EXISTS }, + }), + ).resolves.toEqual({ items: [entity1] }); + + await expect( + client.getEntities({ + filter: { 'relations.relatedTo': 'customkind:default/e2' }, + }), + ).resolves.toEqual({ items: [entity1] }); }); it('getEntitiesByRefs', async () => { diff --git a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts index 355a10bf0e..13ce190e88 100644 --- a/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts +++ b/packages/catalog-client/src/testUtils/InMemoryCatalogClient.ts @@ -48,13 +48,22 @@ function buildEntitySearch(entity: Entity) { const rows = traverse(entity); if (entity.metadata?.name) { - rows.push({ key: 'metadata.name', value: entity.metadata.name }); + rows.push({ + key: 'metadata.name', + value: entity.metadata.name.toLocaleLowerCase('en-US'), + }); } if (entity.metadata?.namespace) { - rows.push({ key: 'metadata.namespace', value: entity.metadata.namespace }); + rows.push({ + key: 'metadata.namespace', + value: entity.metadata.namespace.toLocaleLowerCase('en-US'), + }); } if (entity.metadata?.uid) { - rows.push({ key: 'metadata.uid', value: entity.metadata.uid }); + rows.push({ + key: 'metadata.uid', + value: entity.metadata.uid.toLocaleLowerCase('en-US'), + }); } if (!entity.metadata.namespace) { @@ -64,8 +73,8 @@ function buildEntitySearch(entity: Entity) { // Visit relations for (const relation of entity.relations ?? []) { rows.push({ - key: `relations.${relation.type}`, - value: relation.targetRef, + key: `relations.${relation.type.toLocaleLowerCase('en-US')}`, + value: relation.targetRef.toLocaleLowerCase('en-US'), }); }