From e1bc9cc97dad94d45b3ca8a4b64d5ecc8a593b24 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 6 Oct 2024 13:36:31 +0200 Subject: [PATCH] cli-node: refactor to move getDependencyHash to Lockfile Signed-off-by: Patrik Oldsberg --- .changeset/calm-owls-move.md | 2 +- packages/cli-node/report.api.md | 3 +- packages/cli-node/src/monorepo/Lockfile.ts | 68 ++++++++++++++++--- .../cli-node/src/monorepo/PackageGraph.ts | 51 +------------- packages/cli/src/commands/repo/lint.ts | 24 +++++-- 5 files changed, 82 insertions(+), 66 deletions(-) diff --git a/.changeset/calm-owls-move.md b/.changeset/calm-owls-move.md index a4dcaeeff8..bd7fb72508 100644 --- a/.changeset/calm-owls-move.md +++ b/.changeset/calm-owls-move.md @@ -2,4 +2,4 @@ '@backstage/cli-node': patch --- -Added new `packageGraph.getDependencyHash(name)` utility. +Added new `lockfile.getDependencyTreeHash(name)` utility. diff --git a/packages/cli-node/report.api.md b/packages/cli-node/report.api.md index 6c2294012d..eb18e2e9f8 100644 --- a/packages/cli-node/report.api.md +++ b/packages/cli-node/report.api.md @@ -92,7 +92,7 @@ export function isMonoRepo(): Promise; export class Lockfile { createSimplifiedDependencyGraph(): Map>; diff(otherLockfile: Lockfile): LockfileDiff; - getVersions(name: string): string[]; + getDependencyTreeHash(startName: string): string; static load(path: string): Promise; static parse(content: string): Lockfile; } @@ -117,7 +117,6 @@ export class PackageGraph extends Map { collectFn: (pkg: PackageGraphNode) => Iterable | undefined, ): Set; static fromPackages(packages: Package[]): PackageGraph; - getDependencyHash(name: string): Promise; listChangedPackages(options: { ref: string; analyzeLockfile?: boolean; diff --git a/packages/cli-node/src/monorepo/Lockfile.ts b/packages/cli-node/src/monorepo/Lockfile.ts index d7336cf970..7c4dc04f33 100644 --- a/packages/cli-node/src/monorepo/Lockfile.ts +++ b/packages/cli-node/src/monorepo/Lockfile.ts @@ -15,6 +15,7 @@ */ import { parseSyml } from '@yarnpkg/parsers'; +import crypto from 'node:crypto'; import fs from 'fs-extra'; const ENTRY_PATTERN = /^((?:@[^/]+\/)?[^@/]+)@(.+)$/; @@ -133,14 +134,6 @@ export class Lockfile { private readonly data: LockfileData, ) {} - /** - * Returns all versions of a package in the lockfile. - */ - getVersions(name: string): string[] { - const queries = this.packages.get(name); - return queries ? queries.map(q => q.version) : []; - } - /** * Creates a simplified dependency graph from the lockfile data, where each * key is a package, and the value is a set of all packages that it depends on @@ -223,4 +216,63 @@ export class Lockfile { return diff; } + + /** + * Generates a sha1 hex hash of the dependency graph for a package. + */ + getDependencyTreeHash(startName: string): string { + if (!this.packages.has(startName)) { + throw new Error(`Package '${startName}' not found in lockfile`); + } + + const hash = crypto.createHash('sha1'); + + const queue = [startName]; + const seen = new Set(); + + while (queue.length > 0) { + const name = queue.pop()!; + + if (seen.has(name)) { + continue; + } + seen.add(name); + + const entries = this.packages.get(name); + if (!entries) { + continue; // In case of missing optional peer dependencies + } + + hash.update(`pkg:${name}`); + hash.update('\0'); + + // TODO(Rugvip): This uses the same simplified lookup as createSimplifiedDependencyGraph() + // we could match version queries to make the resulting tree a bit smaller. + const deps = new Array(); + for (const entry of entries) { + // We're not being particular about stable ordering here. If the lockfile ordering changes, so will likely hash. + hash.update(entry.version); + + const data = this.data[entry.dataKey]; + if (!data) { + continue; + } + + const checksum = data.checksum || data.integrity; + if (checksum) { + hash.update('#'); + hash.update(checksum); + } + + hash.update(' '); + + deps.push(...Object.keys(data.dependencies ?? {})); + deps.push(...Object.keys(data.peerDependencies ?? {})); + } + + queue.push(...new Set(deps)); + } + + return hash.digest('hex'); + } } diff --git a/packages/cli-node/src/monorepo/PackageGraph.ts b/packages/cli-node/src/monorepo/PackageGraph.ts index 91c5312bc1..69ec99c2dd 100644 --- a/packages/cli-node/src/monorepo/PackageGraph.ts +++ b/packages/cli-node/src/monorepo/PackageGraph.ts @@ -15,7 +15,6 @@ */ import path from 'path'; -import crypto from 'node:crypto'; import { getPackages, Package } from '@manypkg/get-packages'; import { paths } from '../paths'; import { PackageRole } from '../roles'; @@ -284,43 +283,6 @@ export class PackageGraph extends Map { return targets; } - /** - * Generates a sha1 hex hash of the dependency graph for a package. - */ - async getDependencyHash(name: string): Promise { - const pkg = this.get(name); - if (!pkg) { - throw new Error(`Package '${name}' not found`); - } - - const lockfile = await this.#getLockfile(); - const depGraph = lockfile.createSimplifiedDependencyGraph(); - - const seen = new Set(); - const queue = [name]; - - while (queue.length > 0) { - const deps = depGraph.get(queue.pop()!); - if (deps) { - for (const dep of deps) { - if (!seen.has(dep)) { - seen.add(dep); - queue.push(dep); - } - } - } - } - - const hash = crypto.createHash('sha1'); - for (const dep of Array.from(seen).sort()) { - hash.update(dep); - hash.update('\0'); - hash.update(lockfile.getVersions(dep).join(' ')); - hash.update('\0'); - } - return hash.digest('hex'); - } - /** * Lists all packages that have changed since a given git ref. * @@ -380,7 +342,9 @@ export class PackageGraph extends Map { let thisLockfile: Lockfile; let otherLockfile: Lockfile; try { - thisLockfile = await this.#getLockfile(); + thisLockfile = await Lockfile.load( + paths.resolveTargetRoot('yarn.lock'), + ); otherLockfile = Lockfile.parse( await GitUtils.readFileAtRef('yarn.lock', options.ref), ); @@ -446,13 +410,4 @@ export class PackageGraph extends Map { return result; } - - #lockfilePromise?: Promise; - #getLockfile(): Promise { - if (this.#lockfilePromise) { - return this.#lockfilePromise; - } - this.#lockfilePromise = Lockfile.load(paths.resolveTargetRoot('yarn.lock')); - return this.#lockfilePromise; - } } diff --git a/packages/cli/src/commands/repo/lint.ts b/packages/cli/src/commands/repo/lint.ts index b5a89471a7..655e524d19 100644 --- a/packages/cli/src/commands/repo/lint.ts +++ b/packages/cli/src/commands/repo/lint.ts @@ -19,7 +19,11 @@ import { Command, OptionValues } from 'commander'; import fs from 'fs-extra'; import { createHash } from 'crypto'; import { relative as relativePath, resolve as resolvePath } from 'path'; -import { PackageGraph, BackstagePackageJson } from '@backstage/cli-node'; +import { + PackageGraph, + BackstagePackageJson, + Lockfile, +} from '@backstage/cli-node'; import { paths } from '../../lib/paths'; import { runWorkerQueueThreads } from '../../lib/parallel'; import { createScriptOptionsParser } from './optionsParser'; @@ -63,11 +67,15 @@ export async function command(opts: OptionValues, cmd: Command): Promise { opts.cache === true ? paths.resolveTargetRoot('node_modules/.cache/backstage-cli') : opts.cache; - const cache = cacheDir ? await readCache(cacheDir) : undefined; - - const graph = PackageGraph.fromPackages(packages); + const cacheContext = cacheDir + ? { + cache: await readCache(cacheDir), + lockfile: await Lockfile.load(paths.resolveTargetRoot('yarn.lock')), + } + : undefined; if (opts.since) { + const graph = PackageGraph.fromPackages(packages); packages = await graph.listChangedPackages({ ref: opts.since, analyzeLockfile: true, @@ -100,13 +108,15 @@ export async function command(opts: OptionValues, cmd: Command): Promise { parentHash: undefined, }; - if (!cacheDir) { + if (!cacheContext) { return base; } const hash = createHash('sha1'); - hash.update(await graph.getDependencyHash(pkg.packageJson.name)); + hash.update( + cacheContext.lockfile.getDependencyTreeHash(pkg.packageJson.name), + ); hash.update('\0'); hash.update(JSON.stringify(lintOptions)); hash.update('\0'); @@ -127,7 +137,7 @@ export async function command(opts: OptionValues, cmd: Command): Promise { fix: Boolean(opts.fix), format: opts.format as string | undefined, shouldCache: Boolean(cacheDir), - successCache: cache, + successCache: cacheContext?.cache, }, workerFactory: async ({ fix, format, shouldCache, successCache }) => { const { ESLint } = require('eslint') as typeof import('eslint');