cli: add Lockfile utility

2020-11-22 14:20:13 +01:00
parent 58e964394f
commit 0145faa6ee
5 changed files with 422 additions and 0 deletions
@@ -51,6 +51,7 @@
    "@types/webpack-node-externals": "^2.5.0",
    "@typescript-eslint/eslint-plugin": "^v3.10.1",
    "@typescript-eslint/parser": "^v3.10.1",
+    "@yarnpkg/lockfile": "^1.1.0",
    "bfj": "^7.0.2",
    "chalk": "^4.0.0",
    "chokidar": "^3.3.1",
@@ -92,6 +93,7 @@
    "rollup-plugin-postcss": "^3.1.1",
    "rollup-plugin-typescript2": "^0.27.3",
    "rollup-pluginutils": "^2.8.2",
+    "semver": "^7.3.2",
    "start-server-webpack-plugin": "^2.2.5",
    "style-loader": "^1.2.1",
    "sucrase": "^3.16.0",
@@ -131,6 +133,7 @@
    "@types/tar": "^4.0.3",
    "@types/webpack": "^4.41.7",
    "@types/webpack-dev-server": "^3.11.0",
+    "@types/yarnpkg__lockfile": "^1.1.4",
    "del": "^5.1.0",
    "mock-fs": "^4.13.0",
    "nodemon": "^2.0.2",
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import mockFs from 'mock-fs';
+import { Lockfile } from './Lockfile';
+
+const HEADER = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+`;
+
+const mockA = `${HEADER}
+a@^1:
+  version "1.0.1"
+  resolved "https://my-registry/a-1.0.01.tgz#abc123"
+  integrity sha512-xyz
+  dependencies:
+    b "^2"
+
+b@2.0.x:
+  version "2.0.1"
+
+b@^2:
+  version "2.0.0"
+`;
+
+const mockADedup = `${HEADER}
+a@^1:
+  version "1.0.1"
+  resolved "https://my-registry/a-1.0.01.tgz#abc123"
+  integrity sha512-xyz
+  dependencies:
+    b "^2"
+
+b@2.0.x, b@^2:
+  version "2.0.1"
+`;
+
+const mockB = `${HEADER}
+"@s/a@*", "@s/a@1 || 2", "@s/a@^1":
+  version "1.0.1"
+
+"@s/a@^2.0.x":
+  version "2.0.0"
+`;
+
+const mockBDedup = `${HEADER}
+"@s/a@*", "@s/a@1 || 2", "@s/a@^2.0.x":
+  version "2.0.0"
+
+"@s/a@^1":
+  version "1.0.1"
+`;
+
+describe('Lockfile', () => {
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should load and serialize mockA', async () => {
+    mockFs({
+      '/yarn.lock': mockA,
+    });
+
+    const lockfile = await Lockfile.load('/yarn.lock');
+    expect(lockfile.get('a')).toEqual([{ range: '^1', version: '1.0.1' }]);
+    expect(lockfile.get('b')).toEqual([
+      { range: '2.0.x', version: '2.0.1' },
+      { range: '^2', version: '2.0.0' },
+    ]);
+    expect(lockfile.toString()).toBe(mockA);
+  });
+
+  it('should deduplicate mockA', async () => {
+    mockFs({
+      '/yarn.lock': mockA,
+    });
+
+    const lockfile = await Lockfile.load('/yarn.lock');
+    const result = lockfile.analyze();
+    expect(result).toEqual({
+      invalidRanges: [],
+      newRanges: [],
+      newVersions: [
+        {
+          name: 'b',
+          range: '^2',
+          oldVersion: '2.0.0',
+          newVersion: '2.0.1',
+        },
+      ],
+    });
+
+    expect(lockfile.toString()).toBe(mockA);
+    lockfile.replaceVersions(result.newVersions);
+    expect(lockfile.toString()).toBe(mockADedup);
+  });
+
+  it('should deduplicate mockB', async () => {
+    mockFs({
+      '/yarn.lock': mockB,
+    });
+
+    const lockfile = await Lockfile.load('/yarn.lock');
+    const result = lockfile.analyze();
+    expect(result).toEqual({
+      invalidRanges: [],
+      newRanges: [
+        {
+          name: '@s/a',
+          oldRange: '^1',
+          newRange: '*',
+          oldVersion: '1.0.1',
+          newVersion: '2.0.0',
+        },
+      ],
+      newVersions: [
+        {
+          name: '@s/a',
+          range: '*',
+          oldVersion: '1.0.1',
+          newVersion: '2.0.0',
+        },
+        {
+          name: '@s/a',
+          range: '1 || 2',
+          oldVersion: '1.0.1',
+          newVersion: '2.0.0',
+        },
+      ],
+    });
+
+    expect(lockfile.toString()).toBe(mockB);
+    lockfile.replaceVersions(result.newVersions);
+    expect(lockfile.toString()).toBe(mockBDedup);
+  });
+});
@@ -0,0 +1,242 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import semver from 'semver';
+import {
+  parse as parseLockfile,
+  stringify as stringifyLockfile,
+} from '@yarnpkg/lockfile';
+
+const ENTRY_PATTERN = /^((?:@[^/]+\/)?[^@/]+)@(.+)$/;
+
+type LockfileData = {
+  [entry: string]: {
+    version: string;
+    resolved?: string;
+    integrity?: string;
+    dependencies?: { [name: string]: string };
+  };
+};
+
+type LockfileQueryEntry = {
+  range: string;
+  version: string;
+};
+
+/** Entries that have an invalid version range, for example an NPM tag */
+type AnalyzeResultInvalidRange = {
+  name: string;
+  range: string;
+};
+
+/** Entries that can be deduplicated by bumping to an existing higher version */
+type AnalyzeResultNewVersion = {
+  name: string;
+  range: string;
+  oldVersion: string;
+  newVersion: string;
+};
+
+/** Entries that {would need a dependency update in package.json to be deduplicated */
+type AnalyzeResultNewRange = {
+  name: string;
+  oldRange: string;
+  newRange: string;
+  oldVersion: string;
+  newVersion: string;
+};
+
+type AnalyzeResult = {
+  invalidRanges: AnalyzeResultInvalidRange[];
+  newVersions: AnalyzeResultNewVersion[];
+  newRanges: AnalyzeResultNewRange[];
+};
+
+export class Lockfile {
+  static async load(path: string) {
+    const lockfileContents = await fs.readFile(path, 'utf8');
+    const lockfile = parseLockfile(lockfileContents);
+    if (lockfile.type !== 'success') {
+      throw new Error(`Failed yarn.lock parse with ${lockfile.type}`);
+    }
+
+    const data = lockfile.object as LockfileData;
+    const packages = new Map<string, LockfileQueryEntry[]>();
+
+    for (const [key, value] of Object.entries(data)) {
+      const [, name, range] = ENTRY_PATTERN.exec(key) ?? [];
+      if (!name) {
+        throw new Error(`Failed to parse yarn.lock entry '${key}'`);
+      }
+
+      let queries = packages.get(name);
+      if (!queries) {
+        queries = [];
+        packages.set(name, queries);
+      }
+      queries.push({ range, version: value.version });
+    }
+
+    return new Lockfile(packages, data);
+  }
+
+  constructor(
+    private readonly packages: Map<string, LockfileQueryEntry[]>,
+    private readonly data: LockfileData,
+  ) {}
+
+  get(name: string): LockfileQueryEntry[] | undefined {
+    return this.packages.get(name);
+  }
+
+  /** Analyzes the lockfile to identify possible actions and warnings for the entries */
+  analyze(options?: { filter?: (name: string) => boolean }): AnalyzeResult {
+    const { filter } = options ?? {};
+    const result: AnalyzeResult = {
+      invalidRanges: [],
+      newVersions: [],
+      newRanges: [],
+    };
+
+    for (const [name, allEntries] of this.packages) {
+      if (filter && !filter(name)) {
+        continue;
+      }
+
+      // Get rid of an signal any invalid ranges upfront
+      const invalid = allEntries.filter(e => !semver.validRange(e.range));
+      result.invalidRanges.push(
+        ...invalid.map(({ range }) => ({ name, range })),
+      );
+
+      // Grab all valid entries, if there isn't at least 2 different valid ones we're done
+      const entries = allEntries.filter(e => semver.validRange(e.range));
+      if (entries.length < 2) {
+        continue;
+      }
+
+      // Find all versions currently in use
+      const versions = Array.from(
+        new Set(entries.map(e => e.version)),
+      ).sort((v1, v2) => semver.rcompare(v1, v2));
+
+      // If we're not using at least 2 different versions we're done
+      if (versions.length < 2) {
+        continue;
+      }
+
+      const acceptedVersions = new Set<string>();
+      for (const { version, range } of entries) {
+        // Finds the highest matching version from the the known versions
+        // TODO(Rugvip): We may want to select the version that satisfies the most ranges rather than the highest one
+        const acceptedVersion = versions.find(v => semver.satisfies(v, range));
+        if (!acceptedVersion) {
+          throw new Error(
+            `No existing version was accepted for range ${range}, searching through ${versions}`,
+          );
+        }
+
+        if (acceptedVersion !== version) {
+          result.newVersions.push({
+            name,
+            range,
+            newVersion: acceptedVersion,
+            oldVersion: version,
+          });
+        }
+
+        acceptedVersions.add(acceptedVersion);
+      }
+
+      // If all ranges where able to accept the same version, we're done
+      if (acceptedVersions.size === 1) {
+        continue;
+      }
+
+      // Find the max version and range that we may want bump older packages to
+      const maxVersion = Array.from(acceptedVersions).sort(semver.rcompare)[0];
+      const maxEntry = entries.find(e => semver.satisfies(maxVersion, e.range));
+      if (!maxEntry) {
+        throw new Error(
+          `No entry found that satisfies max version '${maxVersion}'`,
+        );
+      }
+
+      // Find all entries that don't satisfy the max version
+      for (const { version, range } of entries) {
+        if (semver.satisfies(maxVersion, range)) {
+          continue;
+        }
+
+        result.newRanges.push({
+          name,
+          oldRange: range,
+          newRange: maxEntry.range,
+          oldVersion: version,
+          newVersion: maxVersion,
+        });
+      }
+    }
+
+    return result;
+  }
+
+  /** Modifies the lockfile by bumping packages to the suggested versions */
+  replaceVersions(results: AnalyzeResultNewVersion[]) {
+    for (const { name, range, oldVersion, newVersion } of results) {
+      const query = `${name}@${range}`;
+
+      // Update the backing data
+      const entryData = this.data[query];
+      if (!entryData) {
+        throw new Error(`No entry data for ${query}`);
+      }
+      if (entryData.version !== oldVersion) {
+        throw new Error(
+          `Expected existing version data for ${query} to be ${oldVersion}, was ${entryData.version}`,
+        );
+      }
+
+      // Modifying the data in the entry is not enough, we need to reference an existing version object
+      const matchingEntry = Object.entries(this.data).find(
+        ([q, e]) => q.startsWith(`${name}@`) && e.version === newVersion,
+      );
+      if (!matchingEntry) {
+        throw new Error(
+          `No matching entry found for ${name} at version ${newVersion}`,
+        );
+      }
+      this.data[query] = matchingEntry[1];
+
+      // Update out internal data structure
+      const entry = this.packages.get(name)?.find(e => e.range === range);
+      if (!entry) {
+        throw new Error(`No entry data for ${query}`);
+      }
+      if (entry.version !== oldVersion) {
+        throw new Error(
+          `Expected existing version data for ${query} to be ${oldVersion}, was ${entryData.version}`,
+        );
+      }
+      entry.version = newVersion;
+    }
+  }
+
+  toString() {
+    return stringifyLockfile(this.data);
+  }
+}
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { Lockfile } from './Lockfile';
@@ -6088,6 +6088,11 @@
  dependencies:
    "@types/yargs-parser" "*"

+"@types/yarnpkg__lockfile@^1.1.4":
+  version "1.1.4"
+  resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.4.tgz#445251eb00bd9c1e751f82c7c6bf4f714edfd464"
+  integrity sha512-/emrKCfQMQmFCqRqqBJ0JueHBT06jBRM3e8OgnvDUcvuExONujIk2hFA5dNsN9Nt41ljGVDdChvCydATZ+KOZw==
+
 "@types/yup@^0.29.8":
  version "0.29.8"
  resolved "https://registry.npmjs.org/@types/yup/-/yup-0.29.8.tgz#83db15735987db9fe5a38772a0fb9500e3c5bf39"
@@ -6384,6 +6389,11 @@
  resolved "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
  integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==

+"@yarnpkg/lockfile@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31"
+  integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==
+
 "@zkochan/cmd-shim@^3.1.0":
  version "3.1.0"
  resolved "https://registry.npmjs.org/@zkochan/cmd-shim/-/cmd-shim-3.1.0.tgz#2ab8ed81f5bb5452a85f25758eb9b8681982fd2e"