cli/commands/build-cache: store external cache in tar archives

packages/cli/package.json

@@ -32,10 +32,12 @@
     "@types/ora": "^3.2.0",
     "@types/react-dev-utils": "^9.0.4",
     "@types/recursive-readdir": "^2.2.0",
+    "@types/tar": "^4.0.3",
     "@types/webpack": "^4.41.7",
     "@types/webpack-dev-server": "^3.10.0",
     "del": "^5.1.0",
     "nodemon": "^2.0.2",
+    "tar": "^6.0.1",
     "ts-node": "^8.6.2"
   },
   "bin": {

packages/cli/src/commands/build-cache/index.ts

@@ -15,14 +15,20 @@
  */
 
 import fs from 'fs-extra';
-import { resolve as resolvePath, relative as relativePath } from 'path';
+import {
+  dirname,
+  resolve as resolvePath,
+  relative as relativePath,
+} from 'path';
 import { promisify } from 'util';
 import { exec as execCb } from 'child_process';
 import { Command } from 'commander';
+import tar from 'tar';
 import { ExitCodeError } from '../../helpers/errors';
 const exec = promisify(execCb);
 
 const INFO_FILE = '.backstage-build-cache';
+const CACHE_ARCHIVE = 'cache.tgz';
 
 type Options = {
   inputs: string[];
@@ -62,25 +68,29 @@ export default async (cmd: Command, args: string[]) => {
   console.log('DEBUG: trees =', trees);
 
   const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(',');
+  console.log('DEBUG: cacheHit =', cacheHit);
   if (!cacheHit) {
     await build(options);
 
-    await fs.writeFile(
-      resolvePath(options.output, INFO_FILE),
-      JSON.stringify({ trees }, null, 2),
-      'utf-8',
-    );
     if (cache.writable) {
+      await fs.writeFile(
+        resolvePath(options.output, INFO_FILE),
+        JSON.stringify({ trees }, null, 2),
+        'utf8',
+      );
       console.log(`DEBUG: write cache`);
-      console.log('DEBUG: cache.location =', cache.location);
-      await fs.remove(cache.location);
-      console.log('DEBUG: options.output =', options.output);
-      await fs.copy(options.output, cache.location);
+      console.log('DEBUG: cache.archivePath =', cache.archivePath);
+      await fs.remove(cache.archivePath);
+      await fs.ensureDir(dirname(cache.archivePath));
+      await tar.create(
+        { gzip: true, file: cache.archivePath, cwd: options.output },
+        ['.'],
+      );
     }
   } else if (cache.needsCopy) {
-    console.log(`DEBUG: would copy cache`);
     await fs.remove(options.output);
-    await fs.copy(cache.location, options.output);
+    await fs.ensureDir(options.output);
+    await tar.extract({ file: cache.archivePath, cwd: options.output });
   }
 
   const ls = await run('ls derp');
@@ -105,7 +115,7 @@ async function run(cmd: string) {
 
 type Cache = {
   // External location of the cache outside the output folder
-  location: string;
+  archivePath: string;
   readable?: boolean;
   writable?: boolean;
   needsCopy?: boolean;
@@ -115,12 +125,13 @@ type Cache = {
 async function readCache(options: Options): Promise<Cache> {
   const repoPath = relativePath(options.repoRoot, process.cwd());
   const location = resolvePath(options.cacheDir, repoPath);
+  const archivePath = resolvePath(location, CACHE_ARCHIVE);
 
   // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing
   try {
-    await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`);
+    // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`);
   } catch (error) {
-    return { location };
+    return { archivePath };
   }
 
   try {
@@ -131,7 +142,7 @@ async function readCache(options: Options): Promise<Cache> {
       const trees = await readInfoFile(options.output);
       if (trees) {
         return {
-          location,
+          archivePath,
           trees,
           readable: true,
           writable: true,
@@ -141,10 +152,10 @@ async function readCache(options: Options): Promise<Cache> {
 
     const externalCacheExists = await fs.pathExists(location);
     if (externalCacheExists) {
-      const trees = await readInfoFile(location);
+      const trees = await readInfoFileFromArchive(archivePath);
       if (trees) {
         return {
-          location,
+          archivePath,
           trees,
           readable: true,
           writable: true,
@@ -155,15 +166,50 @@ async function readCache(options: Options): Promise<Cache> {
   } catch (error) {
     console.log(`Cache not found, ${error}`);
   }
-  return { location, writable: true };
+  return { archivePath, writable: true };
 }
 
 async function readInfoFile(dir: string): Promise<string[] | undefined> {
-  const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf-8');
+  const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf8');
   const { trees } = JSON.parse(infoContents);
   return trees;
 }
 
+async function readInfoFileFromArchive(
+  archivePath: string,
+): Promise<string[] | undefined> {
+  const reader = fs.createReadStream(archivePath);
+  const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })();
+
+  const infoEntry = await new Promise<tar.ReadEntry>((resolve, reject) => {
+    parser.on('entry', entry => {
+      if (entry.path === `./${INFO_FILE}`) {
+        resolve(entry);
+        reader.close();
+      } else {
+        entry.resume();
+      }
+    });
+    parser.on('end', () => {
+      reject(new Error('cache archive did not contain build info'));
+    });
+    parser.on('error', error => reject(error));
+
+    reader.pipe(parser);
+  });
+
+  const infoData = await new Promise<Buffer>((resolve, reject) => {
+    const chunks = new Array<Buffer>();
+    infoEntry.on('data', chunk => chunks.push(chunk));
+    infoEntry.on('end', () => resolve(Buffer.concat(chunks)));
+    infoEntry.on('error', error => reject(error));
+  });
+
+  const info = JSON.parse(infoData.toString('utf8'));
+
+  return info.trees;
+}
+
 async function getInputHashes(options: Options): Promise<string[]> {
   const trees = [];
   for (const input of options.inputs) {

yarn.lock

@@ -3612,6 +3612,13 @@
   resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
   integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
 
+"@types/minipass@*":
+  version "2.2.0"
+  resolved "https://registry.npmjs.org/@types/minipass/-/minipass-2.2.0.tgz#51ad404e8eb1fa961f75ec61205796807b6f9651"
+  integrity sha512-wuzZksN4w4kyfoOv/dlpov4NOunwutLA/q7uc00xU02ZyUY+aoM5PWIXEKBMnm0NHd4a+N71BMjq+x7+2Af1fg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2":
   version "13.9.2"
   resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"
@@ -3776,6 +3783,14 @@
   resolved "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.5.tgz#9adbc12950582aa65ead76bffdf39fe0c27a3c02"
   integrity sha512-/gG2M/Imw7cQFp8PGvz/SwocNrmKFjFsm5Pb8HdbHkZ1K8pmuPzOX4VeVoiEecFCVf4CsN1r3/BRvx+6sNqwtQ==
 
+"@types/tar@^4.0.3":
+  version "4.0.3"
+  resolved "https://registry.npmjs.org/@types/tar/-/tar-4.0.3.tgz#e2cce0b8ff4f285293243f5971bd7199176ac489"
+  integrity sha512-Z7AVMMlkI8NTWF0qGhC4QIX0zkV/+y0J8x7b/RsHrN0310+YNjoJd8UrApCiGBCWtKjxS9QhNqLi2UJNToh5hA==
+  dependencies:
+    "@types/minipass" "*"
+    "@types/node" "*"
+
 "@types/testing-library__cypress@^5.0.3":
   version "5.0.3"
   resolved "https://registry.npmjs.org/@types/testing-library__cypress/-/testing-library__cypress-5.0.3.tgz#94969b7c1eea96e09d8e023a1d225590fa75a1fe"
@@ -5795,7 +5810,7 @@ chokidar@^3.2.2, chokidar@^3.3.0, chokidar@^3.3.1:
   optionalDependencies:
     fsevents "~2.1.2"
 
-chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.4:
+chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.3, chownr@^1.1.4:
   version "1.1.4"
   resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
   integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
@@ -13039,6 +13054,14 @@ minizlib@^1.2.1:
   dependencies:
     minipass "^2.9.0"
 
+minizlib@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz#fd52c645301ef09a63a2c209697c294c6ce02cf3"
+  integrity sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA==
+  dependencies:
+    minipass "^3.0.0"
+    yallist "^4.0.0"
+
 mississippi@^3.0.0:
   version "3.0.0"
   resolved "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022"
@@ -13078,7 +13101,7 @@ mkdirp-promise@^5.0.1:
   dependencies:
     mkdirp "*"
 
-mkdirp@*:
+mkdirp@*, mkdirp@^1.0.3:
   version "1.0.3"
   resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.3.tgz#4cf2e30ad45959dddea53ad97d518b6c8205e1ea"
   integrity sha512-6uCP4Qc0sWsgMLy1EOqqS/3rjDHOEnsStVr/4vtAIK2Y5i2kA7lFFejYrpIyiN9w0pYf4ckeCYT9f1r1P9KX5g==
@@ -18083,6 +18106,18 @@ tar@^4.4.10, tar@^4.4.12, tar@^4.4.13, tar@^4.4.8:
     safe-buffer "^5.1.2"
     yallist "^3.0.3"
 
+tar@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.npmjs.org/tar/-/tar-6.0.1.tgz#7b3bd6c313cb6e0153770108f8d70ac298607efa"
+  integrity sha512-bKhKrrz2FJJj5s7wynxy/fyxpE0CmCjmOQ1KV4KkgXFWOgoIT/NbTMnB1n+LFNrNk0SSBVGGxcK5AGsyC+pW5Q==
+  dependencies:
+    chownr "^1.1.3"
+    fs-minipass "^2.0.0"
+    minipass "^3.0.0"
+    minizlib "^2.1.0"
+    mkdirp "^1.0.3"
+    yallist "^4.0.0"
+
 telejson@^3.2.0:
   version "3.3.0"
   resolved "https://registry.npmjs.org/telejson/-/telejson-3.3.0.tgz#6d814f3c0d254d5c4770085aad063e266b56ad03"