cli/commands/build-cache: add support for multiple cache archives

packages/cli/src/commands/build-cache/cache.ts

@@ -17,27 +17,29 @@
 import fs from 'fs-extra';
 import { resolve as resolvePath, relative as relativePath } from 'path';
 import { runPlain } from '../../helpers/run';
-import { readFileFromArchive } from './archive';
 import { Options } from './options';
+import { extractArchive, createArchive } from './archive';
 
-export type CacheHit = {
-  // External location of the cache outside the output folder
-  archivePath: string;
-  readable?: boolean;
-  writable?: boolean;
-  needsCopy?: boolean;
-  trees?: string[];
+const MAX_ENTRIES = 10;
+const INFO_FILE = '.backstage-build-cache';
+
+export type CacheQueryResult = {
+  hit: boolean;
+  copy?: (outputDir: string) => Promise<void>;
+  archive?: (outputDir: string) => Promise<void>;
 };
 
 export type CacheKey = string[];
 
 type CacheEntry = {
   key: CacheKey;
-  archivePath: string;
+  path: string;
 };
 
-const CACHE_ARCHIVE = 'cache.tgz';
-const INFO_FILE = '.backstage-build-cache';
+type CacheInfo = {
+  key?: CacheKey;
+  entries?: CacheEntry[];
+};
 
 export class Cache {
   static async read(options: Options) {
@@ -51,31 +53,11 @@ export class Cache {
       return new Cache();
     }
 
-    let localKey: CacheKey = [];
+    const outputInfo = await readCacheInfo(options.output);
+    const localKey = outputInfo?.key;
 
-    const localInfoFile = resolvePath(options.output, INFO_FILE);
-    const outputCacheExists = await fs.pathExists(localInfoFile);
-    if (outputCacheExists) {
-      const infoData = await fs.readFile(localInfoFile);
-      const { trees } = JSON.parse(infoData.toString('utf8'));
-      localKey = trees;
-    }
-
-    try {
-      const externalCacheExists = await fs.pathExists(location);
-      if (externalCacheExists) {
-        const archivePath = resolvePath(location, CACHE_ARCHIVE);
-        const infoData = await readFileFromArchive(archivePath, INFO_FILE);
-        const { trees } = JSON.parse(infoData.toString('utf8'));
-        if (trees) {
-          return new Cache([{ archivePath, key: trees }], location, localKey);
-        }
-      }
-    } catch (error) {
-      throw new Error(`failed to read external cache archive, ${error}`);
-    }
-
-    return new Cache([], location, localKey);
+    const { entries = [] } = (await readCacheInfo(location)) ?? {};
+    return new Cache(entries, location, localKey);
   }
 
   static async readInputKey(inputPaths: string[]): Promise<CacheKey> {
@@ -94,40 +76,97 @@ export class Cache {
     private readonly localKey?: CacheKey,
   ) {}
 
-  find(key: CacheKey): CacheHit | undefined {
-    if (!this.location) {
-      return undefined;
+  query(key: CacheKey): CacheQueryResult {
+    const { location } = this;
+    if (!location) {
+      return { hit: false };
     }
-    if (this.localKey?.join(',') === key.join(',')) {
-      return {
-        needsCopy: false,
-        archivePath: resolvePath(this.location, CACHE_ARCHIVE),
-      };
+
+    const archive = async (outputDir: string) => {
+      await writeCacheInfo(outputDir, { key });
+
+      const timestamp = new Date().toISOString().replace(/-|:|\..*/g, '');
+      const rand = Math.random()
+        .toString(36)
+        .slice(2, 6);
+      const archiveName = `cache-${timestamp}-${rand}.tgz`;
+      const archivePath = resolvePath(location, archiveName);
+
+      // Read existing entries and prepend the new one
+      const { entries = [] } = (await readCacheInfo(location)) ?? {};
+
+      // Check if there's already aan entry for this key, in that case we just wanna bump it
+      const entryIndex = entries.findIndex(e => compareKeys(e.key, key));
+      if (entryIndex !== -1) {
+        const [existingEntry] = entries.splice(entryIndex, 1);
+        entries.unshift(existingEntry);
+
+        await writeCacheInfo(location, { entries });
+        return;
+      }
+
+      // Create and add new archive to entries
+      await createArchive(archivePath, outputDir);
+      entries.unshift({ key, path: archiveName });
+
+      // Remove old cache entries
+      const removedEntries = entries.splice(MAX_ENTRIES);
+      for (const entry of removedEntries) {
+        try {
+          await fs.remove(resolvePath(location, entry.path));
+        } catch (error) {
+          process.stderr.write(`failed to remove old cache entry, ${error}\n`);
+        }
+      }
+
+      await writeCacheInfo(location, { entries });
+    };
+
+    if (compareKeys(this.localKey, key)) {
+      return { hit: true, archive };
     }
-    const matchingEntry = this.entries.find(entry => entry.key === key);
+
+    const matchingEntry = this.entries.find(e => compareKeys(e.key, key));
     if (!matchingEntry) {
-      return undefined;
+      return { hit: false, archive };
     }
+
     return {
-      needsCopy: true,
-      archivePath: matchingEntry.archivePath,
+      hit: true,
+      archive,
+      copy: async (outputDir: string) => {
+        const archivePath = resolvePath(location, matchingEntry.path);
+        await extractArchive(archivePath, outputDir);
+      },
     };
   }
-
-  get shouldCacheOutput() {
-    return Boolean(this.location);
-  }
-
-  async prepareOutput(key: CacheKey, outputDir: string): Promise<string> {
-    if (!this.location) {
-      throw new Error("can't write cache output, no location set");
-    }
-    const infoData = Buffer.from(
-      JSON.stringify({ trees: key }, null, 2),
-      'utf8',
-    );
-    await fs.writeFile(resolvePath(outputDir, INFO_FILE), infoData);
-    const archivePath = resolvePath(this.location, CACHE_ARCHIVE);
-    return archivePath;
-  }
+}
+
+function compareKeys(a?: CacheKey, b?: CacheKey): boolean {
+  if (!a || !b) {
+    return false;
+  }
+  return a.join(',') === b.join(',');
+}
+
+async function readCacheInfo(
+  parentDir: string,
+): Promise<CacheInfo | undefined> {
+  const infoFile = resolvePath(parentDir, INFO_FILE);
+  const exists = await fs.pathExists(infoFile);
+  if (!exists) {
+    return undefined;
+  }
+
+  const infoData = await fs.readFile(infoFile);
+  const cacheInfo = JSON.parse(infoData.toString('utf8')) as CacheInfo;
+  return cacheInfo;
+}
+
+async function writeCacheInfo(
+  parentDir: string,
+  cacheInfo: CacheInfo,
+): Promise<void> {
+  const infoData = Buffer.from(JSON.stringify(cacheInfo, null, 2), 'utf8');
+  await fs.writeFile(resolvePath(parentDir, INFO_FILE), infoData);
 }

packages/cli/src/commands/build-cache/index.ts

@@ -16,7 +16,6 @@
 
 import { Command } from 'commander';
 import { run } from '../../helpers/run';
-import { extractArchive, createArchive } from './archive';
 import { Cache } from './cache';
 import { parseOptions } from './options';
 
@@ -34,11 +33,11 @@ export default async (cmd: Command, args: string[]) => {
   const cache = await Cache.read(options);
   const key = await Cache.readInputKey(options.inputs);
 
-  const cacheHit = cache.find(key);
-  if (cacheHit) {
-    if (cacheHit.needsCopy) {
-      print('external cache hit, copying from external cache');
-      await extractArchive(cacheHit.archivePath, options.output);
+  const cacheResult = cache.query(key);
+  if (cacheResult.hit) {
+    if (cacheResult.copy) {
+      print('external cache hit, copying archive to output folder');
+      await cacheResult.copy(options.output);
     } else {
       print('cache hit, nothing to be done');
     }
@@ -47,10 +46,9 @@ export default async (cmd: Command, args: string[]) => {
 
     await run(args[0], args.slice(1));
 
-    if (cache.shouldCacheOutput) {
+    if (cacheResult.archive) {
       print('caching build output');
-      const archivePath = await cache.prepareOutput(key, options.output);
-      await createArchive(archivePath, options.output);
+      await cacheResult.archive(options.output);
     }
   }
 };