diff --git a/packages/cli/package.json b/packages/cli/package.json index f26b13a421..ba039dcd5e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -32,10 +32,12 @@ "@types/ora": "^3.2.0", "@types/react-dev-utils": "^9.0.4", "@types/recursive-readdir": "^2.2.0", + "@types/tar": "^4.0.3", "@types/webpack": "^4.41.7", "@types/webpack-dev-server": "^3.10.0", "del": "^5.1.0", "nodemon": "^2.0.2", + "tar": "^6.0.1", "ts-node": "^8.6.2" }, "bin": { diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 2d9a33d9f2..8138db54f7 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -15,14 +15,20 @@ */ import fs from 'fs-extra'; -import { resolve as resolvePath, relative as relativePath } from 'path'; +import { + dirname, + resolve as resolvePath, + relative as relativePath, +} from 'path'; import { promisify } from 'util'; import { exec as execCb } from 'child_process'; import { Command } from 'commander'; +import tar from 'tar'; import { ExitCodeError } from '../../helpers/errors'; const exec = promisify(execCb); const INFO_FILE = '.backstage-build-cache'; +const CACHE_ARCHIVE = 'cache.tgz'; type Options = { inputs: string[]; @@ -62,25 +68,29 @@ export default async (cmd: Command, args: string[]) => { console.log('DEBUG: trees =', trees); const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); + console.log('DEBUG: cacheHit =', cacheHit); if (!cacheHit) { await build(options); - await fs.writeFile( - resolvePath(options.output, INFO_FILE), - JSON.stringify({ trees }, null, 2), - 'utf-8', - ); if (cache.writable) { + await fs.writeFile( + resolvePath(options.output, INFO_FILE), + JSON.stringify({ trees }, null, 2), + 'utf8', + ); console.log(`DEBUG: write cache`); - console.log('DEBUG: cache.location =', cache.location); - await fs.remove(cache.location); - console.log('DEBUG: options.output =', options.output); - await fs.copy(options.output, cache.location); + console.log('DEBUG: cache.archivePath =', cache.archivePath); + await fs.remove(cache.archivePath); + await fs.ensureDir(dirname(cache.archivePath)); + await tar.create( + { gzip: true, file: cache.archivePath, cwd: options.output }, + ['.'], + ); } } else if (cache.needsCopy) { - console.log(`DEBUG: would copy cache`); await fs.remove(options.output); - await fs.copy(cache.location, options.output); + await fs.ensureDir(options.output); + await tar.extract({ file: cache.archivePath, cwd: options.output }); } const ls = await run('ls derp'); @@ -105,7 +115,7 @@ async function run(cmd: string) { type Cache = { // External location of the cache outside the output folder - location: string; + archivePath: string; readable?: boolean; writable?: boolean; needsCopy?: boolean; @@ -115,12 +125,13 @@ type Cache = { async function readCache(options: Options): Promise { const repoPath = relativePath(options.repoRoot, process.cwd()); const location = resolvePath(options.cacheDir, repoPath); + const archivePath = resolvePath(location, CACHE_ARCHIVE); // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing try { - await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); + // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); } catch (error) { - return { location }; + return { archivePath }; } try { @@ -131,7 +142,7 @@ async function readCache(options: Options): Promise { const trees = await readInfoFile(options.output); if (trees) { return { - location, + archivePath, trees, readable: true, writable: true, @@ -141,10 +152,10 @@ async function readCache(options: Options): Promise { const externalCacheExists = await fs.pathExists(location); if (externalCacheExists) { - const trees = await readInfoFile(location); + const trees = await readInfoFileFromArchive(archivePath); if (trees) { return { - location, + archivePath, trees, readable: true, writable: true, @@ -155,15 +166,50 @@ async function readCache(options: Options): Promise { } catch (error) { console.log(`Cache not found, ${error}`); } - return { location, writable: true }; + return { archivePath, writable: true }; } async function readInfoFile(dir: string): Promise { - const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf-8'); + const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf8'); const { trees } = JSON.parse(infoContents); return trees; } +async function readInfoFileFromArchive( + archivePath: string, +): Promise { + const reader = fs.createReadStream(archivePath); + const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); + + const infoEntry = await new Promise((resolve, reject) => { + parser.on('entry', entry => { + if (entry.path === `./${INFO_FILE}`) { + resolve(entry); + reader.close(); + } else { + entry.resume(); + } + }); + parser.on('end', () => { + reject(new Error('cache archive did not contain build info')); + }); + parser.on('error', error => reject(error)); + + reader.pipe(parser); + }); + + const infoData = await new Promise((resolve, reject) => { + const chunks = new Array(); + infoEntry.on('data', chunk => chunks.push(chunk)); + infoEntry.on('end', () => resolve(Buffer.concat(chunks))); + infoEntry.on('error', error => reject(error)); + }); + + const info = JSON.parse(infoData.toString('utf8')); + + return info.trees; +} + async function getInputHashes(options: Options): Promise { const trees = []; for (const input of options.inputs) { diff --git a/yarn.lock b/yarn.lock index dcde1de95a..db975122b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3612,6 +3612,13 @@ resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== +"@types/minipass@*": + version "2.2.0" + resolved "https://registry.npmjs.org/@types/minipass/-/minipass-2.2.0.tgz#51ad404e8eb1fa961f75ec61205796807b6f9651" + integrity sha512-wuzZksN4w4kyfoOv/dlpov4NOunwutLA/q7uc00xU02ZyUY+aoM5PWIXEKBMnm0NHd4a+N71BMjq+x7+2Af1fg== + dependencies: + "@types/node" "*" + "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2": version "13.9.2" resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349" @@ -3776,6 +3783,14 @@ resolved "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.5.tgz#9adbc12950582aa65ead76bffdf39fe0c27a3c02" integrity sha512-/gG2M/Imw7cQFp8PGvz/SwocNrmKFjFsm5Pb8HdbHkZ1K8pmuPzOX4VeVoiEecFCVf4CsN1r3/BRvx+6sNqwtQ== +"@types/tar@^4.0.3": + version "4.0.3" + resolved "https://registry.npmjs.org/@types/tar/-/tar-4.0.3.tgz#e2cce0b8ff4f285293243f5971bd7199176ac489" + integrity sha512-Z7AVMMlkI8NTWF0qGhC4QIX0zkV/+y0J8x7b/RsHrN0310+YNjoJd8UrApCiGBCWtKjxS9QhNqLi2UJNToh5hA== + dependencies: + "@types/minipass" "*" + "@types/node" "*" + "@types/testing-library__cypress@^5.0.3": version "5.0.3" resolved "https://registry.npmjs.org/@types/testing-library__cypress/-/testing-library__cypress-5.0.3.tgz#94969b7c1eea96e09d8e023a1d225590fa75a1fe" @@ -5795,7 +5810,7 @@ chokidar@^3.2.2, chokidar@^3.3.0, chokidar@^3.3.1: optionalDependencies: fsevents "~2.1.2" -chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.4: +chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.3, chownr@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== @@ -13039,6 +13054,14 @@ minizlib@^1.2.1: dependencies: minipass "^2.9.0" +minizlib@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz#fd52c645301ef09a63a2c209697c294c6ce02cf3" + integrity sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mississippi@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022" @@ -13078,7 +13101,7 @@ mkdirp-promise@^5.0.1: dependencies: mkdirp "*" -mkdirp@*: +mkdirp@*, mkdirp@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.3.tgz#4cf2e30ad45959dddea53ad97d518b6c8205e1ea" integrity sha512-6uCP4Qc0sWsgMLy1EOqqS/3rjDHOEnsStVr/4vtAIK2Y5i2kA7lFFejYrpIyiN9w0pYf4ckeCYT9f1r1P9KX5g== @@ -18083,6 +18106,18 @@ tar@^4.4.10, tar@^4.4.12, tar@^4.4.13, tar@^4.4.8: safe-buffer "^5.1.2" yallist "^3.0.3" +tar@^6.0.1: + version "6.0.1" + resolved "https://registry.npmjs.org/tar/-/tar-6.0.1.tgz#7b3bd6c313cb6e0153770108f8d70ac298607efa" + integrity sha512-bKhKrrz2FJJj5s7wynxy/fyxpE0CmCjmOQ1KV4KkgXFWOgoIT/NbTMnB1n+LFNrNk0SSBVGGxcK5AGsyC+pW5Q== + dependencies: + chownr "^1.1.3" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.0" + mkdirp "^1.0.3" + yallist "^4.0.0" + telejson@^3.2.0: version "3.3.0" resolved "https://registry.npmjs.org/telejson/-/telejson-3.3.0.tgz#6d814f3c0d254d5c4770085aad063e266b56ad03"