From 555db720ae17e38946787da393c9e0e51a17ff68 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 26 Mar 2020 14:57:54 +0100 Subject: [PATCH 01/24] cli: added initial build-cache command --- .../cli/src/commands/build-cache/index.ts | 22 +++++++++++++++++++ packages/cli/src/index.ts | 18 +++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 packages/cli/src/commands/build-cache/index.ts diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts new file mode 100644 index 0000000000..4cf3d8ef02 --- /dev/null +++ b/packages/cli/src/commands/build-cache/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * The build-cache command is used to make builds a no-op if there are no changes to the package. + * It supports both local development where the output directory remains intact, as well as CI + * where the output directory is stored in a separate cache dir. + */ +export default async () => {}; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 5f24190237..997c05c18c 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -19,6 +19,7 @@ import chalk from 'chalk'; import fs from 'fs'; import createPluginCommand from './commands/create-plugin/createPlugin'; import watch from './commands/watch-deps'; +import buildCache from './commands/build-cache'; import lintCommand from './commands/lint'; import testCommand from './commands/testCommand'; import appBuild from './commands/app/build'; @@ -76,6 +77,23 @@ const main = (argv: string[]) => { .description('Watch all dependencies while running another command') .action(actionHandler(watch)); + program + .command('build-cache') + .description('Wrap build command with a cache') + .option( + '--input ', + 'List of input directories that invalidate the cache [.]', + (value, acc) => acc.concat(value), + [], + ) + .option('--output ', 'Output directory to cache', 'dist') + .option( + '--cache-dir ', + 'Cache dir', + '/node_modules/.cache/backstage-builds', + ) + .action(actionHandler(buildCache)); + program.on('command:*', () => { console.log(); console.log( From de34ab58e29d560c30183400a93c069cd7980d2f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 10:21:44 +0100 Subject: [PATCH 02/24] cli/commands/build-cache: initial caching without build --- .../cli/src/commands/build-cache/index.ts | 155 +++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 4cf3d8ef02..2d9a33d9f2 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -14,9 +14,162 @@ * limitations under the License. */ +import fs from 'fs-extra'; +import { resolve as resolvePath, relative as relativePath } from 'path'; +import { promisify } from 'util'; +import { exec as execCb } from 'child_process'; +import { Command } from 'commander'; +import { ExitCodeError } from '../../helpers/errors'; +const exec = promisify(execCb); + +const INFO_FILE = '.backstage-build-cache'; + +type Options = { + inputs: string[]; + output: string; + cacheDir: string; + repoRoot: string; +}; + +async function parseOptions(cmd: Command): Promise { + const repoRoot = await run('git rev-parse --show-toplevel'); + const argTransformer = (arg: string) => + resolvePath(arg.replace(//g, repoRoot).replace(/'/g, '')); + + const inputs = cmd.input.map(argTransformer) as string[]; + if (inputs.length === 0) { + inputs.push(argTransformer('.')); + } + const output = argTransformer(cmd.output); + const cacheDir = argTransformer(cmd.cacheDir); + return { inputs, output, cacheDir, repoRoot }; +} + /* * The build-cache command is used to make builds a no-op if there are no changes to the package. * It supports both local development where the output directory remains intact, as well as CI * where the output directory is stored in a separate cache dir. */ -export default async () => {}; +export default async (cmd: Command, args: string[]) => { + const options = await parseOptions(cmd); + console.log('DEBUG: options =', options); + console.log('DEBUG: args =', args); + + const cache = await readCache(options); + console.log('DEBUG: cache =', cache); + + const trees = await getInputHashes(options); + console.log('DEBUG: trees =', trees); + + const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); + if (!cacheHit) { + await build(options); + + await fs.writeFile( + resolvePath(options.output, INFO_FILE), + JSON.stringify({ trees }, null, 2), + 'utf-8', + ); + if (cache.writable) { + console.log(`DEBUG: write cache`); + console.log('DEBUG: cache.location =', cache.location); + await fs.remove(cache.location); + console.log('DEBUG: options.output =', options.output); + await fs.copy(options.output, cache.location); + } + } else if (cache.needsCopy) { + console.log(`DEBUG: would copy cache`); + await fs.remove(options.output); + await fs.copy(cache.location, options.output); + } + + const ls = await run('ls derp'); + console.log('DEBUG: ls =', ls); +}; + +async function build(options: Options): Promise { + console.log(`Imma build ${options.output}`); +} + +async function run(cmd: string) { + try { + const { stdout } = await exec(cmd); + return stdout.trim(); + } catch (error) { + if (error.stderr) { + process.stderr.write(error.stderr); + } + throw new ExitCodeError(error.code, cmd); + } +} + +type Cache = { + // External location of the cache outside the output folder + location: string; + readable?: boolean; + writable?: boolean; + needsCopy?: boolean; + trees?: string[]; +}; + +async function readCache(options: Options): Promise { + const repoPath = relativePath(options.repoRoot, process.cwd()); + const location = resolvePath(options.cacheDir, repoPath); + + // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing + try { + await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); + } catch (error) { + return { location }; + } + + try { + const outputCacheExists = await fs.pathExists( + resolvePath(options.output, INFO_FILE), + ); + if (outputCacheExists) { + const trees = await readInfoFile(options.output); + if (trees) { + return { + location, + trees, + readable: true, + writable: true, + }; + } + } + + const externalCacheExists = await fs.pathExists(location); + if (externalCacheExists) { + const trees = await readInfoFile(location); + if (trees) { + return { + location, + trees, + readable: true, + writable: true, + needsCopy: true, + }; + } + } + } catch (error) { + console.log(`Cache not found, ${error}`); + } + return { location, writable: true }; +} + +async function readInfoFile(dir: string): Promise { + const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf-8'); + const { trees } = JSON.parse(infoContents); + return trees; +} + +async function getInputHashes(options: Options): Promise { + const trees = []; + for (const input of options.inputs) { + const output = await run(`git ls-tree HEAD '${input}'`); + const [, , sha] = output.split(/\s+/, 3); + trees.push(sha); + } + return trees; +} From d4fc4375641fc64d4a71ff4ec2e99bbdd007bc9d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 11:23:04 +0100 Subject: [PATCH 03/24] cli/commands/build-cache: store external cache in tar archives --- packages/cli/package.json | 2 + .../cli/src/commands/build-cache/index.ts | 86 ++++++++++++++----- yarn.lock | 39 ++++++++- 3 files changed, 105 insertions(+), 22 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index f26b13a421..ba039dcd5e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -32,10 +32,12 @@ "@types/ora": "^3.2.0", "@types/react-dev-utils": "^9.0.4", "@types/recursive-readdir": "^2.2.0", + "@types/tar": "^4.0.3", "@types/webpack": "^4.41.7", "@types/webpack-dev-server": "^3.10.0", "del": "^5.1.0", "nodemon": "^2.0.2", + "tar": "^6.0.1", "ts-node": "^8.6.2" }, "bin": { diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 2d9a33d9f2..8138db54f7 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -15,14 +15,20 @@ */ import fs from 'fs-extra'; -import { resolve as resolvePath, relative as relativePath } from 'path'; +import { + dirname, + resolve as resolvePath, + relative as relativePath, +} from 'path'; import { promisify } from 'util'; import { exec as execCb } from 'child_process'; import { Command } from 'commander'; +import tar from 'tar'; import { ExitCodeError } from '../../helpers/errors'; const exec = promisify(execCb); const INFO_FILE = '.backstage-build-cache'; +const CACHE_ARCHIVE = 'cache.tgz'; type Options = { inputs: string[]; @@ -62,25 +68,29 @@ export default async (cmd: Command, args: string[]) => { console.log('DEBUG: trees =', trees); const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); + console.log('DEBUG: cacheHit =', cacheHit); if (!cacheHit) { await build(options); - await fs.writeFile( - resolvePath(options.output, INFO_FILE), - JSON.stringify({ trees }, null, 2), - 'utf-8', - ); if (cache.writable) { + await fs.writeFile( + resolvePath(options.output, INFO_FILE), + JSON.stringify({ trees }, null, 2), + 'utf8', + ); console.log(`DEBUG: write cache`); - console.log('DEBUG: cache.location =', cache.location); - await fs.remove(cache.location); - console.log('DEBUG: options.output =', options.output); - await fs.copy(options.output, cache.location); + console.log('DEBUG: cache.archivePath =', cache.archivePath); + await fs.remove(cache.archivePath); + await fs.ensureDir(dirname(cache.archivePath)); + await tar.create( + { gzip: true, file: cache.archivePath, cwd: options.output }, + ['.'], + ); } } else if (cache.needsCopy) { - console.log(`DEBUG: would copy cache`); await fs.remove(options.output); - await fs.copy(cache.location, options.output); + await fs.ensureDir(options.output); + await tar.extract({ file: cache.archivePath, cwd: options.output }); } const ls = await run('ls derp'); @@ -105,7 +115,7 @@ async function run(cmd: string) { type Cache = { // External location of the cache outside the output folder - location: string; + archivePath: string; readable?: boolean; writable?: boolean; needsCopy?: boolean; @@ -115,12 +125,13 @@ type Cache = { async function readCache(options: Options): Promise { const repoPath = relativePath(options.repoRoot, process.cwd()); const location = resolvePath(options.cacheDir, repoPath); + const archivePath = resolvePath(location, CACHE_ARCHIVE); // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing try { - await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); + // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); } catch (error) { - return { location }; + return { archivePath }; } try { @@ -131,7 +142,7 @@ async function readCache(options: Options): Promise { const trees = await readInfoFile(options.output); if (trees) { return { - location, + archivePath, trees, readable: true, writable: true, @@ -141,10 +152,10 @@ async function readCache(options: Options): Promise { const externalCacheExists = await fs.pathExists(location); if (externalCacheExists) { - const trees = await readInfoFile(location); + const trees = await readInfoFileFromArchive(archivePath); if (trees) { return { - location, + archivePath, trees, readable: true, writable: true, @@ -155,15 +166,50 @@ async function readCache(options: Options): Promise { } catch (error) { console.log(`Cache not found, ${error}`); } - return { location, writable: true }; + return { archivePath, writable: true }; } async function readInfoFile(dir: string): Promise { - const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf-8'); + const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf8'); const { trees } = JSON.parse(infoContents); return trees; } +async function readInfoFileFromArchive( + archivePath: string, +): Promise { + const reader = fs.createReadStream(archivePath); + const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); + + const infoEntry = await new Promise((resolve, reject) => { + parser.on('entry', entry => { + if (entry.path === `./${INFO_FILE}`) { + resolve(entry); + reader.close(); + } else { + entry.resume(); + } + }); + parser.on('end', () => { + reject(new Error('cache archive did not contain build info')); + }); + parser.on('error', error => reject(error)); + + reader.pipe(parser); + }); + + const infoData = await new Promise((resolve, reject) => { + const chunks = new Array(); + infoEntry.on('data', chunk => chunks.push(chunk)); + infoEntry.on('end', () => resolve(Buffer.concat(chunks))); + infoEntry.on('error', error => reject(error)); + }); + + const info = JSON.parse(infoData.toString('utf8')); + + return info.trees; +} + async function getInputHashes(options: Options): Promise { const trees = []; for (const input of options.inputs) { diff --git a/yarn.lock b/yarn.lock index dcde1de95a..db975122b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3612,6 +3612,13 @@ resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== +"@types/minipass@*": + version "2.2.0" + resolved "https://registry.npmjs.org/@types/minipass/-/minipass-2.2.0.tgz#51ad404e8eb1fa961f75ec61205796807b6f9651" + integrity sha512-wuzZksN4w4kyfoOv/dlpov4NOunwutLA/q7uc00xU02ZyUY+aoM5PWIXEKBMnm0NHd4a+N71BMjq+x7+2Af1fg== + dependencies: + "@types/node" "*" + "@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2": version "13.9.2" resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349" @@ -3776,6 +3783,14 @@ resolved "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.5.tgz#9adbc12950582aa65ead76bffdf39fe0c27a3c02" integrity sha512-/gG2M/Imw7cQFp8PGvz/SwocNrmKFjFsm5Pb8HdbHkZ1K8pmuPzOX4VeVoiEecFCVf4CsN1r3/BRvx+6sNqwtQ== +"@types/tar@^4.0.3": + version "4.0.3" + resolved "https://registry.npmjs.org/@types/tar/-/tar-4.0.3.tgz#e2cce0b8ff4f285293243f5971bd7199176ac489" + integrity sha512-Z7AVMMlkI8NTWF0qGhC4QIX0zkV/+y0J8x7b/RsHrN0310+YNjoJd8UrApCiGBCWtKjxS9QhNqLi2UJNToh5hA== + dependencies: + "@types/minipass" "*" + "@types/node" "*" + "@types/testing-library__cypress@^5.0.3": version "5.0.3" resolved "https://registry.npmjs.org/@types/testing-library__cypress/-/testing-library__cypress-5.0.3.tgz#94969b7c1eea96e09d8e023a1d225590fa75a1fe" @@ -5795,7 +5810,7 @@ chokidar@^3.2.2, chokidar@^3.3.0, chokidar@^3.3.1: optionalDependencies: fsevents "~2.1.2" -chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.4: +chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.3, chownr@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== @@ -13039,6 +13054,14 @@ minizlib@^1.2.1: dependencies: minipass "^2.9.0" +minizlib@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz#fd52c645301ef09a63a2c209697c294c6ce02cf3" + integrity sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mississippi@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022" @@ -13078,7 +13101,7 @@ mkdirp-promise@^5.0.1: dependencies: mkdirp "*" -mkdirp@*: +mkdirp@*, mkdirp@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.3.tgz#4cf2e30ad45959dddea53ad97d518b6c8205e1ea" integrity sha512-6uCP4Qc0sWsgMLy1EOqqS/3rjDHOEnsStVr/4vtAIK2Y5i2kA7lFFejYrpIyiN9w0pYf4ckeCYT9f1r1P9KX5g== @@ -18083,6 +18106,18 @@ tar@^4.4.10, tar@^4.4.12, tar@^4.4.13, tar@^4.4.8: safe-buffer "^5.1.2" yallist "^3.0.3" +tar@^6.0.1: + version "6.0.1" + resolved "https://registry.npmjs.org/tar/-/tar-6.0.1.tgz#7b3bd6c313cb6e0153770108f8d70ac298607efa" + integrity sha512-bKhKrrz2FJJj5s7wynxy/fyxpE0CmCjmOQ1KV4KkgXFWOgoIT/NbTMnB1n+LFNrNk0SSBVGGxcK5AGsyC+pW5Q== + dependencies: + chownr "^1.1.3" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.0" + mkdirp "^1.0.3" + yallist "^4.0.0" + telejson@^3.2.0: version "3.3.0" resolved "https://registry.npmjs.org/telejson/-/telejson-3.3.0.tgz#6d814f3c0d254d5c4770085aad063e266b56ad03" From 282af223519616e1ffba7db7c43159d3d95cda07 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 11:38:46 +0100 Subject: [PATCH 04/24] cli/commands/build-cache: refactor + cleanup output --- .../cli/src/commands/build-cache/index.ts | 109 +++++++++--------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 8138db54f7..6d62159eeb 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -51,6 +51,10 @@ async function parseOptions(cmd: Command): Promise { return { inputs, output, cacheDir, repoRoot }; } +function print(msg: string) { + process.stdout.write(`[build-cache] ${msg}\n`); +} + /* * The build-cache command is used to make builds a no-op if there are no changes to the package. * It supports both local development where the output directory remains intact, as well as CI @@ -58,47 +62,54 @@ async function parseOptions(cmd: Command): Promise { */ export default async (cmd: Command, args: string[]) => { const options = await parseOptions(cmd); - console.log('DEBUG: options =', options); - console.log('DEBUG: args =', args); - const cache = await readCache(options); - console.log('DEBUG: cache =', cache); - const trees = await getInputHashes(options); - console.log('DEBUG: trees =', trees); const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); - console.log('DEBUG: cacheHit =', cacheHit); - if (!cacheHit) { - await build(options); + if (cacheHit) { + print('cache hit, no need to build'); + + if (cache.needsCopy) { + print('copying external cache to build output dir'); + await copyFromExternalCache(cache, options); + } + } else { + print('cache miss, need to build'); + + await build(args); if (cache.writable) { - await fs.writeFile( - resolvePath(options.output, INFO_FILE), - JSON.stringify({ trees }, null, 2), - 'utf8', - ); - console.log(`DEBUG: write cache`); - console.log('DEBUG: cache.archivePath =', cache.archivePath); - await fs.remove(cache.archivePath); - await fs.ensureDir(dirname(cache.archivePath)); - await tar.create( - { gzip: true, file: cache.archivePath, cwd: options.output }, - ['.'], - ); + print('caching build output'); + const infoData = Buffer.from(JSON.stringify({ trees }, null, 2), 'utf8'); + await fs.writeFile(resolvePath(options.output, INFO_FILE), infoData); + await copyToExternalCache(cache, options); } - } else if (cache.needsCopy) { - await fs.remove(options.output); - await fs.ensureDir(options.output); - await tar.extract({ file: cache.archivePath, cwd: options.output }); } - - const ls = await run('ls derp'); - console.log('DEBUG: ls =', ls); }; -async function build(options: Options): Promise { - console.log(`Imma build ${options.output}`); +async function copyToExternalCache( + cache: Cache, + options: Options, +): Promise { + await fs.remove(cache.archivePath); + await fs.ensureDir(dirname(cache.archivePath)); + await tar.create( + { gzip: true, file: cache.archivePath, cwd: options.output }, + ['.'], + ); +} + +async function copyFromExternalCache( + cache: Cache, + options: Options, +): Promise { + await fs.remove(options.output); + await fs.ensureDir(options.output); + await tar.extract({ file: cache.archivePath, cwd: options.output }); +} + +async function build(args: string[]): Promise { + console.log(`Will build using '${args.join(' ')}'`); } async function run(cmd: string) { @@ -134,22 +145,22 @@ async function readCache(options: Options): Promise { return { archivePath }; } - try { - const outputCacheExists = await fs.pathExists( - resolvePath(options.output, INFO_FILE), - ); - if (outputCacheExists) { - const trees = await readInfoFile(options.output); - if (trees) { - return { - archivePath, - trees, - readable: true, - writable: true, - }; - } + const infoFilePath = resolvePath(options.output, INFO_FILE); + const outputCacheExists = await fs.pathExists(infoFilePath); + if (outputCacheExists) { + const infoData = await fs.readFile(infoFilePath); + const { trees } = JSON.parse(infoData.toString('utf8')); + if (trees) { + return { + archivePath, + trees, + readable: true, + writable: true, + }; } + } + try { const externalCacheExists = await fs.pathExists(location); if (externalCacheExists) { const trees = await readInfoFileFromArchive(archivePath); @@ -164,17 +175,11 @@ async function readCache(options: Options): Promise { } } } catch (error) { - console.log(`Cache not found, ${error}`); + print(`failed to read external cache archive, ${error}`); } return { archivePath, writable: true }; } -async function readInfoFile(dir: string): Promise { - const infoContents = await fs.readFile(resolvePath(dir, INFO_FILE), 'utf8'); - const { trees } = JSON.parse(infoContents); - return trees; -} - async function readInfoFileFromArchive( archivePath: string, ): Promise { From eaf80a4271d23d7736e490470cc44057cd1f8a48 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 11:38:55 +0100 Subject: [PATCH 05/24] cli/commands/build-cache: fix unhandled error --- packages/cli/src/commands/build-cache/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 6d62159eeb..5fe9852a28 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -199,6 +199,7 @@ async function readInfoFileFromArchive( reject(new Error('cache archive did not contain build info')); }); parser.on('error', error => reject(error)); + reader.on('error', error => reject(error)); reader.pipe(parser); }); From 4e79446dc91e2ff84fe52aa79e16f501bb30b7bf Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 11:49:32 +0100 Subject: [PATCH 06/24] cli/commands/build-cache: run build command --- packages/cli/src/commands/build-cache/index.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 5fe9852a28..6ae1844412 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -25,6 +25,7 @@ import { exec as execCb } from 'child_process'; import { Command } from 'commander'; import tar from 'tar'; import { ExitCodeError } from '../../helpers/errors'; +import { run } from '../../helpers/run'; const exec = promisify(execCb); const INFO_FILE = '.backstage-build-cache'; @@ -38,7 +39,7 @@ type Options = { }; async function parseOptions(cmd: Command): Promise { - const repoRoot = await run('git rev-parse --show-toplevel'); + const repoRoot = await cmdOutput('git rev-parse --show-toplevel'); const argTransformer = (arg: string) => resolvePath(arg.replace(//g, repoRoot).replace(/'/g, '')); @@ -67,11 +68,11 @@ export default async (cmd: Command, args: string[]) => { const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); if (cacheHit) { - print('cache hit, no need to build'); - if (cache.needsCopy) { - print('copying external cache to build output dir'); + print('external cache hit, copying from external cache'); await copyFromExternalCache(cache, options); + } else { + print('cache hit, nothing to be done'); } } else { print('cache miss, need to build'); @@ -108,11 +109,11 @@ async function copyFromExternalCache( await tar.extract({ file: cache.archivePath, cwd: options.output }); } -async function build(args: string[]): Promise { - console.log(`Will build using '${args.join(' ')}'`); +async function build([prog, ...args]: string[]): Promise { + await run(prog, args); } -async function run(cmd: string) { +async function cmdOutput(cmd: string) { try { const { stdout } = await exec(cmd); return stdout.trim(); @@ -219,7 +220,7 @@ async function readInfoFileFromArchive( async function getInputHashes(options: Options): Promise { const trees = []; for (const input of options.inputs) { - const output = await run(`git ls-tree HEAD '${input}'`); + const output = await cmdOutput(`git ls-tree HEAD '${input}'`); const [, , sha] = output.split(/\s+/, 3); trees.push(sha); } From 2dae5fccafcb924e37580ca40a1252e80b0cd032 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 12:00:54 +0100 Subject: [PATCH 07/24] cli/commands/build-cache: move archive handling to separate module --- .../cli/src/commands/build-cache/archive.ts | 74 +++++++++++++++++++ .../cli/src/commands/build-cache/index.ts | 72 ++---------------- 2 files changed, 80 insertions(+), 66 deletions(-) create mode 100644 packages/cli/src/commands/build-cache/archive.ts diff --git a/packages/cli/src/commands/build-cache/archive.ts b/packages/cli/src/commands/build-cache/archive.ts new file mode 100644 index 0000000000..be74504f1f --- /dev/null +++ b/packages/cli/src/commands/build-cache/archive.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import tar from 'tar'; +import { dirname } from 'path'; + +export async function readFileFromArchive( + archivePath: string, + filePath: string, +): Promise { + const reader = fs.createReadStream(archivePath); + const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); + + const fileEntry = await new Promise((resolve, reject) => { + parser.on('entry', entry => { + if (entry.path === `./${filePath}`) { + resolve(entry); + reader.close(); + } else { + entry.resume(); + } + }); + parser.on('end', () => { + reject(new Error('cache archive did not contain build info')); + }); + parser.on('error', error => reject(error)); + reader.on('error', error => reject(error)); + + reader.pipe(parser); + }); + + const data = await new Promise((resolve, reject) => { + const chunks = new Array(); + fileEntry.on('data', chunk => chunks.push(chunk)); + fileEntry.on('end', () => resolve(Buffer.concat(chunks))); + fileEntry.on('error', error => reject(error)); + }); + + return data; +} + +// packages all files in inputDir into an archive at archivePath, deleting any existing archive +export async function createArchive( + archivePath: string, + inputDir: string, +): Promise { + await fs.remove(archivePath); + await fs.ensureDir(dirname(archivePath)); + await tar.create({ gzip: true, file: archivePath, cwd: inputDir }, ['.']); +} + +// extracts archive at archive path into outputDir, deleting any existing files at outputDir +export async function extractArchive( + archivePath: string, + outputDir: string, +): Promise { + await fs.remove(outputDir); + await fs.ensureDir(outputDir); + await tar.extract({ file: archivePath, cwd: outputDir }); +} diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 6ae1844412..49a02f6939 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -15,17 +15,13 @@ */ import fs from 'fs-extra'; -import { - dirname, - resolve as resolvePath, - relative as relativePath, -} from 'path'; +import { resolve as resolvePath, relative as relativePath } from 'path'; import { promisify } from 'util'; import { exec as execCb } from 'child_process'; import { Command } from 'commander'; -import tar from 'tar'; import { ExitCodeError } from '../../helpers/errors'; import { run } from '../../helpers/run'; +import { readFileFromArchive, extractArchive, createArchive } from './archive'; const exec = promisify(execCb); const INFO_FILE = '.backstage-build-cache'; @@ -70,7 +66,7 @@ export default async (cmd: Command, args: string[]) => { if (cacheHit) { if (cache.needsCopy) { print('external cache hit, copying from external cache'); - await copyFromExternalCache(cache, options); + await extractArchive(cache.archivePath, options.output); } else { print('cache hit, nothing to be done'); } @@ -83,32 +79,11 @@ export default async (cmd: Command, args: string[]) => { print('caching build output'); const infoData = Buffer.from(JSON.stringify({ trees }, null, 2), 'utf8'); await fs.writeFile(resolvePath(options.output, INFO_FILE), infoData); - await copyToExternalCache(cache, options); + await createArchive(cache.archivePath, options.output); } } }; -async function copyToExternalCache( - cache: Cache, - options: Options, -): Promise { - await fs.remove(cache.archivePath); - await fs.ensureDir(dirname(cache.archivePath)); - await tar.create( - { gzip: true, file: cache.archivePath, cwd: options.output }, - ['.'], - ); -} - -async function copyFromExternalCache( - cache: Cache, - options: Options, -): Promise { - await fs.remove(options.output); - await fs.ensureDir(options.output); - await tar.extract({ file: cache.archivePath, cwd: options.output }); -} - async function build([prog, ...args]: string[]): Promise { await run(prog, args); } @@ -164,7 +139,8 @@ async function readCache(options: Options): Promise { try { const externalCacheExists = await fs.pathExists(location); if (externalCacheExists) { - const trees = await readInfoFileFromArchive(archivePath); + const infoData = await readFileFromArchive(archivePath, INFO_FILE); + const { trees } = JSON.parse(infoData.toString('utf8')); if (trees) { return { archivePath, @@ -181,42 +157,6 @@ async function readCache(options: Options): Promise { return { archivePath, writable: true }; } -async function readInfoFileFromArchive( - archivePath: string, -): Promise { - const reader = fs.createReadStream(archivePath); - const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); - - const infoEntry = await new Promise((resolve, reject) => { - parser.on('entry', entry => { - if (entry.path === `./${INFO_FILE}`) { - resolve(entry); - reader.close(); - } else { - entry.resume(); - } - }); - parser.on('end', () => { - reject(new Error('cache archive did not contain build info')); - }); - parser.on('error', error => reject(error)); - reader.on('error', error => reject(error)); - - reader.pipe(parser); - }); - - const infoData = await new Promise((resolve, reject) => { - const chunks = new Array(); - infoEntry.on('data', chunk => chunks.push(chunk)); - infoEntry.on('end', () => resolve(Buffer.concat(chunks))); - infoEntry.on('error', error => reject(error)); - }); - - const info = JSON.parse(infoData.toString('utf8')); - - return info.trees; -} - async function getInputHashes(options: Options): Promise { const trees = []; for (const input of options.inputs) { From 0644f47645bcaf3f9675b3ecdd8c8675b9c75d22 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 12:07:31 +0100 Subject: [PATCH 08/24] cli/commands/build-cache: simplify build and move cmdOutput to helpers as runPlain --- .../cli/src/commands/build-cache/index.ts | 28 +++---------------- packages/cli/src/helpers/run.ts | 21 +++++++++++++- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 49a02f6939..b01b5309ab 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -16,13 +16,9 @@ import fs from 'fs-extra'; import { resolve as resolvePath, relative as relativePath } from 'path'; -import { promisify } from 'util'; -import { exec as execCb } from 'child_process'; import { Command } from 'commander'; -import { ExitCodeError } from '../../helpers/errors'; -import { run } from '../../helpers/run'; +import { run, runPlain } from '../../helpers/run'; import { readFileFromArchive, extractArchive, createArchive } from './archive'; -const exec = promisify(execCb); const INFO_FILE = '.backstage-build-cache'; const CACHE_ARCHIVE = 'cache.tgz'; @@ -35,7 +31,7 @@ type Options = { }; async function parseOptions(cmd: Command): Promise { - const repoRoot = await cmdOutput('git rev-parse --show-toplevel'); + const repoRoot = await runPlain('git rev-parse --show-toplevel'); const argTransformer = (arg: string) => resolvePath(arg.replace(//g, repoRoot).replace(/'/g, '')); @@ -73,7 +69,7 @@ export default async (cmd: Command, args: string[]) => { } else { print('cache miss, need to build'); - await build(args); + await run(args[0], args.slice(1)); if (cache.writable) { print('caching build output'); @@ -84,22 +80,6 @@ export default async (cmd: Command, args: string[]) => { } }; -async function build([prog, ...args]: string[]): Promise { - await run(prog, args); -} - -async function cmdOutput(cmd: string) { - try { - const { stdout } = await exec(cmd); - return stdout.trim(); - } catch (error) { - if (error.stderr) { - process.stderr.write(error.stderr); - } - throw new ExitCodeError(error.code, cmd); - } -} - type Cache = { // External location of the cache outside the output folder archivePath: string; @@ -160,7 +140,7 @@ async function readCache(options: Options): Promise { async function getInputHashes(options: Options): Promise { const trees = []; for (const input of options.inputs) { - const output = await cmdOutput(`git ls-tree HEAD '${input}'`); + const output = await runPlain(`git ls-tree HEAD '${input}'`); const [, , sha] = output.split(/\s+/, 3); trees.push(sha); } diff --git a/packages/cli/src/helpers/run.ts b/packages/cli/src/helpers/run.ts index 3eec19a148..6a1376baf8 100644 --- a/packages/cli/src/helpers/run.ts +++ b/packages/cli/src/helpers/run.ts @@ -14,8 +14,15 @@ * limitations under the License. */ -import { SpawnOptions, spawn, ChildProcess } from 'child_process'; +import { + SpawnOptions, + spawn, + ChildProcess, + exec as execCb, +} from 'child_process'; import { ExitCodeError } from './errors'; +import { promisify } from 'util'; +const exec = promisify(execCb); type SpawnOptionsPartialEnv = Omit & { env?: Partial; @@ -43,6 +50,18 @@ export async function run( await waitForExit(child, name); } +export async function runPlain(cmd: string) { + try { + const { stdout } = await exec(cmd); + return stdout.trim(); + } catch (error) { + if (error.stderr) { + process.stderr.write(error.stderr); + } + throw new ExitCodeError(error.code, cmd); + } +} + export async function waitForExit( child: ChildProcess & { exitCode?: number }, name?: string, From 8a35da15e5449151440f66cbeea06eae555c580a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 12:15:19 +0100 Subject: [PATCH 09/24] cli/commands/build-cache: separate out cache and options handling --- .../cli/src/commands/build-cache/cache.ts | 99 +++++++++++++++++ .../cli/src/commands/build-cache/index.ts | 104 +----------------- .../cli/src/commands/build-cache/options.ts | 40 +++++++ 3 files changed, 145 insertions(+), 98 deletions(-) create mode 100644 packages/cli/src/commands/build-cache/cache.ts create mode 100644 packages/cli/src/commands/build-cache/options.ts diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts new file mode 100644 index 0000000000..67030e6eb2 --- /dev/null +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, relative as relativePath } from 'path'; +import { runPlain } from '../../helpers/run'; +import { readFileFromArchive } from './archive'; +import { Options } from './options'; + +export type Cache = { + // External location of the cache outside the output folder + archivePath: string; + readable?: boolean; + writable?: boolean; + needsCopy?: boolean; + trees?: string[]; +}; + +const CACHE_ARCHIVE = 'cache.tgz'; +const INFO_FILE = '.backstage-build-cache'; + +export async function readCache(options: Options): Promise { + const repoPath = relativePath(options.repoRoot, process.cwd()); + const location = resolvePath(options.cacheDir, repoPath); + const archivePath = resolvePath(location, CACHE_ARCHIVE); + + // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing + try { + // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); + } catch (error) { + return { archivePath }; + } + + const infoFilePath = resolvePath(options.output, INFO_FILE); + const outputCacheExists = await fs.pathExists(infoFilePath); + if (outputCacheExists) { + const infoData = await fs.readFile(infoFilePath); + const { trees } = JSON.parse(infoData.toString('utf8')); + if (trees) { + return { + archivePath, + trees, + readable: true, + writable: true, + }; + } + } + + try { + const externalCacheExists = await fs.pathExists(location); + if (externalCacheExists) { + const infoData = await readFileFromArchive(archivePath, INFO_FILE); + const { trees } = JSON.parse(infoData.toString('utf8')); + if (trees) { + return { + archivePath, + trees, + readable: true, + writable: true, + needsCopy: true, + }; + } + } + } catch (error) { + throw new Error(`failed to read external cache archive, ${error}`); + } + return { archivePath, writable: true }; +} + +export async function writeCacheInfo( + trees: string[], + options: Options, +): Promise { + const infoData = Buffer.from(JSON.stringify({ trees }, null, 2), 'utf8'); + await fs.writeFile(resolvePath(options.output, INFO_FILE), infoData); +} + +export async function readInputHashes(options: Options): Promise { + const trees = []; + for (const input of options.inputs) { + const output = await runPlain(`git ls-tree HEAD '${input}'`); + const [, , sha] = output.split(/\s+/, 3); + trees.push(sha); + } + return trees; +} diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index b01b5309ab..07bd6c89b9 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -14,35 +14,11 @@ * limitations under the License. */ -import fs from 'fs-extra'; -import { resolve as resolvePath, relative as relativePath } from 'path'; import { Command } from 'commander'; -import { run, runPlain } from '../../helpers/run'; -import { readFileFromArchive, extractArchive, createArchive } from './archive'; - -const INFO_FILE = '.backstage-build-cache'; -const CACHE_ARCHIVE = 'cache.tgz'; - -type Options = { - inputs: string[]; - output: string; - cacheDir: string; - repoRoot: string; -}; - -async function parseOptions(cmd: Command): Promise { - const repoRoot = await runPlain('git rev-parse --show-toplevel'); - const argTransformer = (arg: string) => - resolvePath(arg.replace(//g, repoRoot).replace(/'/g, '')); - - const inputs = cmd.input.map(argTransformer) as string[]; - if (inputs.length === 0) { - inputs.push(argTransformer('.')); - } - const output = argTransformer(cmd.output); - const cacheDir = argTransformer(cmd.cacheDir); - return { inputs, output, cacheDir, repoRoot }; -} +import { run } from '../../helpers/run'; +import { extractArchive, createArchive } from './archive'; +import { readCache, readInputHashes, writeCacheInfo } from './cache'; +import { parseOptions } from './options'; function print(msg: string) { process.stdout.write(`[build-cache] ${msg}\n`); @@ -56,7 +32,7 @@ function print(msg: string) { export default async (cmd: Command, args: string[]) => { const options = await parseOptions(cmd); const cache = await readCache(options); - const trees = await getInputHashes(options); + const trees = await readInputHashes(options); const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); if (cacheHit) { @@ -73,76 +49,8 @@ export default async (cmd: Command, args: string[]) => { if (cache.writable) { print('caching build output'); - const infoData = Buffer.from(JSON.stringify({ trees }, null, 2), 'utf8'); - await fs.writeFile(resolvePath(options.output, INFO_FILE), infoData); + await writeCacheInfo(trees, options); await createArchive(cache.archivePath, options.output); } } }; - -type Cache = { - // External location of the cache outside the output folder - archivePath: string; - readable?: boolean; - writable?: boolean; - needsCopy?: boolean; - trees?: string[]; -}; - -async function readCache(options: Options): Promise { - const repoPath = relativePath(options.repoRoot, process.cwd()); - const location = resolvePath(options.cacheDir, repoPath); - const archivePath = resolvePath(location, CACHE_ARCHIVE); - - // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing - try { - // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); - } catch (error) { - return { archivePath }; - } - - const infoFilePath = resolvePath(options.output, INFO_FILE); - const outputCacheExists = await fs.pathExists(infoFilePath); - if (outputCacheExists) { - const infoData = await fs.readFile(infoFilePath); - const { trees } = JSON.parse(infoData.toString('utf8')); - if (trees) { - return { - archivePath, - trees, - readable: true, - writable: true, - }; - } - } - - try { - const externalCacheExists = await fs.pathExists(location); - if (externalCacheExists) { - const infoData = await readFileFromArchive(archivePath, INFO_FILE); - const { trees } = JSON.parse(infoData.toString('utf8')); - if (trees) { - return { - archivePath, - trees, - readable: true, - writable: true, - needsCopy: true, - }; - } - } - } catch (error) { - print(`failed to read external cache archive, ${error}`); - } - return { archivePath, writable: true }; -} - -async function getInputHashes(options: Options): Promise { - const trees = []; - for (const input of options.inputs) { - const output = await runPlain(`git ls-tree HEAD '${input}'`); - const [, , sha] = output.split(/\s+/, 3); - trees.push(sha); - } - return trees; -} diff --git a/packages/cli/src/commands/build-cache/options.ts b/packages/cli/src/commands/build-cache/options.ts new file mode 100644 index 0000000000..8df62afaa5 --- /dev/null +++ b/packages/cli/src/commands/build-cache/options.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import { Command } from 'commander'; +import { runPlain } from '../../helpers/run'; + +export type Options = { + inputs: string[]; + output: string; + cacheDir: string; + repoRoot: string; +}; + +export async function parseOptions(cmd: Command): Promise { + const repoRoot = await runPlain('git rev-parse --show-toplevel'); + const argTransformer = (arg: string) => + resolvePath(arg.replace(//g, repoRoot).replace(/'/g, '')); + + const inputs = cmd.input.map(argTransformer) as string[]; + if (inputs.length === 0) { + inputs.push(argTransformer('.')); + } + const output = argTransformer(cmd.output); + const cacheDir = argTransformer(cmd.cacheDir); + return { inputs, output, cacheDir, repoRoot }; +} From 39efe52c9dab6bdf782ed10e9de3d69e3996d151 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 13:04:53 +0100 Subject: [PATCH 10/24] cli/commands/build-cache: refactor cache handling --- .../cli/src/commands/build-cache/cache.ts | 144 +++++++++++------- .../cli/src/commands/build-cache/index.ts | 18 +-- 2 files changed, 98 insertions(+), 64 deletions(-) diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts index 67030e6eb2..b4123f017f 100644 --- a/packages/cli/src/commands/build-cache/cache.ts +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -20,7 +20,7 @@ import { runPlain } from '../../helpers/run'; import { readFileFromArchive } from './archive'; import { Options } from './options'; -export type Cache = { +export type CacheHit = { // External location of the cache outside the output folder archivePath: string; readable?: boolean; @@ -29,71 +29,105 @@ export type Cache = { trees?: string[]; }; +export type CacheKey = string[]; + +type CacheEntry = { + key: CacheKey; + archivePath: string; +}; + const CACHE_ARCHIVE = 'cache.tgz'; const INFO_FILE = '.backstage-build-cache'; -export async function readCache(options: Options): Promise { - const repoPath = relativePath(options.repoRoot, process.cwd()); - const location = resolvePath(options.cacheDir, repoPath); - const archivePath = resolvePath(location, CACHE_ARCHIVE); +export class Cache { + static async read(options: Options) { + const repoPath = relativePath(options.repoRoot, process.cwd()); + const location = resolvePath(options.cacheDir, repoPath); - // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing - try { - // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); - } catch (error) { - return { archivePath }; + // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing + try { + // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); + } catch (error) { + return new Cache(); + } + + let localKey: CacheKey = []; + + const localInfoFile = resolvePath(options.output, INFO_FILE); + const outputCacheExists = await fs.pathExists(localInfoFile); + if (outputCacheExists) { + const infoData = await fs.readFile(localInfoFile); + const { trees } = JSON.parse(infoData.toString('utf8')); + localKey = trees; + } + + try { + const externalCacheExists = await fs.pathExists(location); + if (externalCacheExists) { + const archivePath = resolvePath(location, CACHE_ARCHIVE); + const infoData = await readFileFromArchive(archivePath, INFO_FILE); + const { trees } = JSON.parse(infoData.toString('utf8')); + if (trees) { + return new Cache([{ archivePath, key: trees }], location, localKey); + } + } + } catch (error) { + throw new Error(`failed to read external cache archive, ${error}`); + } + + return new Cache([], location, localKey); } - const infoFilePath = resolvePath(options.output, INFO_FILE); - const outputCacheExists = await fs.pathExists(infoFilePath); - if (outputCacheExists) { - const infoData = await fs.readFile(infoFilePath); - const { trees } = JSON.parse(infoData.toString('utf8')); - if (trees) { + static async readInputKey(inputPaths: string[]): Promise { + const trees = []; + for (const inputPath of inputPaths) { + const output = await runPlain(`git ls-tree HEAD '${inputPath}'`); + const [, , sha] = output.split(/\s+/, 3); + trees.push(sha); + } + return trees; + } + + constructor( + private readonly entries: CacheEntry[] = [], + private readonly location?: string, + private readonly localKey?: CacheKey, + ) {} + + find(key: CacheKey): CacheHit | undefined { + if (!this.location) { + return undefined; + } + if (this.localKey?.join(',') === key.join(',')) { return { - archivePath, - trees, - readable: true, - writable: true, + needsCopy: false, + archivePath: resolvePath(this.location, CACHE_ARCHIVE), }; } - } - - try { - const externalCacheExists = await fs.pathExists(location); - if (externalCacheExists) { - const infoData = await readFileFromArchive(archivePath, INFO_FILE); - const { trees } = JSON.parse(infoData.toString('utf8')); - if (trees) { - return { - archivePath, - trees, - readable: true, - writable: true, - needsCopy: true, - }; - } + const matchingEntry = this.entries.find(entry => entry.key === key); + if (!matchingEntry) { + return undefined; } - } catch (error) { - throw new Error(`failed to read external cache archive, ${error}`); + return { + needsCopy: true, + archivePath: matchingEntry.archivePath, + }; } - return { archivePath, writable: true }; -} -export async function writeCacheInfo( - trees: string[], - options: Options, -): Promise { - const infoData = Buffer.from(JSON.stringify({ trees }, null, 2), 'utf8'); - await fs.writeFile(resolvePath(options.output, INFO_FILE), infoData); -} - -export async function readInputHashes(options: Options): Promise { - const trees = []; - for (const input of options.inputs) { - const output = await runPlain(`git ls-tree HEAD '${input}'`); - const [, , sha] = output.split(/\s+/, 3); - trees.push(sha); + get shouldCacheOutput() { + return Boolean(this.location); + } + + async prepareOutput(key: CacheKey, outputDir: string): Promise { + if (!this.location) { + throw new Error("can't write cache output, no location set"); + } + const infoData = Buffer.from( + JSON.stringify({ trees: key }, null, 2), + 'utf8', + ); + await fs.writeFile(resolvePath(outputDir, INFO_FILE), infoData); + const archivePath = resolvePath(this.location, CACHE_ARCHIVE); + return archivePath; } - return trees; } diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 07bd6c89b9..003560c6bd 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -17,7 +17,7 @@ import { Command } from 'commander'; import { run } from '../../helpers/run'; import { extractArchive, createArchive } from './archive'; -import { readCache, readInputHashes, writeCacheInfo } from './cache'; +import { Cache } from './cache'; import { parseOptions } from './options'; function print(msg: string) { @@ -31,14 +31,14 @@ function print(msg: string) { */ export default async (cmd: Command, args: string[]) => { const options = await parseOptions(cmd); - const cache = await readCache(options); - const trees = await readInputHashes(options); + const cache = await Cache.read(options); + const key = await Cache.readInputKey(options.inputs); - const cacheHit = cache.readable && cache.trees?.join(',') === trees.join(','); + const cacheHit = cache.find(key); if (cacheHit) { - if (cache.needsCopy) { + if (cacheHit.needsCopy) { print('external cache hit, copying from external cache'); - await extractArchive(cache.archivePath, options.output); + await extractArchive(cacheHit.archivePath, options.output); } else { print('cache hit, nothing to be done'); } @@ -47,10 +47,10 @@ export default async (cmd: Command, args: string[]) => { await run(args[0], args.slice(1)); - if (cache.writable) { + if (cache.shouldCacheOutput) { print('caching build output'); - await writeCacheInfo(trees, options); - await createArchive(cache.archivePath, options.output); + const archivePath = await cache.prepareOutput(key, options.output); + await createArchive(archivePath, options.output); } } }; From f2d7d43b77ac6e0a2726a37bbd294e33f5e2f8ca Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 13:56:51 +0100 Subject: [PATCH 11/24] cli/commands/build-cache: add support for multiple cache archives --- .../cli/src/commands/build-cache/cache.ts | 167 +++++++++++------- .../cli/src/commands/build-cache/index.ts | 16 +- 2 files changed, 110 insertions(+), 73 deletions(-) diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts index b4123f017f..d5ed39e148 100644 --- a/packages/cli/src/commands/build-cache/cache.ts +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -17,27 +17,29 @@ import fs from 'fs-extra'; import { resolve as resolvePath, relative as relativePath } from 'path'; import { runPlain } from '../../helpers/run'; -import { readFileFromArchive } from './archive'; import { Options } from './options'; +import { extractArchive, createArchive } from './archive'; -export type CacheHit = { - // External location of the cache outside the output folder - archivePath: string; - readable?: boolean; - writable?: boolean; - needsCopy?: boolean; - trees?: string[]; +const MAX_ENTRIES = 10; +const INFO_FILE = '.backstage-build-cache'; + +export type CacheQueryResult = { + hit: boolean; + copy?: (outputDir: string) => Promise; + archive?: (outputDir: string) => Promise; }; export type CacheKey = string[]; type CacheEntry = { key: CacheKey; - archivePath: string; + path: string; }; -const CACHE_ARCHIVE = 'cache.tgz'; -const INFO_FILE = '.backstage-build-cache'; +type CacheInfo = { + key?: CacheKey; + entries?: CacheEntry[]; +}; export class Cache { static async read(options: Options) { @@ -51,31 +53,11 @@ export class Cache { return new Cache(); } - let localKey: CacheKey = []; + const outputInfo = await readCacheInfo(options.output); + const localKey = outputInfo?.key; - const localInfoFile = resolvePath(options.output, INFO_FILE); - const outputCacheExists = await fs.pathExists(localInfoFile); - if (outputCacheExists) { - const infoData = await fs.readFile(localInfoFile); - const { trees } = JSON.parse(infoData.toString('utf8')); - localKey = trees; - } - - try { - const externalCacheExists = await fs.pathExists(location); - if (externalCacheExists) { - const archivePath = resolvePath(location, CACHE_ARCHIVE); - const infoData = await readFileFromArchive(archivePath, INFO_FILE); - const { trees } = JSON.parse(infoData.toString('utf8')); - if (trees) { - return new Cache([{ archivePath, key: trees }], location, localKey); - } - } - } catch (error) { - throw new Error(`failed to read external cache archive, ${error}`); - } - - return new Cache([], location, localKey); + const { entries = [] } = (await readCacheInfo(location)) ?? {}; + return new Cache(entries, location, localKey); } static async readInputKey(inputPaths: string[]): Promise { @@ -94,40 +76,97 @@ export class Cache { private readonly localKey?: CacheKey, ) {} - find(key: CacheKey): CacheHit | undefined { - if (!this.location) { - return undefined; + query(key: CacheKey): CacheQueryResult { + const { location } = this; + if (!location) { + return { hit: false }; } - if (this.localKey?.join(',') === key.join(',')) { - return { - needsCopy: false, - archivePath: resolvePath(this.location, CACHE_ARCHIVE), - }; + + const archive = async (outputDir: string) => { + await writeCacheInfo(outputDir, { key }); + + const timestamp = new Date().toISOString().replace(/-|:|\..*/g, ''); + const rand = Math.random() + .toString(36) + .slice(2, 6); + const archiveName = `cache-${timestamp}-${rand}.tgz`; + const archivePath = resolvePath(location, archiveName); + + // Read existing entries and prepend the new one + const { entries = [] } = (await readCacheInfo(location)) ?? {}; + + // Check if there's already aan entry for this key, in that case we just wanna bump it + const entryIndex = entries.findIndex(e => compareKeys(e.key, key)); + if (entryIndex !== -1) { + const [existingEntry] = entries.splice(entryIndex, 1); + entries.unshift(existingEntry); + + await writeCacheInfo(location, { entries }); + return; + } + + // Create and add new archive to entries + await createArchive(archivePath, outputDir); + entries.unshift({ key, path: archiveName }); + + // Remove old cache entries + const removedEntries = entries.splice(MAX_ENTRIES); + for (const entry of removedEntries) { + try { + await fs.remove(resolvePath(location, entry.path)); + } catch (error) { + process.stderr.write(`failed to remove old cache entry, ${error}\n`); + } + } + + await writeCacheInfo(location, { entries }); + }; + + if (compareKeys(this.localKey, key)) { + return { hit: true, archive }; } - const matchingEntry = this.entries.find(entry => entry.key === key); + + const matchingEntry = this.entries.find(e => compareKeys(e.key, key)); if (!matchingEntry) { - return undefined; + return { hit: false, archive }; } + return { - needsCopy: true, - archivePath: matchingEntry.archivePath, + hit: true, + archive, + copy: async (outputDir: string) => { + const archivePath = resolvePath(location, matchingEntry.path); + await extractArchive(archivePath, outputDir); + }, }; } - - get shouldCacheOutput() { - return Boolean(this.location); - } - - async prepareOutput(key: CacheKey, outputDir: string): Promise { - if (!this.location) { - throw new Error("can't write cache output, no location set"); - } - const infoData = Buffer.from( - JSON.stringify({ trees: key }, null, 2), - 'utf8', - ); - await fs.writeFile(resolvePath(outputDir, INFO_FILE), infoData); - const archivePath = resolvePath(this.location, CACHE_ARCHIVE); - return archivePath; - } +} + +function compareKeys(a?: CacheKey, b?: CacheKey): boolean { + if (!a || !b) { + return false; + } + return a.join(',') === b.join(','); +} + +async function readCacheInfo( + parentDir: string, +): Promise { + const infoFile = resolvePath(parentDir, INFO_FILE); + const exists = await fs.pathExists(infoFile); + if (!exists) { + return undefined; + } + + const infoData = await fs.readFile(infoFile); + const cacheInfo = JSON.parse(infoData.toString('utf8')) as CacheInfo; + return cacheInfo; +} + +async function writeCacheInfo( + parentDir: string, + cacheInfo: CacheInfo, +): Promise { + const infoData = Buffer.from(JSON.stringify(cacheInfo, null, 2), 'utf8'); + await fs.writeFile(resolvePath(parentDir, INFO_FILE), infoData); } diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 003560c6bd..2aa2911116 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -16,7 +16,6 @@ import { Command } from 'commander'; import { run } from '../../helpers/run'; -import { extractArchive, createArchive } from './archive'; import { Cache } from './cache'; import { parseOptions } from './options'; @@ -34,11 +33,11 @@ export default async (cmd: Command, args: string[]) => { const cache = await Cache.read(options); const key = await Cache.readInputKey(options.inputs); - const cacheHit = cache.find(key); - if (cacheHit) { - if (cacheHit.needsCopy) { - print('external cache hit, copying from external cache'); - await extractArchive(cacheHit.archivePath, options.output); + const cacheResult = cache.query(key); + if (cacheResult.hit) { + if (cacheResult.copy) { + print('external cache hit, copying archive to output folder'); + await cacheResult.copy(options.output); } else { print('cache hit, nothing to be done'); } @@ -47,10 +46,9 @@ export default async (cmd: Command, args: string[]) => { await run(args[0], args.slice(1)); - if (cache.shouldCacheOutput) { + if (cacheResult.archive) { print('caching build output'); - const archivePath = await cache.prepareOutput(key, options.output); - await createArchive(archivePath, options.output); + await cacheResult.archive(options.output); } } }; From 140cd7e4db030401bba660a1ffe934e0c3f1e529 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 13:57:26 +0100 Subject: [PATCH 12/24] cli/commands/build-cache: remove unused archive function --- .../cli/src/commands/build-cache/archive.ts | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/packages/cli/src/commands/build-cache/archive.ts b/packages/cli/src/commands/build-cache/archive.ts index be74504f1f..b4a736240d 100644 --- a/packages/cli/src/commands/build-cache/archive.ts +++ b/packages/cli/src/commands/build-cache/archive.ts @@ -18,41 +18,6 @@ import fs from 'fs-extra'; import tar from 'tar'; import { dirname } from 'path'; -export async function readFileFromArchive( - archivePath: string, - filePath: string, -): Promise { - const reader = fs.createReadStream(archivePath); - const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); - - const fileEntry = await new Promise((resolve, reject) => { - parser.on('entry', entry => { - if (entry.path === `./${filePath}`) { - resolve(entry); - reader.close(); - } else { - entry.resume(); - } - }); - parser.on('end', () => { - reject(new Error('cache archive did not contain build info')); - }); - parser.on('error', error => reject(error)); - reader.on('error', error => reject(error)); - - reader.pipe(parser); - }); - - const data = await new Promise((resolve, reject) => { - const chunks = new Array(); - fileEntry.on('data', chunk => chunks.push(chunk)); - fileEntry.on('end', () => resolve(Buffer.concat(chunks))); - fileEntry.on('error', error => reject(error)); - }); - - return data; -} - // packages all files in inputDir into an archive at archivePath, deleting any existing archive export async function createArchive( archivePath: string, From ec28ed8e2acd9f113984de0823d831527abcea91 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 14:31:42 +0100 Subject: [PATCH 13/24] packages/app: rename build script to bundle and only run in CI when app changed --- .github/workflows/frontend.yml | 2 ++ package.json | 1 + packages/app/package.json | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 8095106910..196cfd7541 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -43,5 +43,7 @@ jobs: - run: yarn lint - run: yarn build - run: yarn test + - name: yarn bundle, if app was changed + run: git diff --quiet origin/master HEAD -- packages/app || yarn bundle - name: verify storybook run: yarn workspace storybook build-storybook diff --git a/package.json b/package.json index eaf85f7567..f6c20e56fa 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ }, "scripts": { "start": "yarn build && yarn workspace example-app start", + "bundle": "yarn build && yarn workspace example-app bundle", "build": "lerna run build", "test": "cross-env CI=true lerna run test --since origin/master -- --coverage", "create-plugin": "backstage-cli create-plugin", diff --git a/packages/app/package.json b/packages/app/package.json index c332d2f0da..681593a369 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -26,7 +26,7 @@ }, "scripts": { "start": "backstage-cli app:serve", - "build": "backstage-cli app:build", + "bundle": "backstage-cli app:build", "test": "backstage-cli test", "test:e2e": "start-server-and-test start http://localhost:3000 cy:dev", "test:e2e:ci": "start-server-and-test start http://localhost:3000 cy:run", From e59ddb4d7c7efc35a84d1e77b578fa2d65c6bcec Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 14:49:47 +0100 Subject: [PATCH 14/24] packages,plugins: use build-cache command --- packages/cli/package.json | 2 +- packages/cli/templates/default-plugin/package.json.hbs | 2 +- packages/core/package.json | 2 +- plugins/home-page/package.json | 2 +- plugins/welcome/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index ba039dcd5e..005fcf01ad 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -19,7 +19,7 @@ "main": "dist", "scripts": { "exec": "npx ts-node ./src", - "build": "tsc --outDir dist --noEmit false --module CommonJS", + "build": "backstage-cli build-cache -- tsc --outDir dist --noEmit false --module CommonJS", "lint": "backstage-cli lint", "test": "backstage-cli test", "start": "nodemon ." diff --git a/packages/cli/templates/default-plugin/package.json.hbs b/packages/cli/templates/default-plugin/package.json.hbs index b2bd6f2c82..c7220a8fd3 100644 --- a/packages/cli/templates/default-plugin/package.json.hbs +++ b/packages/cli/templates/default-plugin/package.json.hbs @@ -6,7 +6,7 @@ "license": "Apache-2.0", "private": true, "scripts": { - "build": "backstage-cli plugin:build", + "build": "backstage-cli build-cache -- backstage-cli plugin:build", "lint": "backstage-cli lint", "test": "backstage-cli test" }, diff --git a/packages/core/package.json b/packages/core/package.json index 8adb1307fe..1105d79c46 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,7 +19,7 @@ "main": "dist/cjs/index.js", "types": "dist/cjs/index.d.ts", "scripts": { - "build": "tsc --outDir dist/cjs --noEmit false --module CommonJS", + "build": "backstage-cli build-cache -- tsc --outDir dist/cjs --noEmit false --module CommonJS", "lint": "backstage-cli lint", "test": "backstage-cli test" }, diff --git a/plugins/home-page/package.json b/plugins/home-page/package.json index 33be457cba..91e3da417f 100644 --- a/plugins/home-page/package.json +++ b/plugins/home-page/package.json @@ -19,7 +19,7 @@ "react-dom": "^16.12.0" }, "scripts": { - "build": "backstage-cli plugin:build", + "build": "backstage-cli build-cache -- backstage-cli plugin:build", "lint": "backstage-cli lint", "test": "backstage-cli test" }, diff --git a/plugins/welcome/package.json b/plugins/welcome/package.json index b38977afab..e1672b4c40 100644 --- a/plugins/welcome/package.json +++ b/plugins/welcome/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "private": true, "scripts": { - "build": "backstage-cli plugin:build", + "build": "backstage-cli build-cache -- backstage-cli plugin:build", "lint": "backstage-cli lint", "test": "backstage-cli test" }, From e2014836b5b2c8773812f47fab54f2ab9dd3ceb1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 14:55:19 +0100 Subject: [PATCH 15/24] github/workflows: repurpose node_modules cache to cache .cache --- .github/workflows/frontend.yml | 6 +++--- .github/workflows/master.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 196cfd7541..ad9744b12d 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -29,11 +29,11 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: cache node_modules + - name: cache node_modules/.cache uses: actions/cache@v1 with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + path: node_modules/.cache + key: node_modules-cache - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e82c6c3a4b..9a1791a594 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -27,11 +27,11 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: cache node_modules + - name: cache node_modules/.cache uses: actions/cache@v1 with: - path: node_modules - key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} + path: node_modules/.cache + key: node_modules-cache - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: From b87a760dd5fec4be396cbe4beac833886e317b9b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 14:56:15 +0100 Subject: [PATCH 16/24] package: only lint changes since master --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f6c20e56fa..137c4de0bb 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "test": "cross-env CI=true lerna run test --since origin/master -- --coverage", "create-plugin": "backstage-cli create-plugin", "release": "if [ \"$(git symbolic-ref --short HEAD)\" = master ]; then echo \"don't try to release master\"; exit 1; else lerna version --no-push; fi", - "lint": "lerna run lint", + "lint": "cross-env CI=true lerna run lint --since origin/master --", "storybook": "yarn workspace storybook start" }, "workspaces": { From 3ca00be6071c5c97572f762e8732db973274e371 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 15:07:21 +0100 Subject: [PATCH 17/24] cli/commands/build-cache: enable dirty check --- packages/cli/src/commands/build-cache/cache.ts | 9 +++++---- packages/cli/src/helpers/run.ts | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts index d5ed39e148..da653a0331 100644 --- a/packages/cli/src/commands/build-cache/cache.ts +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -16,7 +16,7 @@ import fs from 'fs-extra'; import { resolve as resolvePath, relative as relativePath } from 'path'; -import { runPlain } from '../../helpers/run'; +import { runPlain, runCheck } from '../../helpers/run'; import { Options } from './options'; import { extractArchive, createArchive } from './archive'; @@ -47,9 +47,10 @@ export class Cache { const location = resolvePath(options.cacheDir, repoPath); // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing - try { - // await exec(`git diff --quiet HEAD -- ${options.inputs.join(' ')}`); - } catch (error) { + const noChanges = await runCheck( + `git diff --quiet HEAD -- ${options.inputs.join(' ')}`, + ); + if (!noChanges) { return new Cache(); } diff --git a/packages/cli/src/helpers/run.ts b/packages/cli/src/helpers/run.ts index 6a1376baf8..cbeb2c080c 100644 --- a/packages/cli/src/helpers/run.ts +++ b/packages/cli/src/helpers/run.ts @@ -62,6 +62,15 @@ export async function runPlain(cmd: string) { } } +export async function runCheck(cmd: string): Promise { + try { + await exec(cmd); + return true; + } catch (error) { + return false; + } +} + export async function waitForExit( child: ChildProcess & { exitCode?: number }, name?: string, From 2961e216ccbf032b241517820e18dcbd40d39549 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 15:52:45 +0100 Subject: [PATCH 18/24] cli/commands/build-cache: allow cache dir to be set through env --- packages/cli/src/commands/build-cache/options.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/build-cache/options.ts b/packages/cli/src/commands/build-cache/options.ts index 8df62afaa5..f70131ff0a 100644 --- a/packages/cli/src/commands/build-cache/options.ts +++ b/packages/cli/src/commands/build-cache/options.ts @@ -35,6 +35,8 @@ export async function parseOptions(cmd: Command): Promise { inputs.push(argTransformer('.')); } const output = argTransformer(cmd.output); - const cacheDir = argTransformer(cmd.cacheDir); + const cacheDir = argTransformer( + process.env.BACKSTAGE_CACHE_DIR || cmd.cacheDir, + ); return { inputs, output, cacheDir, repoRoot }; } From 9cb19a62aa5658645020d6500ec7cdecac35f8f6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 16:04:31 +0100 Subject: [PATCH 19/24] github/workflows: separate out build cache and use proper key + restore keys --- .github/workflows/frontend.yml | 9 ++++++--- .github/workflows/master.yml | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index ad9744b12d..cea02fec90 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -14,6 +14,7 @@ jobs: env: CI: true + BACKSTAGE_CACHE_DIR: /.backstage-build-cache steps: - uses: actions/checkout@v2 @@ -29,11 +30,13 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: cache node_modules/.cache + - name: cache build cache uses: actions/cache@v1 with: - path: node_modules/.cache - key: node_modules-cache + path: .backstage-build-cache + key: build-cache-${{ github.sha }} + restore-keys: | + build-cache- - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9a1791a594..1b08426d29 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -14,6 +14,7 @@ jobs: env: CI: true + BACKSTAGE_CACHE_DIR: /.backstage-build-cache steps: - uses: actions/checkout@v2 @@ -27,11 +28,13 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - name: cache node_modules/.cache + - name: cache build cache uses: actions/cache@v1 with: - path: node_modules/.cache - key: node_modules-cache + path: .backstage-build-cache + key: build-cache-${{ github.sha }} + restore-keys: | + build-cache- - name: use node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: From 56a7969eac1fdbf66a5d85cfd9ecee996f2e414a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 16:36:37 +0100 Subject: [PATCH 20/24] github/workflows: add back node_modules cache --- .github/workflows/frontend.yml | 5 +++++ .github/workflows/master.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index cea02fec90..05fcbd78d4 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -30,6 +30,11 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- + - name: cache node_modules + uses: actions/cache@v1 + with: + path: node_modules + key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} - name: cache build cache uses: actions/cache@v1 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 1b08426d29..1b3b3e3900 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -28,6 +28,11 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- + - name: cache node_modules + uses: actions/cache@v1 + with: + path: node_modules + key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }} - name: cache build cache uses: actions/cache@v1 with: From edd21b984eb8a57c0a30b636e09cd0255b5d3b00 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 27 Mar 2020 16:59:40 +0100 Subject: [PATCH 21/24] cli/commands/build-cache: make max entries configurable and set to 2 in CI --- .github/workflows/frontend.yml | 1 + .github/workflows/master.yml | 1 + packages/cli/src/commands/build-cache/cache.ts | 7 +++---- packages/cli/src/commands/build-cache/index.ts | 2 +- packages/cli/src/commands/build-cache/options.ts | 7 ++++++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 05fcbd78d4..478ff65744 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -15,6 +15,7 @@ jobs: env: CI: true BACKSTAGE_CACHE_DIR: /.backstage-build-cache + BACKSTAGE_CACHE_MAX_ENTRIES: 2 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 1b3b3e3900..f36154aa48 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -15,6 +15,7 @@ jobs: env: CI: true BACKSTAGE_CACHE_DIR: /.backstage-build-cache + BACKSTAGE_CACHE_MAX_ENTRIES: 2 steps: - uses: actions/checkout@v2 diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts index da653a0331..df91598e1e 100644 --- a/packages/cli/src/commands/build-cache/cache.ts +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -20,13 +20,12 @@ import { runPlain, runCheck } from '../../helpers/run'; import { Options } from './options'; import { extractArchive, createArchive } from './archive'; -const MAX_ENTRIES = 10; const INFO_FILE = '.backstage-build-cache'; export type CacheQueryResult = { hit: boolean; copy?: (outputDir: string) => Promise; - archive?: (outputDir: string) => Promise; + archive?: (outputDir: string, maxEntries: number) => Promise; }; export type CacheKey = string[]; @@ -83,7 +82,7 @@ export class Cache { return { hit: false }; } - const archive = async (outputDir: string) => { + const archive = async (outputDir: string, maxEntries: number) => { await writeCacheInfo(outputDir, { key }); const timestamp = new Date().toISOString().replace(/-|:|\..*/g, ''); @@ -111,7 +110,7 @@ export class Cache { entries.unshift({ key, path: archiveName }); // Remove old cache entries - const removedEntries = entries.splice(MAX_ENTRIES); + const removedEntries = entries.splice(maxEntries); for (const entry of removedEntries) { try { await fs.remove(resolvePath(location, entry.path)); diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 2aa2911116..8760d944af 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -48,7 +48,7 @@ export default async (cmd: Command, args: string[]) => { if (cacheResult.archive) { print('caching build output'); - await cacheResult.archive(options.output); + await cacheResult.archive(options.output, options.maxCacheEntries); } } }; diff --git a/packages/cli/src/commands/build-cache/options.ts b/packages/cli/src/commands/build-cache/options.ts index f70131ff0a..bad7d6c27e 100644 --- a/packages/cli/src/commands/build-cache/options.ts +++ b/packages/cli/src/commands/build-cache/options.ts @@ -18,10 +18,13 @@ import { resolve as resolvePath } from 'path'; import { Command } from 'commander'; import { runPlain } from '../../helpers/run'; +const DEFAULT_MAX_ENTRIES = 10; + export type Options = { inputs: string[]; output: string; cacheDir: string; + maxCacheEntries: number; repoRoot: string; }; @@ -38,5 +41,7 @@ export async function parseOptions(cmd: Command): Promise { const cacheDir = argTransformer( process.env.BACKSTAGE_CACHE_DIR || cmd.cacheDir, ); - return { inputs, output, cacheDir, repoRoot }; + const maxCacheEntries = + Number(process.env.BACKSTAGE_CACHE_MAX_ENTRIES) || DEFAULT_MAX_ENTRIES; + return { inputs, output, cacheDir, repoRoot, maxCacheEntries }; } From bfceff2c84e6840649fa1bd01f4ba3f0f4322d25 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 30 Mar 2020 18:06:32 +0200 Subject: [PATCH 22/24] cli/commands/build-cache: more docs --- .../cli/src/commands/build-cache/cache.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts index df91598e1e..82159fc3ca 100644 --- a/packages/cli/src/commands/build-cache/cache.ts +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -22,25 +22,41 @@ import { extractArchive, createArchive } from './archive'; const INFO_FILE = '.backstage-build-cache'; +// Result from a cache query export type CacheQueryResult = { + // True if there was a cache hit hit: boolean; + // If there is a cache hit and this method is defined, it needs to be called to restore the + // output contents before continuing. copy?: (outputDir: string) => Promise; + // If this method is defined, it should be called after a successful build to archive the output content. + // The content will be archived using the same key as was used in the cache. archive?: (outputDir: string, maxEntries: number) => Promise; }; +// Key that determines whether cached output can be reused export type CacheKey = string[]; type CacheEntry = { + // Key for the input of this cache entry key: CacheKey; + // Path to the archive of this cache entry path: string; }; +// Struct containing information about cache entries on the filesystem. +// Stored inside the INFO_FILE as JSON. type CacheInfo = { + // Optional key entry for the contents of the current directory. Used to key the + // output present in the outputs folder, where the info file resides inside the output folder. key?: CacheKey; + // Optional list of cache archives present in the same directory. Resides in the external + // cache location inside one info file for each package. entries?: CacheEntry[]; }; export class Cache { + // Read the current cache state form the filesystem. static async read(options: Options) { const repoPath = relativePath(options.repoRoot, process.cwd()); const location = resolvePath(options.cacheDir, repoPath); @@ -60,6 +76,7 @@ export class Cache { return new Cache(entries, location, localKey); } + // Generates a key based on the contents of the input paths static async readInputKey(inputPaths: string[]): Promise { const trees = []; for (const inputPath of inputPaths) { @@ -76,6 +93,7 @@ export class Cache { private readonly localKey?: CacheKey, ) {} + // Query for the presense of cached output for a given key query(key: CacheKey): CacheQueryResult { const { location } = this; if (!location) { @@ -142,6 +160,7 @@ export class Cache { } } +// Compares to cache keys, returning true if they are both defined and equal function compareKeys(a?: CacheKey, b?: CacheKey): boolean { if (!a || !b) { return false; @@ -149,6 +168,7 @@ function compareKeys(a?: CacheKey, b?: CacheKey): boolean { return a.join(',') === b.join(','); } +// Read and parse a cache info file in the given directory async function readCacheInfo( parentDir: string, ): Promise { @@ -163,6 +183,7 @@ async function readCacheInfo( return cacheInfo; } +// Write a cache info file to the given directory async function writeCacheInfo( parentDir: string, cacheInfo: CacheInfo, From c4b6afffaf095e5b9e34e87fa2656d2b5c0da9ad Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 30 Mar 2020 18:16:15 +0200 Subject: [PATCH 23/24] cli/commands/build-cache: refactor dirty check to make key undefined instead of cache --- .../cli/src/commands/build-cache/cache.ts | 42 ++++++++++--------- .../cli/src/commands/build-cache/index.ts | 26 +++++++----- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/packages/cli/src/commands/build-cache/cache.ts b/packages/cli/src/commands/build-cache/cache.ts index 82159fc3ca..f52e081a3f 100644 --- a/packages/cli/src/commands/build-cache/cache.ts +++ b/packages/cli/src/commands/build-cache/cache.ts @@ -29,9 +29,9 @@ export type CacheQueryResult = { // If there is a cache hit and this method is defined, it needs to be called to restore the // output contents before continuing. copy?: (outputDir: string) => Promise; - // If this method is defined, it should be called after a successful build to archive the output content. - // The content will be archived using the same key as was used in the cache. - archive?: (outputDir: string, maxEntries: number) => Promise; + // Call after a successful build to archive the output content. + // The content will be archived using the same key as was used to query the cache. + archive: (outputDir: string, maxEntries: number) => Promise; }; // Key that determines whether cached output can be reused @@ -61,26 +61,31 @@ export class Cache { const repoPath = relativePath(options.repoRoot, process.cwd()); const location = resolvePath(options.cacheDir, repoPath); - // Make sure we don't have any uncommitted changes to the input, in that case we consider the cache to be missing - const noChanges = await runCheck( - `git diff --quiet HEAD -- ${options.inputs.join(' ')}`, - ); - if (!noChanges) { - return new Cache(); - } - const outputInfo = await readCacheInfo(options.output); const localKey = outputInfo?.key; const { entries = [] } = (await readCacheInfo(location)) ?? {}; - return new Cache(entries, location, localKey); + return new Cache(location, entries, localKey); } - // Generates a key based on the contents of the input paths - static async readInputKey(inputPaths: string[]): Promise { + // Generates a key based on the contents of the input paths. + // Returns undefined if it's not possible to generate a stable key. + static async readInputKey( + inputPaths: string[], + ): Promise { + const quotedInputPaths = inputPaths.map(input => `'${input}'`); + + // Make sure we don't have any uncommitted changes to the input, in that case we skip caching. + const noChanges = await runCheck( + `git diff --quiet HEAD -- ${quotedInputPaths.join(' ')}`, + ); + if (!noChanges) { + return undefined; + } + const trees = []; - for (const inputPath of inputPaths) { - const output = await runPlain(`git ls-tree HEAD '${inputPath}'`); + for (const quotedInputPath of quotedInputPaths) { + const output = await runPlain(`git ls-tree HEAD ${quotedInputPath}`); const [, , sha] = output.split(/\s+/, 3); trees.push(sha); } @@ -88,17 +93,14 @@ export class Cache { } constructor( + private readonly location: string, private readonly entries: CacheEntry[] = [], - private readonly location?: string, private readonly localKey?: CacheKey, ) {} // Query for the presense of cached output for a given key query(key: CacheKey): CacheQueryResult { const { location } = this; - if (!location) { - return { hit: false }; - } const archive = async (outputDir: string, maxEntries: number) => { await writeCacheInfo(outputDir, { key }); diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 8760d944af..ab8984ecc3 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -30,8 +30,15 @@ function print(msg: string) { */ export default async (cmd: Command, args: string[]) => { const options = await parseOptions(cmd); - const cache = await Cache.read(options); + const key = await Cache.readInputKey(options.inputs); + if (!key) { + print('input directory is dirty, skipping cache'); + await run(args[0], args.slice(1)); + return; + } + + const cache = await Cache.read(options); const cacheResult = cache.query(key); if (cacheResult.hit) { @@ -41,14 +48,13 @@ export default async (cmd: Command, args: string[]) => { } else { print('cache hit, nothing to be done'); } - } else { - print('cache miss, need to build'); - - await run(args[0], args.slice(1)); - - if (cacheResult.archive) { - print('caching build output'); - await cacheResult.archive(options.output, options.maxCacheEntries); - } + return; } + + print('cache miss, need to build'); + + await run(args[0], args.slice(1)); + + print('caching build output'); + await cacheResult.archive(options.output, options.maxCacheEntries); }; From c914a628824b42c05af35795b2e3eb7a8ab349c4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 31 Mar 2020 15:31:22 +0200 Subject: [PATCH 24/24] github/workflows: also build app on changes to packages/core --- .github/workflows/frontend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 478ff65744..140969b332 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -53,6 +53,6 @@ jobs: - run: yarn build - run: yarn test - name: yarn bundle, if app was changed - run: git diff --quiet origin/master HEAD -- packages/app || yarn bundle + run: git diff --quiet origin/master HEAD -- packages/app packages/core || yarn bundle - name: verify storybook run: yarn workspace storybook build-storybook