Merge pull request #403 from spotify/rugvip/build-cache

cli: add experimental build-cache command and make sure things are only built on change in CI
This commit is contained in:
Patrik Oldsberg
2020-03-31 16:33:47 +02:00
committed by GitHub
16 changed files with 455 additions and 10 deletions
+11
View File
@@ -14,6 +14,8 @@ jobs:
env:
CI: true
BACKSTAGE_CACHE_DIR: <repoRoot>/.backstage-build-cache
BACKSTAGE_CACHE_MAX_ENTRIES: 2
steps:
- uses: actions/checkout@v2
@@ -34,6 +36,13 @@ jobs:
with:
path: node_modules
key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
- name: cache build cache
uses: actions/cache@v1
with:
path: .backstage-build-cache
key: build-cache-${{ github.sha }}
restore-keys: |
build-cache-
- name: use node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
@@ -43,5 +52,7 @@ jobs:
- run: yarn lint
- run: yarn build
- run: yarn test
- name: yarn bundle, if app was changed
run: git diff --quiet origin/master HEAD -- packages/app packages/core || yarn bundle
- name: verify storybook
run: yarn workspace storybook build-storybook
+9
View File
@@ -14,6 +14,8 @@ jobs:
env:
CI: true
BACKSTAGE_CACHE_DIR: <repoRoot>/.backstage-build-cache
BACKSTAGE_CACHE_MAX_ENTRIES: 2
steps:
- uses: actions/checkout@v2
@@ -32,6 +34,13 @@ jobs:
with:
path: node_modules
key: ${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
- name: cache build cache
uses: actions/cache@v1
with:
path: .backstage-build-cache
key: build-cache-${{ github.sha }}
restore-keys: |
build-cache-
- name: use node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
+2 -1
View File
@@ -6,11 +6,12 @@
},
"scripts": {
"start": "yarn build && yarn workspace example-app start",
"bundle": "yarn build && yarn workspace example-app bundle",
"build": "lerna run build",
"test": "cross-env CI=true lerna run test --since origin/master -- --coverage",
"create-plugin": "backstage-cli create-plugin",
"release": "if [ \"$(git symbolic-ref --short HEAD)\" = master ]; then echo \"don't try to release master\"; exit 1; else lerna version --no-push; fi",
"lint": "lerna run lint",
"lint": "cross-env CI=true lerna run lint --since origin/master --",
"storybook": "yarn workspace storybook start"
},
"workspaces": {
+1 -1
View File
@@ -26,7 +26,7 @@
},
"scripts": {
"start": "backstage-cli app:serve",
"build": "backstage-cli app:build",
"bundle": "backstage-cli app:build",
"test": "backstage-cli test",
"test:e2e": "start-server-and-test start http://localhost:3000 cy:dev",
"test:e2e:ci": "start-server-and-test start http://localhost:3000 cy:run",
+3 -1
View File
@@ -19,7 +19,7 @@
"main": "dist",
"scripts": {
"exec": "npx ts-node ./src",
"build": "tsc --outDir dist --noEmit false --module CommonJS",
"build": "backstage-cli build-cache -- tsc --outDir dist --noEmit false --module CommonJS",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
"start": "nodemon ."
@@ -32,10 +32,12 @@
"@types/ora": "^3.2.0",
"@types/react-dev-utils": "^9.0.4",
"@types/recursive-readdir": "^2.2.0",
"@types/tar": "^4.0.3",
"@types/webpack": "^4.41.7",
"@types/webpack-dev-server": "^3.10.0",
"del": "^5.1.0",
"nodemon": "^2.0.2",
"tar": "^6.0.1",
"ts-node": "^8.6.2"
},
"bin": {
@@ -0,0 +1,39 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import tar from 'tar';
import { dirname } from 'path';
// packages all files in inputDir into an archive at archivePath, deleting any existing archive
export async function createArchive(
archivePath: string,
inputDir: string,
): Promise<void> {
await fs.remove(archivePath);
await fs.ensureDir(dirname(archivePath));
await tar.create({ gzip: true, file: archivePath, cwd: inputDir }, ['.']);
}
// extracts archive at archive path into outputDir, deleting any existing files at outputDir
export async function extractArchive(
archivePath: string,
outputDir: string,
): Promise<void> {
await fs.remove(outputDir);
await fs.ensureDir(outputDir);
await tar.extract({ file: archivePath, cwd: outputDir });
}
@@ -0,0 +1,195 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { resolve as resolvePath, relative as relativePath } from 'path';
import { runPlain, runCheck } from '../../helpers/run';
import { Options } from './options';
import { extractArchive, createArchive } from './archive';
const INFO_FILE = '.backstage-build-cache';
// Result from a cache query
export type CacheQueryResult = {
// True if there was a cache hit
hit: boolean;
// If there is a cache hit and this method is defined, it needs to be called to restore the
// output contents before continuing.
copy?: (outputDir: string) => Promise<void>;
// Call after a successful build to archive the output content.
// The content will be archived using the same key as was used to query the cache.
archive: (outputDir: string, maxEntries: number) => Promise<void>;
};
// Key that determines whether cached output can be reused
export type CacheKey = string[];
type CacheEntry = {
// Key for the input of this cache entry
key: CacheKey;
// Path to the archive of this cache entry
path: string;
};
// Struct containing information about cache entries on the filesystem.
// Stored inside the INFO_FILE as JSON.
type CacheInfo = {
// Optional key entry for the contents of the current directory. Used to key the
// output present in the outputs folder, where the info file resides inside the output folder.
key?: CacheKey;
// Optional list of cache archives present in the same directory. Resides in the external
// cache location inside one info file for each package.
entries?: CacheEntry[];
};
export class Cache {
// Read the current cache state form the filesystem.
static async read(options: Options) {
const repoPath = relativePath(options.repoRoot, process.cwd());
const location = resolvePath(options.cacheDir, repoPath);
const outputInfo = await readCacheInfo(options.output);
const localKey = outputInfo?.key;
const { entries = [] } = (await readCacheInfo(location)) ?? {};
return new Cache(location, entries, localKey);
}
// Generates a key based on the contents of the input paths.
// Returns undefined if it's not possible to generate a stable key.
static async readInputKey(
inputPaths: string[],
): Promise<CacheKey | undefined> {
const quotedInputPaths = inputPaths.map(input => `'${input}'`);
// Make sure we don't have any uncommitted changes to the input, in that case we skip caching.
const noChanges = await runCheck(
`git diff --quiet HEAD -- ${quotedInputPaths.join(' ')}`,
);
if (!noChanges) {
return undefined;
}
const trees = [];
for (const quotedInputPath of quotedInputPaths) {
const output = await runPlain(`git ls-tree HEAD ${quotedInputPath}`);
const [, , sha] = output.split(/\s+/, 3);
trees.push(sha);
}
return trees;
}
constructor(
private readonly location: string,
private readonly entries: CacheEntry[] = [],
private readonly localKey?: CacheKey,
) {}
// Query for the presense of cached output for a given key
query(key: CacheKey): CacheQueryResult {
const { location } = this;
const archive = async (outputDir: string, maxEntries: number) => {
await writeCacheInfo(outputDir, { key });
const timestamp = new Date().toISOString().replace(/-|:|\..*/g, '');
const rand = Math.random()
.toString(36)
.slice(2, 6);
const archiveName = `cache-${timestamp}-${rand}.tgz`;
const archivePath = resolvePath(location, archiveName);
// Read existing entries and prepend the new one
const { entries = [] } = (await readCacheInfo(location)) ?? {};
// Check if there's already aan entry for this key, in that case we just wanna bump it
const entryIndex = entries.findIndex(e => compareKeys(e.key, key));
if (entryIndex !== -1) {
const [existingEntry] = entries.splice(entryIndex, 1);
entries.unshift(existingEntry);
await writeCacheInfo(location, { entries });
return;
}
// Create and add new archive to entries
await createArchive(archivePath, outputDir);
entries.unshift({ key, path: archiveName });
// Remove old cache entries
const removedEntries = entries.splice(maxEntries);
for (const entry of removedEntries) {
try {
await fs.remove(resolvePath(location, entry.path));
} catch (error) {
process.stderr.write(`failed to remove old cache entry, ${error}\n`);
}
}
await writeCacheInfo(location, { entries });
};
if (compareKeys(this.localKey, key)) {
return { hit: true, archive };
}
const matchingEntry = this.entries.find(e => compareKeys(e.key, key));
if (!matchingEntry) {
return { hit: false, archive };
}
return {
hit: true,
archive,
copy: async (outputDir: string) => {
const archivePath = resolvePath(location, matchingEntry.path);
await extractArchive(archivePath, outputDir);
},
};
}
}
// Compares to cache keys, returning true if they are both defined and equal
function compareKeys(a?: CacheKey, b?: CacheKey): boolean {
if (!a || !b) {
return false;
}
return a.join(',') === b.join(',');
}
// Read and parse a cache info file in the given directory
async function readCacheInfo(
parentDir: string,
): Promise<CacheInfo | undefined> {
const infoFile = resolvePath(parentDir, INFO_FILE);
const exists = await fs.pathExists(infoFile);
if (!exists) {
return undefined;
}
const infoData = await fs.readFile(infoFile);
const cacheInfo = JSON.parse(infoData.toString('utf8')) as CacheInfo;
return cacheInfo;
}
// Write a cache info file to the given directory
async function writeCacheInfo(
parentDir: string,
cacheInfo: CacheInfo,
): Promise<void> {
const infoData = Buffer.from(JSON.stringify(cacheInfo, null, 2), 'utf8');
await fs.writeFile(resolvePath(parentDir, INFO_FILE), infoData);
}
@@ -0,0 +1,60 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Command } from 'commander';
import { run } from '../../helpers/run';
import { Cache } from './cache';
import { parseOptions } from './options';
function print(msg: string) {
process.stdout.write(`[build-cache] ${msg}\n`);
}
/*
* The build-cache command is used to make builds a no-op if there are no changes to the package.
* It supports both local development where the output directory remains intact, as well as CI
* where the output directory is stored in a separate cache dir.
*/
export default async (cmd: Command, args: string[]) => {
const options = await parseOptions(cmd);
const key = await Cache.readInputKey(options.inputs);
if (!key) {
print('input directory is dirty, skipping cache');
await run(args[0], args.slice(1));
return;
}
const cache = await Cache.read(options);
const cacheResult = cache.query(key);
if (cacheResult.hit) {
if (cacheResult.copy) {
print('external cache hit, copying archive to output folder');
await cacheResult.copy(options.output);
} else {
print('cache hit, nothing to be done');
}
return;
}
print('cache miss, need to build');
await run(args[0], args.slice(1));
print('caching build output');
await cacheResult.archive(options.output, options.maxCacheEntries);
};
@@ -0,0 +1,47 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolve as resolvePath } from 'path';
import { Command } from 'commander';
import { runPlain } from '../../helpers/run';
const DEFAULT_MAX_ENTRIES = 10;
export type Options = {
inputs: string[];
output: string;
cacheDir: string;
maxCacheEntries: number;
repoRoot: string;
};
export async function parseOptions(cmd: Command): Promise<Options> {
const repoRoot = await runPlain('git rev-parse --show-toplevel');
const argTransformer = (arg: string) =>
resolvePath(arg.replace(/<repoRoot>/g, repoRoot).replace(/'/g, ''));
const inputs = cmd.input.map(argTransformer) as string[];
if (inputs.length === 0) {
inputs.push(argTransformer('.'));
}
const output = argTransformer(cmd.output);
const cacheDir = argTransformer(
process.env.BACKSTAGE_CACHE_DIR || cmd.cacheDir,
);
const maxCacheEntries =
Number(process.env.BACKSTAGE_CACHE_MAX_ENTRIES) || DEFAULT_MAX_ENTRIES;
return { inputs, output, cacheDir, repoRoot, maxCacheEntries };
}
+29 -1
View File
@@ -14,8 +14,15 @@
* limitations under the License.
*/
import { SpawnOptions, spawn, ChildProcess } from 'child_process';
import {
SpawnOptions,
spawn,
ChildProcess,
exec as execCb,
} from 'child_process';
import { ExitCodeError } from './errors';
import { promisify } from 'util';
const exec = promisify(execCb);
type SpawnOptionsPartialEnv = Omit<SpawnOptions, 'env'> & {
env?: Partial<NodeJS.ProcessEnv>;
@@ -43,6 +50,27 @@ export async function run(
await waitForExit(child, name);
}
export async function runPlain(cmd: string) {
try {
const { stdout } = await exec(cmd);
return stdout.trim();
} catch (error) {
if (error.stderr) {
process.stderr.write(error.stderr);
}
throw new ExitCodeError(error.code, cmd);
}
}
export async function runCheck(cmd: string): Promise<boolean> {
try {
await exec(cmd);
return true;
} catch (error) {
return false;
}
}
export async function waitForExit(
child: ChildProcess & { exitCode?: number },
name?: string,
+18
View File
@@ -19,6 +19,7 @@ import chalk from 'chalk';
import fs from 'fs';
import createPluginCommand from './commands/create-plugin/createPlugin';
import watch from './commands/watch-deps';
import buildCache from './commands/build-cache';
import lintCommand from './commands/lint';
import testCommand from './commands/testCommand';
import appBuild from './commands/app/build';
@@ -76,6 +77,23 @@ const main = (argv: string[]) => {
.description('Watch all dependencies while running another command')
.action(actionHandler(watch));
program
.command('build-cache')
.description('Wrap build command with a cache')
.option(
'--input <dirs>',
'List of input directories that invalidate the cache [.]',
(value, acc) => acc.concat(value),
[],
)
.option('--output <dir>', 'Output directory to cache', 'dist')
.option(
'--cache-dir <dir>',
'Cache dir',
'<repoRoot>/node_modules/.cache/backstage-builds',
)
.action(actionHandler(buildCache));
program.on('command:*', () => {
console.log();
console.log(
@@ -6,7 +6,7 @@
"license": "Apache-2.0",
"private": true,
"scripts": {
"build": "backstage-cli plugin:build",
"build": "backstage-cli build-cache -- backstage-cli plugin:build",
"lint": "backstage-cli lint",
"test": "backstage-cli test"
},
+1 -1
View File
@@ -19,7 +19,7 @@
"main": "dist/cjs/index.js",
"types": "dist/cjs/index.d.ts",
"scripts": {
"build": "tsc --outDir dist/cjs --noEmit false --module CommonJS",
"build": "backstage-cli build-cache -- tsc --outDir dist/cjs --noEmit false --module CommonJS",
"lint": "backstage-cli lint",
"test": "backstage-cli test"
},
+1 -1
View File
@@ -19,7 +19,7 @@
"react-dom": "^16.12.0"
},
"scripts": {
"build": "backstage-cli plugin:build",
"build": "backstage-cli build-cache -- backstage-cli plugin:build",
"lint": "backstage-cli lint",
"test": "backstage-cli test"
},
+1 -1
View File
@@ -6,7 +6,7 @@
"license": "Apache-2.0",
"private": true,
"scripts": {
"build": "backstage-cli plugin:build",
"build": "backstage-cli build-cache -- backstage-cli plugin:build",
"lint": "backstage-cli lint",
"test": "backstage-cli test"
},
+37 -2
View File
@@ -3612,6 +3612,13 @@
resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
"@types/minipass@*":
version "2.2.0"
resolved "https://registry.npmjs.org/@types/minipass/-/minipass-2.2.0.tgz#51ad404e8eb1fa961f75ec61205796807b6f9651"
integrity sha512-wuzZksN4w4kyfoOv/dlpov4NOunwutLA/q7uc00xU02ZyUY+aoM5PWIXEKBMnm0NHd4a+N71BMjq+x7+2Af1fg==
dependencies:
"@types/node" "*"
"@types/node@*", "@types/node@>= 8", "@types/node@^13.7.2":
version "13.9.2"
resolved "https://registry.npmjs.org/@types/node/-/node-13.9.2.tgz#ace1880c03594cc3e80206d96847157d8e7fa349"
@@ -3776,6 +3783,14 @@
resolved "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.5.tgz#9adbc12950582aa65ead76bffdf39fe0c27a3c02"
integrity sha512-/gG2M/Imw7cQFp8PGvz/SwocNrmKFjFsm5Pb8HdbHkZ1K8pmuPzOX4VeVoiEecFCVf4CsN1r3/BRvx+6sNqwtQ==
"@types/tar@^4.0.3":
version "4.0.3"
resolved "https://registry.npmjs.org/@types/tar/-/tar-4.0.3.tgz#e2cce0b8ff4f285293243f5971bd7199176ac489"
integrity sha512-Z7AVMMlkI8NTWF0qGhC4QIX0zkV/+y0J8x7b/RsHrN0310+YNjoJd8UrApCiGBCWtKjxS9QhNqLi2UJNToh5hA==
dependencies:
"@types/minipass" "*"
"@types/node" "*"
"@types/testing-library__cypress@^5.0.3":
version "5.0.3"
resolved "https://registry.npmjs.org/@types/testing-library__cypress/-/testing-library__cypress-5.0.3.tgz#94969b7c1eea96e09d8e023a1d225590fa75a1fe"
@@ -5795,7 +5810,7 @@ chokidar@^3.2.2, chokidar@^3.3.0, chokidar@^3.3.1:
optionalDependencies:
fsevents "~2.1.2"
chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.4:
chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.3, chownr@^1.1.4:
version "1.1.4"
resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
@@ -13039,6 +13054,14 @@ minizlib@^1.2.1:
dependencies:
minipass "^2.9.0"
minizlib@^2.1.0:
version "2.1.0"
resolved "https://registry.npmjs.org/minizlib/-/minizlib-2.1.0.tgz#fd52c645301ef09a63a2c209697c294c6ce02cf3"
integrity sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA==
dependencies:
minipass "^3.0.0"
yallist "^4.0.0"
mississippi@^3.0.0:
version "3.0.0"
resolved "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022"
@@ -13078,7 +13101,7 @@ mkdirp-promise@^5.0.1:
dependencies:
mkdirp "*"
mkdirp@*:
mkdirp@*, mkdirp@^1.0.3:
version "1.0.3"
resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.3.tgz#4cf2e30ad45959dddea53ad97d518b6c8205e1ea"
integrity sha512-6uCP4Qc0sWsgMLy1EOqqS/3rjDHOEnsStVr/4vtAIK2Y5i2kA7lFFejYrpIyiN9w0pYf4ckeCYT9f1r1P9KX5g==
@@ -18083,6 +18106,18 @@ tar@^4.4.10, tar@^4.4.12, tar@^4.4.13, tar@^4.4.8:
safe-buffer "^5.1.2"
yallist "^3.0.3"
tar@^6.0.1:
version "6.0.1"
resolved "https://registry.npmjs.org/tar/-/tar-6.0.1.tgz#7b3bd6c313cb6e0153770108f8d70ac298607efa"
integrity sha512-bKhKrrz2FJJj5s7wynxy/fyxpE0CmCjmOQ1KV4KkgXFWOgoIT/NbTMnB1n+LFNrNk0SSBVGGxcK5AGsyC+pW5Q==
dependencies:
chownr "^1.1.3"
fs-minipass "^2.0.0"
minipass "^3.0.0"
minizlib "^2.1.0"
mkdirp "^1.0.3"
yallist "^4.0.0"
telejson@^3.2.0:
version "3.3.0"
resolved "https://registry.npmjs.org/telejson/-/telejson-3.3.0.tgz#6d814f3c0d254d5c4770085aad063e266b56ad03"