diff --git a/packages/cli/src/commands/build-cache/archive.ts b/packages/cli/src/commands/build-cache/archive.ts new file mode 100644 index 0000000000..be74504f1f --- /dev/null +++ b/packages/cli/src/commands/build-cache/archive.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import tar from 'tar'; +import { dirname } from 'path'; + +export async function readFileFromArchive( + archivePath: string, + filePath: string, +): Promise { + const reader = fs.createReadStream(archivePath); + const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); + + const fileEntry = await new Promise((resolve, reject) => { + parser.on('entry', entry => { + if (entry.path === `./${filePath}`) { + resolve(entry); + reader.close(); + } else { + entry.resume(); + } + }); + parser.on('end', () => { + reject(new Error('cache archive did not contain build info')); + }); + parser.on('error', error => reject(error)); + reader.on('error', error => reject(error)); + + reader.pipe(parser); + }); + + const data = await new Promise((resolve, reject) => { + const chunks = new Array(); + fileEntry.on('data', chunk => chunks.push(chunk)); + fileEntry.on('end', () => resolve(Buffer.concat(chunks))); + fileEntry.on('error', error => reject(error)); + }); + + return data; +} + +// packages all files in inputDir into an archive at archivePath, deleting any existing archive +export async function createArchive( + archivePath: string, + inputDir: string, +): Promise { + await fs.remove(archivePath); + await fs.ensureDir(dirname(archivePath)); + await tar.create({ gzip: true, file: archivePath, cwd: inputDir }, ['.']); +} + +// extracts archive at archive path into outputDir, deleting any existing files at outputDir +export async function extractArchive( + archivePath: string, + outputDir: string, +): Promise { + await fs.remove(outputDir); + await fs.ensureDir(outputDir); + await tar.extract({ file: archivePath, cwd: outputDir }); +} diff --git a/packages/cli/src/commands/build-cache/index.ts b/packages/cli/src/commands/build-cache/index.ts index 6ae1844412..49a02f6939 100644 --- a/packages/cli/src/commands/build-cache/index.ts +++ b/packages/cli/src/commands/build-cache/index.ts @@ -15,17 +15,13 @@ */ import fs from 'fs-extra'; -import { - dirname, - resolve as resolvePath, - relative as relativePath, -} from 'path'; +import { resolve as resolvePath, relative as relativePath } from 'path'; import { promisify } from 'util'; import { exec as execCb } from 'child_process'; import { Command } from 'commander'; -import tar from 'tar'; import { ExitCodeError } from '../../helpers/errors'; import { run } from '../../helpers/run'; +import { readFileFromArchive, extractArchive, createArchive } from './archive'; const exec = promisify(execCb); const INFO_FILE = '.backstage-build-cache'; @@ -70,7 +66,7 @@ export default async (cmd: Command, args: string[]) => { if (cacheHit) { if (cache.needsCopy) { print('external cache hit, copying from external cache'); - await copyFromExternalCache(cache, options); + await extractArchive(cache.archivePath, options.output); } else { print('cache hit, nothing to be done'); } @@ -83,32 +79,11 @@ export default async (cmd: Command, args: string[]) => { print('caching build output'); const infoData = Buffer.from(JSON.stringify({ trees }, null, 2), 'utf8'); await fs.writeFile(resolvePath(options.output, INFO_FILE), infoData); - await copyToExternalCache(cache, options); + await createArchive(cache.archivePath, options.output); } } }; -async function copyToExternalCache( - cache: Cache, - options: Options, -): Promise { - await fs.remove(cache.archivePath); - await fs.ensureDir(dirname(cache.archivePath)); - await tar.create( - { gzip: true, file: cache.archivePath, cwd: options.output }, - ['.'], - ); -} - -async function copyFromExternalCache( - cache: Cache, - options: Options, -): Promise { - await fs.remove(options.output); - await fs.ensureDir(options.output); - await tar.extract({ file: cache.archivePath, cwd: options.output }); -} - async function build([prog, ...args]: string[]): Promise { await run(prog, args); } @@ -164,7 +139,8 @@ async function readCache(options: Options): Promise { try { const externalCacheExists = await fs.pathExists(location); if (externalCacheExists) { - const trees = await readInfoFileFromArchive(archivePath); + const infoData = await readFileFromArchive(archivePath, INFO_FILE); + const { trees } = JSON.parse(infoData.toString('utf8')); if (trees) { return { archivePath, @@ -181,42 +157,6 @@ async function readCache(options: Options): Promise { return { archivePath, writable: true }; } -async function readInfoFileFromArchive( - archivePath: string, -): Promise { - const reader = fs.createReadStream(archivePath); - const parser = new ((tar.Parse as unknown) as { new (): tar.ParseStream })(); - - const infoEntry = await new Promise((resolve, reject) => { - parser.on('entry', entry => { - if (entry.path === `./${INFO_FILE}`) { - resolve(entry); - reader.close(); - } else { - entry.resume(); - } - }); - parser.on('end', () => { - reject(new Error('cache archive did not contain build info')); - }); - parser.on('error', error => reject(error)); - reader.on('error', error => reject(error)); - - reader.pipe(parser); - }); - - const infoData = await new Promise((resolve, reject) => { - const chunks = new Array(); - infoEntry.on('data', chunk => chunks.push(chunk)); - infoEntry.on('end', () => resolve(Buffer.concat(chunks))); - infoEntry.on('error', error => reject(error)); - }); - - const info = JSON.parse(infoData.toString('utf8')); - - return info.trees; -} - async function getInputHashes(options: Options): Promise { const trees = []; for (const input of options.inputs) {