diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts index 57a490650c..6b8f29de8c 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts @@ -36,10 +36,51 @@ describe('ArchiveResponse', () => { mockFs.restore(); }); + it('should read files', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp'); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + { + path: 'docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + + it('should read files with filter', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path => + path.endsWith('.yml'), + ); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + ]); + const content = await files[0].content(); + expect(content.toString('utf8').trim()).toEqual('site_name: Test'); + }); + it('should read as archive and files', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo', '/tmp'); + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( @@ -51,11 +92,11 @@ describe('ArchiveResponse', () => { expect(files).toEqual([ { - path: './mkdocs.yml', + path: 'mkdocs.yml', content: expect.any(Function), }, { - path: './docs/index.md', + path: 'docs/index.md', content: expect.any(Function), }, ]); @@ -83,11 +124,29 @@ describe('ArchiveResponse', () => { it('should extract archive into directory with a subpath', async () => { const stream = fs.createReadStream('/test-archive.tar.gz'); - const res = new ArchiveResponse(stream, 'mock-repo/docs', '/tmp'); + const res = new ArchiveResponse(stream, 'mock-repo/docs/', '/tmp'); const dir = await res.dir(); + expect(dir).toMatch(/^\/tmp\/.*$/); await expect( fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), ).resolves.toBe('# Test\n'); }); + + it('should extract archive into directory with a subpath and filter', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path => + path.endsWith('.yml'), + ); + const dir = await res.dir({ targetDir: '/tmp' }); + + expect(dir).toBe('/tmp'); + await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( + true, + ); + await expect( + fs.pathExists(resolvePath(dir, 'docs/index.md')), + ).resolves.toBe(false); + }); }); diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.ts index 65ad8c18fe..83f6f6a8e5 100644 --- a/packages/backend-common/src/reading/tree/ArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.ts @@ -21,7 +21,7 @@ import fs from 'fs-extra'; import { Readable, pipeline as pipelineCb } from 'stream'; import { promisify } from 'util'; import concatStream from 'concat-stream'; -import { ReadTreeResponse, File } from '../types'; +import { ReadTreeResponse, File, ReadTreeResponseDirOptions } from '../types'; // Tar types for `Parse` is not a proper constructor, but it should be const TarParseStream = (Parse as unknown) as { new (): ParseStream }; @@ -38,7 +38,17 @@ export class ArchiveResponse implements ReadTreeResponse { private readonly stream: Readable, private readonly subPath: string, private readonly workDir: string = os.tmpdir(), - ) {} + private readonly filter?: (path: string) => boolean, + ) { + if (subPath) { + if (!subPath.endsWith('/')) { + throw new TypeError('ArchiveResponse subPath must end with a /'); + } + if (subPath.startsWith('/')) { + throw new TypeError('ArchiveResponse subPath must not start with a /'); + } + } + } // Make sure the input stream is only read once private onlyOnce() { @@ -60,14 +70,28 @@ export class ArchiveResponse implements ReadTreeResponse { return; } - const contentPromise = new Promise(async resolve => { + if (this.subPath) { + if (!entry.path.startsWith(this.subPath)) { + entry.resume(); + return; + } + } + + const path = this.subPath + ? entry.path.replace(this.subPath, '') + : entry.path; + if (this.filter) { + if (!this.filter(path)) { + entry.resume(); + return; + } + } + + const content = new Promise(async resolve => { await pipeline(entry, concatStream(resolve)); }); - files.push({ - path: entry.path, - content: () => contentPromise, - }); + files.push({ path, content: () => content }); entry.resume(); }); @@ -93,7 +117,7 @@ export class ArchiveResponse implements ReadTreeResponse { try { return await new Promise(async resolve => { await pipeline( - tar.create({ cwd: tmpDir }, ['.']), + tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); }); @@ -102,20 +126,30 @@ export class ArchiveResponse implements ReadTreeResponse { } } - async dir(outDir?: string): Promise { + async dir(options?: ReadTreeResponseDirOptions): Promise { this.onlyOnce(); const dir = - outDir ?? (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); + options?.targetDir ?? + (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); - const strip = this.subPath ? this.subPath.split('/').length : 0; + const strip = this.subPath ? this.subPath.split('/').length - 1 : 0; await pipeline( this.stream, tar.extract({ strip, cwd: dir, - filter: path => path.startsWith(this.subPath), + filter: path => { + if (this.subPath && !path.startsWith(this.subPath)) { + return false; + } + if (this.filter) { + const innerPath = path.split('/').slice(strip).join('/'); + return this.filter(innerPath); + } + return true; + }, }), ); diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 0cdf926720..5044cdabdc 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -25,6 +25,8 @@ type FromArchiveOptions = { stream: Readable; // If set, root of the tree will be set to the given path. Should not have a trailing `/`. path?: string; + // Filter passed on from the ReadTreeOptions + filter?: (path: string) => boolean; }; export class ReadTreeResponseFactory { @@ -42,6 +44,7 @@ export class ReadTreeResponseFactory { options.stream, options.path ?? '', this.workDir, + options.filter, ); } }