diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 9be6de7a30..736248b10b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -156,8 +156,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -211,8 +210,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -273,8 +271,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readTree( diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 19d7ed2fab..7d20385635 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -59,6 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { async fromReadableArray( options: FromReadableArrayOptions, ): Promise { - return new ReadableArrayResponse(options.stream, options.etag); + return new ReadableArrayResponse( + options.stream, + this.workDir, + options.etag, + ); } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index c2e3c26269..bf457f02e4 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ import { ReadableArrayResponse } from './ReadableArrayResponse'; -import path from 'path'; +import path, { resolve as resolvePath } from 'path'; + import { Readable } from 'stream'; -import fs from 'fs'; +import fs from 'fs-extra'; const arr: Readable[] = []; +const arr2: Readable[] = []; const file1 = path.resolve( 'src', 'reading', @@ -33,14 +35,15 @@ const file2 = path.resolve( 'awsS3', 'awsS3-mock-object2.yaml', ); -const stream1 = fs.createReadStream(file1); -const stream2 = fs.createReadStream(file2); -arr.push(stream1); -arr.push(stream2); describe('ReadableArrayResponse', () => { it('should read files', async () => { - const res = new ReadableArrayResponse(arr, 'etag'); + const stream1 = fs.createReadStream(file1); + const stream2 = fs.createReadStream(file2); + arr.push(stream1); + arr.push(stream2); + + const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -59,4 +62,21 @@ describe('ReadableArrayResponse', () => { 'site_name: Test2', ]); }); + + it('should extract entire archive into directory', async () => { + const stream1 = fs.createReadStream(file1); + const stream2 = fs.createReadStream(file2); + + arr2.push(stream1); + arr2.push(stream2); + + const res = new ReadableArrayResponse(arr2, '/tmp', 'etag'); + const dir = await res.dir(); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test2\n'); + }); }); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index 8b5d3ff0b5..bf74a19603 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -14,9 +14,21 @@ * limitations under the License. */ +import concatStream from 'concat-stream'; +import platformPath, { basename } from 'path'; + import getRawBody from 'raw-body'; -import { Readable } from 'stream'; -import { ReadTreeResponse, ReadTreeResponseFile } from '../types'; +import fs from 'fs-extra'; +import { promisify } from 'util'; +import tar from 'tar'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import { + ReadTreeResponse, + ReadTreeResponseFile, + ReadTreeResponseDirOptions, +} from '../types'; + +const pipeline = promisify(pipelineCb); /** * Wraps a array of Readable objects into a tree response reader. @@ -26,6 +38,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { constructor( private readonly stream: Readable[], + private readonly workDir: string, public readonly etag: string, ) { this.etag = etag; @@ -56,11 +69,40 @@ export class ReadableArrayResponse implements ReadTreeResponse { return files; } - archive(): Promise { - throw new Error('Method not implemented.'); + async archive(): Promise { + const tmpDir = await this.dir(); + + try { + const data = await new Promise(async resolve => { + await pipeline( + tar.create({ cwd: tmpDir }, ['']), + concatStream(resolve), + ); + }); + return Readable.from(data); + } finally { + await fs.remove(tmpDir); + } } - dir(): Promise { - throw new Error('Method not implemented.'); + async dir(options?: ReadTreeResponseDirOptions): Promise { + this.onlyOnce(); + + const dir = + options?.targetDir ?? + (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); + + for (let i = 0; i < this.stream.length; i++) { + if (!(this.stream[i] as any).path.endsWith('/')) { + await pipeline( + this.stream[i], + fs.createWriteStream( + platformPath.join(dir, basename((this.stream[i] as any).path)), + ), + ); + } + } + + return dir; } }