Finish ReadableArrayResponse changes

Signed-off-by: Sean Tan <seant@splunk.com>

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -156,8 +156,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
-      treeResponseFactory,
+      { s3, treeResponseFactory },
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -211,8 +210,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
-      treeResponseFactory,
+      { s3, treeResponseFactory },
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -273,8 +271,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
-      treeResponseFactory,
+      { s3, treeResponseFactory },
     );
     it('returns contents of an object in a bucket', async () => {
       const response = await awsS3UrlReader.readTree(

packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts

@@ -59,6 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
   async fromReadableArray(
     options: FromReadableArrayOptions,
   ): Promise<ReadTreeResponse> {
-    return new ReadableArrayResponse(options.stream, options.etag);
+    return new ReadableArrayResponse(
+      options.stream,
+      this.workDir,
+      options.etag,
+    );
   }
 }

packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts

@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 import { ReadableArrayResponse } from './ReadableArrayResponse';
-import path from 'path';
+import path, { resolve as resolvePath } from 'path';
+
 import { Readable } from 'stream';
-import fs from 'fs';
+import fs from 'fs-extra';
 
 const arr: Readable[] = [];
+const arr2: Readable[] = [];
 const file1 = path.resolve(
   'src',
   'reading',
@@ -33,14 +35,15 @@ const file2 = path.resolve(
   'awsS3',
   'awsS3-mock-object2.yaml',
 );
-const stream1 = fs.createReadStream(file1);
-const stream2 = fs.createReadStream(file2);
-arr.push(stream1);
-arr.push(stream2);
 
 describe('ReadableArrayResponse', () => {
   it('should read files', async () => {
-    const res = new ReadableArrayResponse(arr, 'etag');
+    const stream1 = fs.createReadStream(file1);
+    const stream2 = fs.createReadStream(file2);
+    arr.push(stream1);
+    arr.push(stream2);
+
+    const res = new ReadableArrayResponse(arr, '/tmp', 'etag');
     const files = await res.files();
 
     expect(files).toEqual([
@@ -59,4 +62,21 @@ describe('ReadableArrayResponse', () => {
       'site_name: Test2',
     ]);
   });
+
+  it('should extract entire archive into directory', async () => {
+    const stream1 = fs.createReadStream(file1);
+    const stream2 = fs.createReadStream(file2);
+
+    arr2.push(stream1);
+    arr2.push(stream2);
+
+    const res = new ReadableArrayResponse(arr2, '/tmp', 'etag');
+    const dir = await res.dir();
+    await expect(
+      fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'),
+    ).resolves.toBe('site_name: Test\n');
+    await expect(
+      fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'),
+    ).resolves.toBe('site_name: Test2\n');
+  });
 });

packages/backend-common/src/reading/tree/ReadableArrayResponse.ts

@@ -14,9 +14,21 @@
  * limitations under the License.
  */
 
+import concatStream from 'concat-stream';
+import platformPath, { basename } from 'path';
+
 import getRawBody from 'raw-body';
-import { Readable } from 'stream';
-import { ReadTreeResponse, ReadTreeResponseFile } from '../types';
+import fs from 'fs-extra';
+import { promisify } from 'util';
+import tar from 'tar';
+import { pipeline as pipelineCb, Readable } from 'stream';
+import {
+  ReadTreeResponse,
+  ReadTreeResponseFile,
+  ReadTreeResponseDirOptions,
+} from '../types';
+
+const pipeline = promisify(pipelineCb);
 
 /**
  * Wraps a array of Readable objects into a tree response reader.
@@ -26,6 +38,7 @@ export class ReadableArrayResponse implements ReadTreeResponse {
 
   constructor(
     private readonly stream: Readable[],
+    private readonly workDir: string,
     public readonly etag: string,
   ) {
     this.etag = etag;
@@ -56,11 +69,40 @@ export class ReadableArrayResponse implements ReadTreeResponse {
     return files;
   }
 
-  archive(): Promise<NodeJS.ReadableStream> {
-    throw new Error('Method not implemented.');
+  async archive(): Promise<NodeJS.ReadableStream> {
+    const tmpDir = await this.dir();
+
+    try {
+      const data = await new Promise<Buffer>(async resolve => {
+        await pipeline(
+          tar.create({ cwd: tmpDir }, ['']),
+          concatStream(resolve),
+        );
+      });
+      return Readable.from(data);
+    } finally {
+      await fs.remove(tmpDir);
+    }
   }
 
-  dir(): Promise<string> {
-    throw new Error('Method not implemented.');
+  async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
+    this.onlyOnce();
+
+    const dir =
+      options?.targetDir ??
+      (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
+
+    for (let i = 0; i < this.stream.length; i++) {
+      if (!(this.stream[i] as any).path.endsWith('/')) {
+        await pipeline(
+          this.stream[i],
+          fs.createWriteStream(
+            platformPath.join(dir, basename((this.stream[i] as any).path)),
+          ),
+        );
+      }
+    }
+
+    return dir;
   }
 }