First go

Signed-off-by: Sean Tan <seant@splunk.com>

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -26,6 +26,7 @@ import {
 } from './types';
 import getRawBody from 'raw-body';
 import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+import { Readable } from 'stream';
 
 const parseURL = (
   url: string,
@@ -158,8 +159,62 @@ export class AwsS3UrlReader implements UrlReader {
     }
   }
 
-  async readTree(): Promise<ReadTreeResponse> {
-    throw new Error('AwsS3Reader does not implement readTree');
+  async readTree(url: string): Promise<ReadTreeResponse> {
+    try {
+      const { path, bucket, region } = parseURL(url);
+      aws.config.update({ region: region });
+
+      let moreKeys = true;
+      let awsS3Readables: Readable[] = [];
+      let continuationToken = '';
+
+      while (moreKeys) {
+        let params;
+        if (continuationToken === '') {
+          params = {
+            Bucket: bucket,
+            Prefix: path,
+          };
+        } else {
+          params = {
+            Bucket: bucket,
+            Prefix: path,
+            ContinuationToken: continuationToken,
+          };
+        }
+        const { Contents, IsTruncated, NextContinuationToken } = await this.s3
+          .listObjectsV2(params)
+          .promise();
+
+        const responses = await Promise.all(
+          (Contents || []).map(({ Key }) => {
+            const s3Response = this.s3
+              .getObject({ Bucket: bucket, Key: String(Key) })
+              .createReadStream();
+            Object.defineProperty(s3Response, 'path', {
+              value: String(Key),
+              writable: false,
+            });
+            return s3Response;
+          }),
+        );
+
+        if (IsTruncated) {
+          continuationToken = String(NextContinuationToken);
+        } else {
+          continuationToken = '';
+          moreKeys = false;
+        }
+        awsS3Readables = awsS3Readables.concat(responses);
+      }
+
+      return await this.treeResponseFactory.fromReadableArray({
+        stream: awsS3Readables,
+        etag: '',
+      });
+    } catch (e) {
+      throw new Error(`Could not retrieve file tree from S3: ${e.message}`);
+    }
   }
 
   async search(): Promise<SearchResponse> {

packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts

@@ -20,9 +20,11 @@ import {
   ReadTreeResponse,
   FromArchiveOptions,
   ReadTreeResponseFactory,
+  FromReadableArrayOptions,
 } from '../types';
 import { TarArchiveResponse } from './TarArchiveResponse';
 import { ZipArchiveResponse } from './ZipArchiveResponse';
+import { ReadableArrayResponse } from './ReadableArrayResponse';
 
 export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
   static create(options: { config: Config }): DefaultReadTreeResponseFactory {
@@ -53,4 +55,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
       options.filter,
     );
   }
+
+  async fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse> {
+    return new ReadableArrayResponse(options.stream, options.etag);
+  }
 }

packages/backend-common/src/reading/tree/ReadableArrayResponse.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import getRawBody from 'raw-body';
+import { Readable } from 'stream';
+import { ReadTreeResponse, ReadTreeResponseFile } from '../types';
+
+/**
+ * Wraps a array of Readable objects into a tree response reader.
+ */
+export class ReadableArrayResponse implements ReadTreeResponse {
+  private read = false;
+
+  constructor(
+    private readonly stream: Readable[],
+    public readonly etag: string,
+  ) {
+    this.etag = etag;
+  }
+
+  // Make sure the input stream is only read once
+  private onlyOnce() {
+    if (this.read) {
+      throw new Error('Response has already been read');
+    }
+    this.read = true;
+  }
+
+  async files(): Promise<ReadTreeResponseFile[]> {
+    this.onlyOnce();
+
+    const files = Array<ReadTreeResponseFile>();
+
+    for (let i = 0; i < this.stream.length; i++) {
+      if (!(this.stream[i] as any).path.endsWith('/')) {
+        files.push({
+          path: (this.stream[i] as any).path,
+          content: () => getRawBody(this.stream[i]),
+        });
+      }
+    }
+
+    return files;
+  }
+
+  archive(): Promise<NodeJS.ReadableStream> {
+    throw new Error('Method not implemented.');
+  }
+
+  dir(): Promise<string> {
+    throw new Error('Method not implemented.');
+  }
+}

packages/backend-common/src/reading/types.ts

@@ -167,9 +167,19 @@ export type FromArchiveOptions = {
   filter?: (path: string, info?: { size: number }) => boolean;
 };
 
+export type FromReadableArrayOptions = {
+  // An array of readable streams
+  stream: Readable[];
+  // etag of the file tree
+  etag: string;
+};
+
 export interface ReadTreeResponseFactory {
   fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
   fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
+  fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse>;
 }
 
 /**