Preparing for PR

Signed-off-by: Sean Tan <seant@splunk.com>

docs/integrations/aws-s3/readtree.md

@@ -0,0 +1,28 @@
+---
+id: read-tree
+title: AWS S3 Read Tree
+sidebar_label: Read Tree
+# prettier-ignore
+description: Discovering multiple catalog entities from an AWS S3 Bucket
+---
+
+The AWS S3 integration has a special processor for using AwsS3UrlReader's
+readTree() method. The method readTree() allows a user to retrieve more than one
+file hosted inside an S3 bucket. The processor passes a valid S3 URL and
+credentials to AwsS3UrlReader's readTree(). A user can supply a URL which points
+to root of a bucket or a subdirectory inside the bucket. This can be useful as
+an alternative to static locations or manually adding things to the catalog.
+
+To use the discovery processor, you'll need an AWS S3 integration
+[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and
+optionally a `roleArn`. Then you can add a location target to the catalog
+configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: aws-read-tree
+      target: https://sample-bucket.s3.us-east-2.amazonaws.com/
+```
+
+Note the `aws-read-tree` type, as this is not a regular `url` processor.

microsite/sidebars.json

@@ -107,7 +107,7 @@
       {
         "type": "subcategory",
         "label": "AWS S3",
-        "ids": ["integrations/aws-s3/locations"]
+        "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/readtree"]
       },
       {
         "type": "subcategory",

mkdocs.yml

@@ -81,6 +81,7 @@ nav:
       - Overview: 'integrations/index.md'
       - AWS S3:
           - Locations: 'integrations/aws-s3/locations.md'
+          - Read Tree: 'integrations/aws-s3/readtree.md'
       - Azure:
           - Locations: 'integrations/azure/locations.md'
           - Org Data: 'integrations/azure/org.md'

packages/backend-common/src/index.ts

@@ -23,6 +23,7 @@ export * from './logging';
 export * from './middleware';
 export * from './paths';
 export * from './reading';
+export * from './reading/tree';
 export * from './scm';
 export * from './service';
 export * from './util';

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -187,7 +187,6 @@ export class AwsS3UrlReader implements UrlReader {
         const { Contents, IsTruncated, NextContinuationToken } = await this.s3
           .listObjectsV2(params)
           .promise();
-
         const responses = await Promise.all(
           (Contents || []).map(({ Key }) => {
             const s3Response = this.s3