diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md new file mode 100644 index 0000000000..7ae1fece63 --- /dev/null +++ b/docs/integrations/aws-s3/readtree.md @@ -0,0 +1,28 @@ +--- +id: read-tree +title: AWS S3 Read Tree +sidebar_label: Read Tree +# prettier-ignore +description: Discovering multiple catalog entities from an AWS S3 Bucket +--- + +The AWS S3 integration has a special processor for using AwsS3UrlReader's +readTree() method. The method readTree() allows a user to retrieve more than one +file hosted inside an S3 bucket. The processor passes a valid S3 URL and +credentials to AwsS3UrlReader's readTree(). A user can supply a URL which points +to root of a bucket or a subdirectory inside the bucket. This can be useful as +an alternative to static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need an AWS S3 integration +[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and +optionally a `roleArn`. Then you can add a location target to the catalog +configuration: + +```yaml +catalog: + locations: + - type: aws-read-tree + target: https://sample-bucket.s3.us-east-2.amazonaws.com/ +``` + +Note the `aws-read-tree` type, as this is not a regular `url` processor. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index d787fb28aa..3517c8d70c 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -107,7 +107,7 @@ { "type": "subcategory", "label": "AWS S3", - "ids": ["integrations/aws-s3/locations"] + "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/readtree"] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 73ad52341e..3b4dad765c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - Overview: 'integrations/index.md' - AWS S3: - Locations: 'integrations/aws-s3/locations.md' + - Read Tree: 'integrations/aws-s3/readtree.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index f2f38d9aab..5b37c2fb9f 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -23,6 +23,7 @@ export * from './logging'; export * from './middleware'; export * from './paths'; export * from './reading'; +export * from './reading/tree'; export * from './scm'; export * from './service'; export * from './util'; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index f7904ca70f..615a095e50 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -187,7 +187,6 @@ export class AwsS3UrlReader implements UrlReader { const { Contents, IsTruncated, NextContinuationToken } = await this.s3 .listObjectsV2(params) .promise(); - const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3