From 0058a553ce22c28cf4e0f8fd5b8cb3d39d057e1f Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Fri, 13 Aug 2021 17:24:14 -0700
Subject: [PATCH] Merge work done on previous branch Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 app-config.yaml                                        |  2 ++
 .../backend-common/src/reading/AwsS3UrlReader.test.ts  | 10 +++++++---
 packages/backend/src/plugins/catalog.ts                |  3 ++-
 .../catalog-backend/src/ingestion/processors/index.ts  |  1 +
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 9f365becef..fcfd92a66f 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -259,6 +259,8 @@ catalog:
     # Backstage example groups and users
     - type: file
       target: ../catalog-model/examples/acme-corp.yaml
+    - type: aws-read-tree
+      target: https://seant-splunk.s3.us-east-2.amazonaws.com
 
 scaffolder:
   # Use to customize default commit author info used when new components are created
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index 084f7e3e13..78bfdbb0e2 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
 
+const treeResponseFactory = DefaultReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});
+
 describe('AwsS3UrlReader', () => {
   const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
     return AwsS3UrlReader.factory({
       config: new ConfigReader(config),
       logger: getVoidLogger(),
-      treeResponseFactory: DefaultReadTreeResponseFactory.create({
-        config: new ConfigReader({}),
-      }),
+      treeResponseFactory,
     });
   };
 
@@ -154,6 +156,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ),
       s3,
+      treeResponseFactory,
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -207,6 +210,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ),
       s3,
+      treeResponseFactory,
     );
 
     it('returns contents of an object in a bucket', async () => {
diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 055595dcb5..9c94b3851e 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor';
 import {
   CatalogBuilder,
   createRouter,
@@ -25,6 +25,7 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader));
   const {
     entitiesCatalog,
     locationAnalyzer,
diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts
index fe4d374829..1cfb9980f2 100644
--- a/plugins/catalog-backend/src/ingestion/processors/index.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/index.ts
@@ -19,6 +19,7 @@ import * as results from './results';
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+export { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
 export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';