Merge work done on previous branch

Signed-off-by: Sean Tan <seant@splunk.com>

app-config.yaml

@@ -259,6 +259,8 @@ catalog:
     # Backstage example groups and users
     - type: file
       target: ../catalog-model/examples/acme-corp.yaml
+    - type: aws-read-tree
+      target: https://seant-splunk.s3.us-east-2.amazonaws.com
 
 scaffolder:
   # Use to customize default commit author info used when new components are created

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
 
+const treeResponseFactory = DefaultReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});
+
 describe('AwsS3UrlReader', () => {
   const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
     return AwsS3UrlReader.factory({
       config: new ConfigReader(config),
       logger: getVoidLogger(),
-      treeResponseFactory: DefaultReadTreeResponseFactory.create({
-        config: new ConfigReader({}),
-      }),
+      treeResponseFactory,
     });
   };
 
@@ -154,6 +156,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ),
       s3,
+      treeResponseFactory,
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -207,6 +210,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ),
       s3,
+      treeResponseFactory,
     );
 
     it('returns contents of an object in a bucket', async () => {

packages/backend/src/plugins/catalog.ts

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor';
 import {
   CatalogBuilder,
   createRouter,
@@ -25,6 +25,7 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader));
   const {
     entitiesCatalog,
     locationAnalyzer,

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -19,6 +19,7 @@ import * as results from './results';
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+export { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
 export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';