Begin changes on PR

Signed-off-by: Sean Tan <seant@splunk.com>

docs/integrations/aws-s3/readtree.md

@@ -21,8 +21,8 @@ configuration:
 ```yaml
 catalog:
   locations:
-    - type: aws-read-tree
+    - type: s3-bucket
       target: https://sample-bucket.s3.us-east-2.amazonaws.com/
 ```
 
-Note the `aws-read-tree` type, as this is not a regular `url` processor.
+Note the `s3-bucket` type, as this is not a regular `url` processor.

packages/backend-common/src/index.ts

@@ -23,7 +23,6 @@ export * from './logging';
 export * from './middleware';
 export * from './paths';
 export * from './reading';
-export * from './reading/tree';
 export * from './scm';
 export * from './service';
 export * from './util';

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -72,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader {
         apiVersion: '2006-03-01',
         credentials: creds,
       });
-      const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory);
+      const reader = new AwsS3UrlReader(integration, {
+        s3,
+        treeResponseFactory,
+      });
       const predicate = (url: URL) =>
         url.host.endsWith(integration.config.host);
       return { reader, predicate };
@@ -81,8 +84,10 @@ export class AwsS3UrlReader implements UrlReader {
 
   constructor(
     private readonly integration: AwsS3Integration,
-    private readonly s3: S3,
-    private readonly treeResponseFactory: ReadTreeResponseFactory,
+    private readonly deps: {
+      s3: S3;
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
   ) {}
 
   /**
@@ -148,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader {
         };
       }
 
-      const response = this.s3.getObject(params);
+      const response = this.deps.s3.getObject(params);
       const buffer = await getRawBody(response.createReadStream());
       const etag = (await response.promise()).ETag;
 
@@ -184,19 +189,11 @@ export class AwsS3UrlReader implements UrlReader {
             ContinuationToken: continuationToken,
           };
         }
-        const { Contents, IsTruncated, NextContinuationToken } = await this.s3
-          .listObjectsV2(params)
-          .promise();
-<<<<<<< HEAD
-<<<<<<< HEAD
-=======
-
->>>>>>> First go
-=======
->>>>>>> Preparing for PR
+        const { Contents, IsTruncated, NextContinuationToken } =
+          await this.deps.s3.listObjectsV2(params).promise();
         const responses = await Promise.all(
           (Contents || []).map(({ Key }) => {
-            const s3Response = this.s3
+            const s3Response = this.deps.s3
               .getObject({ Bucket: bucket, Key: String(Key) })
               .createReadStream();
             Object.defineProperty(s3Response, 'path', {
@@ -216,7 +213,7 @@ export class AwsS3UrlReader implements UrlReader {
         awsS3Readables = awsS3Readables.concat(responses);
       }
 
-      return await this.treeResponseFactory.fromReadableArray({
+      return await this.deps.treeResponseFactory.fromReadableArray({
         stream: awsS3Readables,
         etag: '',
       });

plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts

@@ -13,16 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import {
-  AwsS3UrlReader,
-  DefaultReadTreeResponseFactory,
-} from '@backstage/backend-common';
-
+import { getVoidLogger, UrlReaders } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
-import {
-  AwsS3Integration,
-  readAwsS3IntegrationConfig,
-} from '@backstage/integration';
 import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
 import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types';
 import { defaultEntityDataParser } from './util/parse';
@@ -30,10 +22,6 @@ import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
 
-const treeResponseFactory = DefaultReadTreeResponseFactory.create({
-  config: new ConfigReader({}),
-});
-
 AWSMock.setSDKInstance(aws);
 const object: aws.S3.Types.Object = {
   Key: 'awsS3-mock-object.yaml',
@@ -61,25 +49,18 @@ AWSMock.mock(
   ),
 );
 
-const s3 = new aws.S3();
-const reader = new AwsS3UrlReader(
-  new AwsS3Integration(
-    readAwsS3IntegrationConfig(
-      new ConfigReader({
-        host: 'amazonaws.com',
-        accessKeyId: 'fake-access-key',
-        secretAccessKey: 'fake-secret-key',
-      }),
-    ),
-  ),
-  s3,
-  treeResponseFactory,
-);
+const logger = getVoidLogger();
+const reader = UrlReaders.default({
+  logger,
+  config: new ConfigReader({
+    backend: { reading: { allow: [{ host: 'localhost' }] } },
+  }),
+});
 
 describe('readLocation', () => {
   const processor = new AwsS3ReadTreeProcessor(reader);
   const spec = {
-    type: 'aws-read-tree',
+    type: 's3-bucket',
     target: 'https://testbucket.s3.us-east-2.amazonaws.com',
   };
 
@@ -90,7 +71,7 @@ describe('readLocation', () => {
     expect(generated.type).toBe('entity');
     expect(generated.location).toEqual({
       target: 'awsS3-mock-object.yaml',
-      type: 'aws-read-tree',
+      type: 's3-bucket',
     });
     expect(generated.entity).toEqual({ site_name: 'Test' });
   });

plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts

@@ -32,7 +32,7 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor {
     emit: CatalogProcessorEmit,
     parser: CatalogProcessorParser,
   ): Promise<boolean> {
-    if (location.type !== 'aws-read-tree') {
+    if (location.type !== 's3-bucket') {
       return false;
     }