Finish AwsS3ReadTreeProcessor tests

Signed-off-by: Sean Tan <seant@splunk.com>

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -235,4 +235,55 @@ describe('AwsS3UrlReader', () => {
       );
     });
   });
+  describe('readTree', () => {
+    const object: aws.S3.Types.Object = {
+      Key: 'awsS3-mock-object.yaml',
+    };
+    const objectList: aws.S3.ObjectList = [object];
+    const output: aws.S3.Types.ListObjectsV2Output = {
+      Contents: objectList,
+    };
+    AWSMock.setSDKInstance(aws);
+    AWSMock.mock('S3', 'listObjectsV2', output);
+
+    AWSMock.mock(
+      'S3',
+      'getObject',
+      Buffer.from(
+        require('fs').readFileSync(
+          path.resolve(
+            'src',
+            'reading',
+            '__fixtures__',
+            'awsS3',
+            'awsS3-mock-object.yaml',
+          ),
+        ),
+      ),
+    );
+
+    const s3 = new aws.S3();
+    const awsS3UrlReader = new AwsS3UrlReader(
+      new AwsS3Integration(
+        readAwsS3IntegrationConfig(
+          new ConfigReader({
+            host: '.amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          }),
+        ),
+      ),
+      s3,
+      treeResponseFactory,
+    );
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.readTree(
+        'https://test.s3.us-east-2.amazonaws.com',
+      );
+      const files = await response.files();
+      const body = await files[0].content();
+
+      expect(body.toString()).toBe('site_name: Test\n');
+    });
+  });
 });

packages/backend-common/src/reading/index.ts

@@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader';
 export { BitbucketUrlReader } from './BitbucketUrlReader';
 export { GithubUrlReader } from './GithubUrlReader';
 export { GitlabUrlReader } from './GitlabUrlReader';
+export { AwsS3UrlReader } from './AwsS3UrlReader';
 export type {
   ReaderFactory,
   ReadTreeOptions,

plugins/catalog-backend/package.json

@@ -63,8 +63,14 @@
     "yup": "^0.29.3"
   },
   "devDependencies": {
+<<<<<<< HEAD
     "@backstage/backend-test-utils": "^0.1.6",
     "@backstage/cli": "^0.7.9",
+=======
+    "aws-sdk-mock": "^5.2.1",
+    "@backstage/backend-test-utils": "^0.1.5",
+    "@backstage/cli": "^0.7.8",
+>>>>>>> Finish AwsS3ReadTreeProcessor tests
     "@backstage/test-utils": "^0.1.17",
     "@types/core-js": "^2.5.4",
     "@types/git-url-parse": "^9.0.0",

plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  AwsS3UrlReader,
+  DefaultReadTreeResponseFactory,
+} from '@backstage/backend-common';
+
+import { ConfigReader } from '@backstage/config';
+import {
+  AwsS3Integration,
+  readAwsS3IntegrationConfig,
+} from '@backstage/integration';
+import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
+import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types';
+import { defaultEntityDataParser } from './util/parse';
+import AWSMock from 'aws-sdk-mock';
+import aws from 'aws-sdk';
+import path from 'path';
+
+const treeResponseFactory = DefaultReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});
+
+AWSMock.setSDKInstance(aws);
+const object: aws.S3.Types.Object = {
+  Key: 'awsS3-mock-object.yaml',
+};
+const objectList: aws.S3.ObjectList = [object];
+const output: aws.S3.Types.ListObjectsV2Output = {
+  Contents: objectList,
+};
+AWSMock.mock('S3', 'listObjectsV2', output);
+AWSMock.mock(
+  'S3',
+  'getObject',
+  Buffer.from(
+    require('fs').readFileSync(
+      path.resolve(
+        'src',
+        'ingestion',
+        'processors',
+        '__fixtures__',
+        'fileReaderProcessor',
+        'awsS3',
+        'awsS3-mock-object.yaml',
+      ),
+    ),
+  ),
+);
+
+const s3 = new aws.S3();
+const reader = new AwsS3UrlReader(
+  new AwsS3Integration(
+    readAwsS3IntegrationConfig(
+      new ConfigReader({
+        host: 'amazonaws.com',
+        accessKeyId: 'fake-access-key',
+        secretAccessKey: 'fake-secret-key',
+      }),
+    ),
+  ),
+  s3,
+  treeResponseFactory,
+);
+
+describe('readLocation', () => {
+  const processor = new AwsS3ReadTreeProcessor(reader);
+  const spec = {
+    type: 'aws-read-tree',
+    target: 'https://testbucket.s3.us-east-2.amazonaws.com',
+  };
+
+  it('should load from url', async () => {
+    const generated = (await new Promise<CatalogProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit, defaultEntityDataParser),
+    )) as CatalogProcessorEntityResult;
+    expect(generated.type).toBe('entity');
+    expect(generated.location).toEqual({
+      target: 'awsS3-mock-object.yaml',
+      type: 'aws-read-tree',
+    });
+    expect(generated.entity).toEqual({ site_name: 'Test' });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml

@@ -0,0 +1 @@
+site_name: Test