initial

Signed-off-by: Leon Stein <leons727@gmail.com>

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -332,4 +332,47 @@ describe('AwsS3UrlReader', () => {
       expect(body.toString().trim()).toBe('site_name: Test');
     });
   });
+
+  describe('readNonAwsHost', () => {
+    let awsS3UrlReader: AwsS3UrlReader;
+
+    beforeAll(() => {
+      AWSMock.setSDKInstance(aws);
+      AWSMock.mock(
+        'S3',
+        'getObject',
+        Buffer.from(
+          require('fs').readFileSync(
+            path.resolve(
+              __dirname,
+              '__fixtures__/awsS3/awsS3-mock-object.yaml',
+            ),
+          ),
+        ),
+      );
+
+      const s3 = new aws.S3();
+      awsS3UrlReader = new AwsS3UrlReader(
+        new AwsS3Integration(
+          readAwsS3IntegrationConfig(
+            new ConfigReader({
+              host: 'localhost:4566',
+              accessKeyId: 'fake-access-key',
+              secretAccessKey: 'fake-secret-key',
+              validateHost: false,
+              ssl: false,
+            }),
+          ),
+        ),
+        { s3, treeResponseFactory },
+      );
+    });
+
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.read(
+        'http://test-bucket.localhost:4566/awsS3-mock-object.yaml',
+      );
+      expect(response.toString().trim()).toBe('site_name: Test');
+    });
+  });
 });

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -27,12 +27,17 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
-import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+import {
+  AwsS3Integration,
+  ScmIntegrations,
+  AMAZON_AWS_HOST,
+} from '@backstage/integration';
 import { ForwardedError, NotModifiedError } from '@backstage/errors';
 import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
 
 const parseURL = (
   url: string,
+  validateHost: boolean,
 ): { path: string; bucket: string; region: string } => {
   let { host, pathname } = new URL(url);
 
@@ -47,15 +52,23 @@ const parseURL = (
    * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
    */
   const validHost = new RegExp(
-    /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
+    /^[a-z\d][a-z\d.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
   );
-  if (!validHost.test(host)) {
-    throw new Error(`not a valid AWS S3 URL: ${url}`);
-  }
 
-  const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
-  host = host.substring(bucket.length);
-  const [, , region, ,] = host.split('.');
+  let bucket;
+  let region;
+
+  if (!validHost.test(host)) {
+    if (validateHost) {
+      throw new Error(`not a valid AWS S3 URL: ${url}`);
+    }
+    [bucket] = host.split('.');
+    region = 'none';
+  } else {
+    [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
+    host = host.substring(bucket.length);
+    [, , region, ,] = host.split('.');
+  }
 
   return {
     path: pathname,
@@ -70,9 +83,17 @@ export class AwsS3UrlReader implements UrlReader {
 
     return integrations.awsS3.list().map(integration => {
       const creds = AwsS3UrlReader.buildCredentials(integration);
+      const defaultHost = integration.config.host === AMAZON_AWS_HOST;
+
       const s3 = new S3({
         apiVersion: '2006-03-01',
         credentials: creds,
+        endpoint: defaultHost
+          ? undefined
+          : `${integration.config.ssl ? 'https://' : 'http://'}${
+              integration.config.host
+            }`,
+        s3ForcePathStyle: !defaultHost,
       });
       const reader = new AwsS3UrlReader(integration, {
         s3,
@@ -138,7 +159,10 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadUrlOptions,
   ): Promise<ReadUrlResponse> {
     try {
-      const { path, bucket, region } = parseURL(url);
+      const { path, bucket, region } = parseURL(
+        url,
+        this.integration.config.validateHost,
+      );
       aws.config.update({ region: region });
 
       let params;
@@ -178,7 +202,10 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadTreeOptions,
   ): Promise<ReadTreeResponse> {
     try {
-      const { path, bucket, region } = parseURL(url);
+      const { path, bucket, region } = parseURL(
+        url,
+        this.integration.config.validateHost,
+      );
       const allObjects: ObjectList = [];
       const responses = [];
       let continuationToken: string | undefined;

packages/integration/src/awsS3/config.ts

@@ -17,7 +17,7 @@
 import { Config } from '@backstage/config';
 import { isValidHost } from '../helpers';
 
-const AMAZON_AWS_HOST = 'amazonaws.com';
+export const AMAZON_AWS_HOST = 'amazonaws.com';
 
 /**
  * The configuration parameters for a single AWS S3 provider.
@@ -28,7 +28,9 @@ export type AwsS3IntegrationConfig = {
   /**
    * The host of the target that this matches on, e.g. "amazonaws.com"
    *
-   * Currently only "amazonaws.com" is supported.
+   * If validateHost is true, "amazonaws.com" host is enforced. To test with localstack or similar AWS S3 emulators,
+   * setting validateHost to false allows hosts like "localhost:4566". Set ssl to false to access emulated S3
+   * endpoint over http (vs https). In that case, S3 urls would look like "http://<bucket>.localhost:4566/<path>".
    */
   host: string;
 
@@ -46,6 +48,16 @@ export type AwsS3IntegrationConfig = {
    * roleArn
    */
   roleArn?: string;
+
+  /**
+   * validateHost
+   */
+  validateHost: boolean;
+
+  /**
+   * ssl
+   */
+  ssl: boolean;
 };
 
 /**
@@ -62,6 +74,8 @@ export function readAwsS3IntegrationConfig(
   const accessKeyId = config.getOptionalString('accessKeyId');
   const secretAccessKey = config.getOptionalString('secretAccessKey');
   const roleArn = config.getOptionalString('roleArn');
+  const validateHost = config.getOptionalBoolean('validateHost') ?? true;
+  const ssl = config.getOptionalBoolean('ssl') ?? true;
 
   if (!isValidHost(host)) {
     throw new Error(
@@ -69,7 +83,7 @@ export function readAwsS3IntegrationConfig(
     );
   }
 
-  return { host, accessKeyId, secretAccessKey, roleArn };
+  return { host, accessKeyId, secretAccessKey, roleArn, validateHost, ssl };
 }
 
 /**
@@ -90,6 +104,8 @@ export function readAwsS3IntegrationConfigs(
   if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
     result.push({
       host: AMAZON_AWS_HOST,
+      validateHost: true,
+      ssl: true,
     });
   }
   return result;

packages/integration/src/awsS3/index.ts

@@ -17,5 +17,6 @@ export { AwsS3Integration } from './AwsS3Integration';
 export {
   readAwsS3IntegrationConfig,
   readAwsS3IntegrationConfigs,
+  AMAZON_AWS_HOST,
 } from './config';
 export type { AwsS3IntegrationConfig } from './config';