From e74854410d81ec7c1439fcf9840054ff9654152d Mon Sep 17 00:00:00 2001
From: Leon Stein <leon.stein@thetradedesk.com>
Date: Thu, 23 Dec 2021 16:36:35 -0500
Subject: [PATCH 01/10] initial

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 .../src/reading/AwsS3UrlReader.test.ts        | 43 +++++++++++++++++
 .../src/reading/AwsS3UrlReader.ts             | 47 +++++++++++++++----
 packages/integration/src/awsS3/config.ts      | 22 +++++++--
 packages/integration/src/awsS3/index.ts       |  1 +
 4 files changed, 100 insertions(+), 13 deletions(-)

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index a3ae8b647f..81d34c29a4 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -332,4 +332,47 @@ describe('AwsS3UrlReader', () => {
       expect(body.toString().trim()).toBe('site_name: Test');
     });
   });
+
+  describe('readNonAwsHost', () => {
+    let awsS3UrlReader: AwsS3UrlReader;
+
+    beforeAll(() => {
+      AWSMock.setSDKInstance(aws);
+      AWSMock.mock(
+        'S3',
+        'getObject',
+        Buffer.from(
+          require('fs').readFileSync(
+            path.resolve(
+              __dirname,
+              '__fixtures__/awsS3/awsS3-mock-object.yaml',
+            ),
+          ),
+        ),
+      );
+
+      const s3 = new aws.S3();
+      awsS3UrlReader = new AwsS3UrlReader(
+        new AwsS3Integration(
+          readAwsS3IntegrationConfig(
+            new ConfigReader({
+              host: 'localhost:4566',
+              accessKeyId: 'fake-access-key',
+              secretAccessKey: 'fake-secret-key',
+              validateHost: false,
+              ssl: false,
+            }),
+          ),
+        ),
+        { s3, treeResponseFactory },
+      );
+    });
+
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.read(
+        'http://test-bucket.localhost:4566/awsS3-mock-object.yaml',
+      );
+      expect(response.toString().trim()).toBe('site_name: Test');
+    });
+  });
 });
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index 0d806fcfaa..055dd9384e 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -27,12 +27,17 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
-import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+import {
+  AwsS3Integration,
+  ScmIntegrations,
+  AMAZON_AWS_HOST,
+} from '@backstage/integration';
 import { ForwardedError, NotModifiedError } from '@backstage/errors';
 import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
 
 const parseURL = (
   url: string,
+  validateHost: boolean,
 ): { path: string; bucket: string; region: string } => {
   let { host, pathname } = new URL(url);
 
@@ -47,15 +52,23 @@ const parseURL = (
    * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
    */
   const validHost = new RegExp(
-    /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
+    /^[a-z\d][a-z\d.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
   );
-  if (!validHost.test(host)) {
-    throw new Error(`not a valid AWS S3 URL: ${url}`);
-  }
 
-  const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
-  host = host.substring(bucket.length);
-  const [, , region, ,] = host.split('.');
+  let bucket;
+  let region;
+
+  if (!validHost.test(host)) {
+    if (validateHost) {
+      throw new Error(`not a valid AWS S3 URL: ${url}`);
+    }
+    [bucket] = host.split('.');
+    region = 'none';
+  } else {
+    [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
+    host = host.substring(bucket.length);
+    [, , region, ,] = host.split('.');
+  }
 
   return {
     path: pathname,
@@ -70,9 +83,17 @@ export class AwsS3UrlReader implements UrlReader {
 
     return integrations.awsS3.list().map(integration => {
       const creds = AwsS3UrlReader.buildCredentials(integration);
+      const defaultHost = integration.config.host === AMAZON_AWS_HOST;
+
       const s3 = new S3({
         apiVersion: '2006-03-01',
         credentials: creds,
+        endpoint: defaultHost
+          ? undefined
+          : `${integration.config.ssl ? 'https://' : 'http://'}${
+              integration.config.host
+            }`,
+        s3ForcePathStyle: !defaultHost,
       });
       const reader = new AwsS3UrlReader(integration, {
         s3,
@@ -138,7 +159,10 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadUrlOptions,
   ): Promise<ReadUrlResponse> {
     try {
-      const { path, bucket, region } = parseURL(url);
+      const { path, bucket, region } = parseURL(
+        url,
+        this.integration.config.validateHost,
+      );
       aws.config.update({ region: region });
 
       let params;
@@ -178,7 +202,10 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadTreeOptions,
   ): Promise<ReadTreeResponse> {
     try {
-      const { path, bucket, region } = parseURL(url);
+      const { path, bucket, region } = parseURL(
+        url,
+        this.integration.config.validateHost,
+      );
       const allObjects: ObjectList = [];
       const responses = [];
       let continuationToken: string | undefined;
diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts
index 6b5a23c85c..8c8c057e58 100644
--- a/packages/integration/src/awsS3/config.ts
+++ b/packages/integration/src/awsS3/config.ts
@@ -17,7 +17,7 @@
 import { Config } from '@backstage/config';
 import { isValidHost } from '../helpers';
 
-const AMAZON_AWS_HOST = 'amazonaws.com';
+export const AMAZON_AWS_HOST = 'amazonaws.com';
 
 /**
  * The configuration parameters for a single AWS S3 provider.
@@ -28,7 +28,9 @@ export type AwsS3IntegrationConfig = {
   /**
    * The host of the target that this matches on, e.g. "amazonaws.com"
    *
-   * Currently only "amazonaws.com" is supported.
+   * If validateHost is true, "amazonaws.com" host is enforced. To test with localstack or similar AWS S3 emulators,
+   * setting validateHost to false allows hosts like "localhost:4566". Set ssl to false to access emulated S3
+   * endpoint over http (vs https). In that case, S3 urls would look like "http://<bucket>.localhost:4566/<path>".
    */
   host: string;
 
@@ -46,6 +48,16 @@ export type AwsS3IntegrationConfig = {
    * roleArn
    */
   roleArn?: string;
+
+  /**
+   * validateHost
+   */
+  validateHost: boolean;
+
+  /**
+   * ssl
+   */
+  ssl: boolean;
 };
 
 /**
@@ -62,6 +74,8 @@ export function readAwsS3IntegrationConfig(
   const accessKeyId = config.getOptionalString('accessKeyId');
   const secretAccessKey = config.getOptionalString('secretAccessKey');
   const roleArn = config.getOptionalString('roleArn');
+  const validateHost = config.getOptionalBoolean('validateHost') ?? true;
+  const ssl = config.getOptionalBoolean('ssl') ?? true;
 
   if (!isValidHost(host)) {
     throw new Error(
@@ -69,7 +83,7 @@ export function readAwsS3IntegrationConfig(
     );
   }
 
-  return { host, accessKeyId, secretAccessKey, roleArn };
+  return { host, accessKeyId, secretAccessKey, roleArn, validateHost, ssl };
 }
 
 /**
@@ -90,6 +104,8 @@ export function readAwsS3IntegrationConfigs(
   if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
     result.push({
       host: AMAZON_AWS_HOST,
+      validateHost: true,
+      ssl: true,
     });
   }
   return result;
diff --git a/packages/integration/src/awsS3/index.ts b/packages/integration/src/awsS3/index.ts
index 1ae3964ec1..de665e2af0 100644
--- a/packages/integration/src/awsS3/index.ts
+++ b/packages/integration/src/awsS3/index.ts
@@ -17,5 +17,6 @@ export { AwsS3Integration } from './AwsS3Integration';
 export {
   readAwsS3IntegrationConfig,
   readAwsS3IntegrationConfigs,
+  AMAZON_AWS_HOST,
 } from './config';
 export type { AwsS3IntegrationConfig } from './config';

From 995e4c7d9d8dca9b9cc2892b31f2a000b3bd5874 Mon Sep 17 00:00:00 2001
From: Leon Stein <leon.stein@thetradedesk.com>
Date: Fri, 24 Dec 2021 16:11:38 -0500
Subject: [PATCH 02/10] added changeset

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 .changeset/proud-rocks-lick.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/proud-rocks-lick.md

diff --git a/.changeset/proud-rocks-lick.md b/.changeset/proud-rocks-lick.md
new file mode 100644
index 0000000000..a3ce9fbf0b
--- /dev/null
+++ b/.changeset/proud-rocks-lick.md
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Added support for non-"amazonaws.com" hosts (for example when testing with localstack) in AwsS3UrlReader.

From bc78cc3dca4f8662bc122e8dd4d4875d43bc7269 Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Tue, 28 Dec 2021 10:23:53 -0500
Subject: [PATCH 03/10] use endpoint/s3ForcePathStyle instead of
 validateHost/ssl per PR feedback

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 .../src/reading/AwsS3UrlReader.test.ts        | 10 +--
 .../src/reading/AwsS3UrlReader.ts             | 72 ++++++++-------
 packages/integration/config.d.ts              |  5 --
 packages/integration/src/awsS3/config.ts      | 87 +++++++++++--------
 4 files changed, 99 insertions(+), 75 deletions(-)

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index 81d34c29a4..2911699a5d 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -178,7 +178,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ).rejects.toThrow(
         Error(
-          `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
+          `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
         ),
       );
     });
@@ -234,7 +234,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ).rejects.toThrow(
         Error(
-          `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
+          `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
         ),
       );
     });
@@ -359,8 +359,8 @@ describe('AwsS3UrlReader', () => {
               host: 'localhost:4566',
               accessKeyId: 'fake-access-key',
               secretAccessKey: 'fake-secret-key',
-              validateHost: false,
-              ssl: false,
+              endpoint: 'http://localhost:4566',
+              s3ForcePathStyle: true,
             }),
           ),
         ),
@@ -370,7 +370,7 @@ describe('AwsS3UrlReader', () => {
 
     it('returns contents of an object in a bucket', async () => {
       const response = await awsS3UrlReader.read(
-        'http://test-bucket.localhost:4566/awsS3-mock-object.yaml',
+        'http://localhost:4566/test-bucket/awsS3-mock-object.yaml',
       );
       expect(response.toString().trim()).toBe('site_name: Test');
     });
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index 055dd9384e..b6655ecd0e 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -31,13 +31,14 @@ import {
   AwsS3Integration,
   ScmIntegrations,
   AMAZON_AWS_HOST,
+  AwsS3IntegrationConfig,
 } from '@backstage/integration';
 import { ForwardedError, NotModifiedError } from '@backstage/errors';
 import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
 
 const parseURL = (
   url: string,
-  validateHost: boolean,
+  config: AwsS3IntegrationConfig,
 ): { path: string; bucket: string; region: string } => {
   let { host, pathname } = new URL(url);
 
@@ -47,27 +48,44 @@ const parseURL = (
    */
   pathname = pathname.substr(1);
 
-  /**
-   * Checks that the given URL is a valid S3 object url.
-   * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
-   */
-  const validHost = new RegExp(
-    /^[a-z\d][a-z\d.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
-  );
-
   let bucket;
   let region;
 
-  if (!validHost.test(host)) {
-    if (validateHost) {
-      throw new Error(`not a valid AWS S3 URL: ${url}`);
+  /**
+   * Path style URLs: https://s3.Region.amazonaws.com/bucket-name/key-name
+   * Virtual hosted style URLs: https://bucket-name.s3.Region.amazonaws.com/key-name
+   * See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access
+   */
+  if (config.s3ForcePathStyle) {
+    if (pathname.indexOf('/') < 0) {
+      throw new Error(
+        `invalid path-style AWS S3 URL, ${url} does not contain bucket in the path`,
+      );
+    }
+    [bucket] = pathname.split('/');
+    pathname = pathname.substr(bucket.length + 1);
+  } else {
+    if (host.indexOf('.') < 0) {
+      throw new Error(
+        `invalid virtual hosted-style AWS S3 URL, ${url} does not contain bucket prefix in the host`,
+      );
     }
     [bucket] = host.split('.');
-    region = 'none';
+    host = host.substr(bucket.length + 1);
+  }
+
+  // Only extract region from *.amazonaws.com hosts
+  if (config.host === AMAZON_AWS_HOST) {
+    // At this point bucket prefix is removed from host for virtual hosted URLs
+    const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws.com$/);
+    if (!match) {
+      throw new Error(
+        `invalid AWS S3 URL, cannot parse region from host in ${url}`,
+      );
+    }
+    region = match[1];
   } else {
-    [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
-    host = host.substring(bucket.length);
-    [, , region, ,] = host.split('.');
+    region = '';
   }
 
   return {
@@ -83,17 +101,15 @@ export class AwsS3UrlReader implements UrlReader {
 
     return integrations.awsS3.list().map(integration => {
       const creds = AwsS3UrlReader.buildCredentials(integration);
-      const defaultHost = integration.config.host === AMAZON_AWS_HOST;
 
       const s3 = new S3({
         apiVersion: '2006-03-01',
         credentials: creds,
-        endpoint: defaultHost
-          ? undefined
-          : `${integration.config.ssl ? 'https://' : 'http://'}${
-              integration.config.host
-            }`,
-        s3ForcePathStyle: !defaultHost,
+        endpoint:
+          integration.config.host === AMAZON_AWS_HOST
+            ? undefined
+            : integration.config.endpoint,
+        s3ForcePathStyle: integration.config.s3ForcePathStyle,
       });
       const reader = new AwsS3UrlReader(integration, {
         s3,
@@ -159,10 +175,7 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadUrlOptions,
   ): Promise<ReadUrlResponse> {
     try {
-      const { path, bucket, region } = parseURL(
-        url,
-        this.integration.config.validateHost,
-      );
+      const { path, bucket, region } = parseURL(url, this.integration.config);
       aws.config.update({ region: region });
 
       let params;
@@ -202,10 +215,7 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadTreeOptions,
   ): Promise<ReadTreeResponse> {
     try {
-      const { path, bucket, region } = parseURL(
-        url,
-        this.integration.config.validateHost,
-      );
+      const { path, bucket, region } = parseURL(url, this.integration.config);
       const allObjects: ObjectList = [];
       const responses = [];
       let continuationToken: string | undefined;
diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts
index 90cc10f77b..85a9c78cf7 100644
--- a/packages/integration/config.d.ts
+++ b/packages/integration/config.d.ts
@@ -166,11 +166,6 @@ export interface Config {
 
     /** Integration configuration for AWS S3 Service */
     awsS3?: Array<{
-      /**
-       * The host of the target that this matches on, e.g. "amazonaws.com".
-       * @visibility frontend
-       */
-      host: string;
       /**
        * Account access key used to authenticate requests.
        * @visibility backend
diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts
index 8c8c057e58..ccbc09a45a 100644
--- a/packages/integration/src/awsS3/config.ts
+++ b/packages/integration/src/awsS3/config.ts
@@ -15,7 +15,6 @@
  */
 
 import { Config } from '@backstage/config';
-import { isValidHost } from '../helpers';
 
 export const AMAZON_AWS_HOST = 'amazonaws.com';
 
@@ -26,38 +25,40 @@ export const AMAZON_AWS_HOST = 'amazonaws.com';
  */
 export type AwsS3IntegrationConfig = {
   /**
-   * The host of the target that this matches on, e.g. "amazonaws.com"
-   *
-   * If validateHost is true, "amazonaws.com" host is enforced. To test with localstack or similar AWS S3 emulators,
-   * setting validateHost to false allows hosts like "localhost:4566". Set ssl to false to access emulated S3
-   * endpoint over http (vs https). In that case, S3 urls would look like "http://<bucket>.localhost:4566/<path>".
+   * Host, derived from endpoint, and defaults to amazonaws.com
    */
   host: string;
 
   /**
-   * accessKeyId
+   * (Optional) AWS Endpoint.
+   * The endpoint URI to send requests to. The default endpoint is built from the configured region.
+   * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
+   *
+   * Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566
+   */
+  endpoint?: string;
+
+  /**
+   * (Optional) Whether to use path style URLs when communicating with S3.
+   * Defaults to false.
+   * This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used.
+   */
+  s3ForcePathStyle?: boolean;
+
+  /**
+   * (Optional) User access key id
    */
   accessKeyId?: string;
 
   /**
-   * secretAccessKey
+   * (Optional) User secret access key
    */
   secretAccessKey?: string;
 
   /**
-   * roleArn
+   * (Optional) ARN of role to be assumed
    */
   roleArn?: string;
-
-  /**
-   * validateHost
-   */
-  validateHost: boolean;
-
-  /**
-   * ssl
-   */
-  ssl: boolean;
 };
 
 /**
@@ -70,20 +71,42 @@ export type AwsS3IntegrationConfig = {
 export function readAwsS3IntegrationConfig(
   config: Config,
 ): AwsS3IntegrationConfig {
-  const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST;
+  const endpoint = config.getOptionalString('endpoint');
+  const s3ForcePathStyle =
+    config.getOptionalBoolean('s3ForcePathStyle') ?? false;
+  let host;
+  let pathname;
+  if (endpoint) {
+    try {
+      const url = new URL(endpoint);
+      host = url.host;
+      pathname = url.pathname;
+    } catch {
+      throw new Error(
+        `Invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`,
+      );
+    }
+    if (pathname !== '/') {
+      throw new Error(
+        `Invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`,
+      );
+    }
+  } else {
+    host = AMAZON_AWS_HOST;
+  }
+
   const accessKeyId = config.getOptionalString('accessKeyId');
   const secretAccessKey = config.getOptionalString('secretAccessKey');
   const roleArn = config.getOptionalString('roleArn');
-  const validateHost = config.getOptionalBoolean('validateHost') ?? true;
-  const ssl = config.getOptionalBoolean('ssl') ?? true;
 
-  if (!isValidHost(host)) {
-    throw new Error(
-      `Invalid awsS3 integration config, '${host}' is not a valid host`,
-    );
-  }
-
-  return { host, accessKeyId, secretAccessKey, roleArn, validateHost, ssl };
+  return {
+    host,
+    endpoint,
+    s3ForcePathStyle,
+    accessKeyId,
+    secretAccessKey,
+    roleArn,
+  };
 }
 
 /**
@@ -102,11 +125,7 @@ export function readAwsS3IntegrationConfigs(
   // If no explicit amazonaws.com integration was added, put one in the list as
   // a convenience
   if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
-    result.push({
-      host: AMAZON_AWS_HOST,
-      validateHost: true,
-      ssl: true,
-    });
+    result.push({ host: AMAZON_AWS_HOST });
   }
   return result;
 }

From 60dee60d9becb2f819d91f4afac4e688a9b1fb3b Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Tue, 28 Dec 2021 10:29:53 -0500
Subject: [PATCH 04/10] update config schema

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 packages/integration/config.d.ts | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts
index 85a9c78cf7..8873f0f220 100644
--- a/packages/integration/config.d.ts
+++ b/packages/integration/config.d.ts
@@ -166,6 +166,24 @@ export interface Config {
 
     /** Integration configuration for AWS S3 Service */
     awsS3?: Array<{
+      /**
+       * AWS Endpoint.
+       * The endpoint URI to send requests to. The default endpoint is built from the configured region.
+       * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
+       *
+       * Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566
+       * @visibility frontend
+       */
+      endpoint?: string;
+
+      /**
+       * Whether to use path style URLs when communicating with S3.
+       * Defaults to false.
+       * This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used.
+       * @visibility frontend
+       */
+      s3ForcePathStyle?: boolean;
+
       /**
        * Account access key used to authenticate requests.
        * @visibility backend

From e662ee528be2906708f58f178f2b87bb9594b2ed Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Tue, 28 Dec 2021 11:42:23 -0500
Subject: [PATCH 05/10] remove AMAZON_AWS_HOST export, fix regex per review
 feedback

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 packages/backend-common/src/reading/AwsS3UrlReader.ts | 10 +++-------
 packages/integration/src/awsS3/config.ts              |  6 +++---
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index b6655ecd0e..1fca72e91c 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -30,7 +30,6 @@ import getRawBody from 'raw-body';
 import {
   AwsS3Integration,
   ScmIntegrations,
-  AMAZON_AWS_HOST,
   AwsS3IntegrationConfig,
 } from '@backstage/integration';
 import { ForwardedError, NotModifiedError } from '@backstage/errors';
@@ -75,9 +74,9 @@ const parseURL = (
   }
 
   // Only extract region from *.amazonaws.com hosts
-  if (config.host === AMAZON_AWS_HOST) {
+  if (config.host === 'amazonaws.com') {
     // At this point bucket prefix is removed from host for virtual hosted URLs
-    const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws.com$/);
+    const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws\.com$/);
     if (!match) {
       throw new Error(
         `invalid AWS S3 URL, cannot parse region from host in ${url}`,
@@ -105,10 +104,7 @@ export class AwsS3UrlReader implements UrlReader {
       const s3 = new S3({
         apiVersion: '2006-03-01',
         credentials: creds,
-        endpoint:
-          integration.config.host === AMAZON_AWS_HOST
-            ? undefined
-            : integration.config.endpoint,
+        endpoint: integration.config.endpoint,
         s3ForcePathStyle: integration.config.s3ForcePathStyle,
       });
       const reader = new AwsS3UrlReader(integration, {
diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts
index ccbc09a45a..48a9f28c9f 100644
--- a/packages/integration/src/awsS3/config.ts
+++ b/packages/integration/src/awsS3/config.ts
@@ -16,7 +16,7 @@
 
 import { Config } from '@backstage/config';
 
-export const AMAZON_AWS_HOST = 'amazonaws.com';
+const AMAZON_AWS_HOST = 'amazonaws.com';
 
 /**
  * The configuration parameters for a single AWS S3 provider.
@@ -83,12 +83,12 @@ export function readAwsS3IntegrationConfig(
       pathname = url.pathname;
     } catch {
       throw new Error(
-        `Invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`,
+        `invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`,
       );
     }
     if (pathname !== '/') {
       throw new Error(
-        `Invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`,
+        `invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`,
       );
     }
   } else {

From c2d09d8c1789da6b07198856045a91716a3c1fdc Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Tue, 28 Dec 2021 11:49:10 -0500
Subject: [PATCH 06/10] cleanup

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 packages/integration/src/awsS3/config.ts | 4 +++-
 packages/integration/src/awsS3/index.ts  | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts
index 48a9f28c9f..45cbdbe210 100644
--- a/packages/integration/src/awsS3/config.ts
+++ b/packages/integration/src/awsS3/config.ts
@@ -125,7 +125,9 @@ export function readAwsS3IntegrationConfigs(
   // If no explicit amazonaws.com integration was added, put one in the list as
   // a convenience
   if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
-    result.push({ host: AMAZON_AWS_HOST });
+    result.push({
+      host: AMAZON_AWS_HOST,
+    });
   }
   return result;
 }
diff --git a/packages/integration/src/awsS3/index.ts b/packages/integration/src/awsS3/index.ts
index de665e2af0..1ae3964ec1 100644
--- a/packages/integration/src/awsS3/index.ts
+++ b/packages/integration/src/awsS3/index.ts
@@ -17,6 +17,5 @@ export { AwsS3Integration } from './AwsS3Integration';
 export {
   readAwsS3IntegrationConfig,
   readAwsS3IntegrationConfigs,
-  AMAZON_AWS_HOST,
 } from './config';
 export type { AwsS3IntegrationConfig } from './config';

From 7615549bf5a8c246e0144e860f6ea23515232a39 Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Tue, 28 Dec 2021 17:35:14 -0500
Subject: [PATCH 07/10] update api-reports

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 .changeset/proud-rocks-lick.md     | 2 +-
 packages/integration/api-report.md | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.changeset/proud-rocks-lick.md b/.changeset/proud-rocks-lick.md
index a3ce9fbf0b..ad33c410e4 100644
--- a/.changeset/proud-rocks-lick.md
+++ b/.changeset/proud-rocks-lick.md
@@ -2,4 +2,4 @@
 '@backstage/backend-common': patch
 ---
 
-Added support for non-"amazonaws.com" hosts (for example when testing with localstack) in AwsS3UrlReader.
+Added support for non-"amazonaws.com" hosts (for example when testing with LocalStack) in AwsS3UrlReader.
diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md
index 00623dc917..8e74470973 100644
--- a/packages/integration/api-report.md
+++ b/packages/integration/api-report.md
@@ -30,6 +30,8 @@ export class AwsS3Integration implements ScmIntegration {
 // @public
 export type AwsS3IntegrationConfig = {
   host: string;
+  endpoint?: string;
+  s3ForcePathStyle?: boolean;
   accessKeyId?: string;
   secretAccessKey?: string;
   roleArn?: string;

From 0dd1971d4a30a13ae4439fd0bb2a630fb5a2878f Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Wed, 29 Dec 2021 08:36:25 -0500
Subject: [PATCH 08/10] fix tests

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 packages/integration/src/awsS3/AwsS3Integration.test.ts | 2 +-
 packages/integration/src/awsS3/config.test.ts           | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/integration/src/awsS3/AwsS3Integration.test.ts b/packages/integration/src/awsS3/AwsS3Integration.test.ts
index 481f3c1951..2987f46a8a 100644
--- a/packages/integration/src/awsS3/AwsS3Integration.test.ts
+++ b/packages/integration/src/awsS3/AwsS3Integration.test.ts
@@ -24,7 +24,7 @@ describe('AwsS3Integration', () => {
         integrations: {
           awsS3: [
             {
-              host: 'a.com',
+              endpoint: 'https://a.com',
               accessKeyId: 'access key',
               secretAccessKey: 'secret key',
             },
diff --git a/packages/integration/src/awsS3/config.test.ts b/packages/integration/src/awsS3/config.test.ts
index 1c7d7d1647..6fcbfc9964 100644
--- a/packages/integration/src/awsS3/config.test.ts
+++ b/packages/integration/src/awsS3/config.test.ts
@@ -36,6 +36,7 @@ describe('readAwsS3IntegrationConfig', () => {
     );
     expect(output).toEqual({
       host: 'amazonaws.com',
+      s3ForcePathStyle: false,
       accessKeyId: 'fake-key',
       secretAccessKey: 'fake-secret-key',
     });
@@ -59,6 +60,7 @@ describe('readAwsS3IntegrationConfigs', () => {
     );
     expect(output).toContainEqual({
       host: 'amazonaws.com',
+      s3ForcePathStyle: false,
       accessKeyId: 'key',
       secretAccessKey: 'secret',
     });

From ca52cf8b289da667f36cba2e1ee5e034d14d4b96 Mon Sep 17 00:00:00 2001
From: Leon Stein <leons727@gmail.com>
Date: Wed, 29 Dec 2021 08:41:00 -0500
Subject: [PATCH 09/10] add aws s3 custom endpoint config test

Signed-off-by: Leon Stein <leons727@gmail.com>
---
 packages/integration/src/awsS3/config.test.ts | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/packages/integration/src/awsS3/config.test.ts b/packages/integration/src/awsS3/config.test.ts
index 6fcbfc9964..4885a41a2a 100644
--- a/packages/integration/src/awsS3/config.test.ts
+++ b/packages/integration/src/awsS3/config.test.ts
@@ -26,10 +26,9 @@ describe('readAwsS3IntegrationConfig', () => {
     return new ConfigReader(data);
   }
 
-  it('reads all values', () => {
+  it('reads all values (default)', () => {
     const output = readAwsS3IntegrationConfig(
       buildConfig({
-        host: 'amazonaws.com',
         accessKeyId: 'fake-key',
         secretAccessKey: 'fake-secret-key',
       }),
@@ -41,6 +40,23 @@ describe('readAwsS3IntegrationConfig', () => {
       secretAccessKey: 'fake-secret-key',
     });
   });
+  it('reads all values (endpoint)', () => {
+    const output = readAwsS3IntegrationConfig(
+      buildConfig({
+        endpoint: 'http://localhost:4566',
+        s3ForcePathStyle: true,
+        accessKeyId: 'fake-key',
+        secretAccessKey: 'fake-secret-key',
+      }),
+    );
+    expect(output).toEqual({
+      host: 'localhost:4566',
+      endpoint: 'http://localhost:4566',
+      s3ForcePathStyle: true,
+      accessKeyId: 'fake-key',
+      secretAccessKey: 'fake-secret-key',
+    });
+  });
 });
 
 describe('readAwsS3IntegrationConfigs', () => {

From cf2e20a79223997baf2480db0d1c63f4c98abe8b Mon Sep 17 00:00:00 2001
From: Patrik Oldsberg <poldsberg@gmail.com>
Date: Thu, 30 Dec 2021 11:09:18 +0100
Subject: [PATCH 10/10] Create red-chefs-travel.md

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
---
 .changeset/red-chefs-travel.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/red-chefs-travel.md

diff --git a/.changeset/red-chefs-travel.md b/.changeset/red-chefs-travel.md
new file mode 100644
index 0000000000..1323b4afca
--- /dev/null
+++ b/.changeset/red-chefs-travel.md
@@ -0,0 +1,5 @@
+---
+'@backstage/integration': patch
+---
+
+Added `endpoint` and `s3ForcePathStyle` as optional configuration for AWS S3 integrations.