From bc78cc3dca4f8662bc122e8dd4d4875d43bc7269 Mon Sep 17 00:00:00 2001 From: Leon Stein Date: Tue, 28 Dec 2021 10:23:53 -0500 Subject: [PATCH] use endpoint/s3ForcePathStyle instead of validateHost/ssl per PR feedback Signed-off-by: Leon Stein --- .../src/reading/AwsS3UrlReader.test.ts | 10 +-- .../src/reading/AwsS3UrlReader.ts | 72 ++++++++------- packages/integration/config.d.ts | 5 -- packages/integration/src/awsS3/config.ts | 87 +++++++++++-------- 4 files changed, 99 insertions(+), 75 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 81d34c29a4..2911699a5d 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -178,7 +178,7 @@ describe('AwsS3UrlReader', () => { ), ).rejects.toThrow( Error( - `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, ), ); }); @@ -234,7 +234,7 @@ describe('AwsS3UrlReader', () => { ), ).rejects.toThrow( Error( - `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, ), ); }); @@ -359,8 +359,8 @@ describe('AwsS3UrlReader', () => { host: 'localhost:4566', accessKeyId: 'fake-access-key', secretAccessKey: 'fake-secret-key', - validateHost: false, - ssl: false, + endpoint: 'http://localhost:4566', + s3ForcePathStyle: true, }), ), ), @@ -370,7 +370,7 @@ describe('AwsS3UrlReader', () => { it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.read( - 'http://test-bucket.localhost:4566/awsS3-mock-object.yaml', + 'http://localhost:4566/test-bucket/awsS3-mock-object.yaml', ); expect(response.toString().trim()).toBe('site_name: Test'); }); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 055dd9384e..b6655ecd0e 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -31,13 +31,14 @@ import { AwsS3Integration, ScmIntegrations, AMAZON_AWS_HOST, + AwsS3IntegrationConfig, } from '@backstage/integration'; import { ForwardedError, NotModifiedError } from '@backstage/errors'; import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( url: string, - validateHost: boolean, + config: AwsS3IntegrationConfig, ): { path: string; bucket: string; region: string } => { let { host, pathname } = new URL(url); @@ -47,27 +48,44 @@ const parseURL = ( */ pathname = pathname.substr(1); - /** - * Checks that the given URL is a valid S3 object url. - * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname - */ - const validHost = new RegExp( - /^[a-z\d][a-z\d.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, - ); - let bucket; let region; - if (!validHost.test(host)) { - if (validateHost) { - throw new Error(`not a valid AWS S3 URL: ${url}`); + /** + * Path style URLs: https://s3.Region.amazonaws.com/bucket-name/key-name + * Virtual hosted style URLs: https://bucket-name.s3.Region.amazonaws.com/key-name + * See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access + */ + if (config.s3ForcePathStyle) { + if (pathname.indexOf('/') < 0) { + throw new Error( + `invalid path-style AWS S3 URL, ${url} does not contain bucket in the path`, + ); + } + [bucket] = pathname.split('/'); + pathname = pathname.substr(bucket.length + 1); + } else { + if (host.indexOf('.') < 0) { + throw new Error( + `invalid virtual hosted-style AWS S3 URL, ${url} does not contain bucket prefix in the host`, + ); } [bucket] = host.split('.'); - region = 'none'; + host = host.substr(bucket.length + 1); + } + + // Only extract region from *.amazonaws.com hosts + if (config.host === AMAZON_AWS_HOST) { + // At this point bucket prefix is removed from host for virtual hosted URLs + const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws.com$/); + if (!match) { + throw new Error( + `invalid AWS S3 URL, cannot parse region from host in ${url}`, + ); + } + region = match[1]; } else { - [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); - host = host.substring(bucket.length); - [, , region, ,] = host.split('.'); + region = ''; } return { @@ -83,17 +101,15 @@ export class AwsS3UrlReader implements UrlReader { return integrations.awsS3.list().map(integration => { const creds = AwsS3UrlReader.buildCredentials(integration); - const defaultHost = integration.config.host === AMAZON_AWS_HOST; const s3 = new S3({ apiVersion: '2006-03-01', credentials: creds, - endpoint: defaultHost - ? undefined - : `${integration.config.ssl ? 'https://' : 'http://'}${ - integration.config.host - }`, - s3ForcePathStyle: !defaultHost, + endpoint: + integration.config.host === AMAZON_AWS_HOST + ? undefined + : integration.config.endpoint, + s3ForcePathStyle: integration.config.s3ForcePathStyle, }); const reader = new AwsS3UrlReader(integration, { s3, @@ -159,10 +175,7 @@ export class AwsS3UrlReader implements UrlReader { options?: ReadUrlOptions, ): Promise { try { - const { path, bucket, region } = parseURL( - url, - this.integration.config.validateHost, - ); + const { path, bucket, region } = parseURL(url, this.integration.config); aws.config.update({ region: region }); let params; @@ -202,10 +215,7 @@ export class AwsS3UrlReader implements UrlReader { options?: ReadTreeOptions, ): Promise { try { - const { path, bucket, region } = parseURL( - url, - this.integration.config.validateHost, - ); + const { path, bucket, region } = parseURL(url, this.integration.config); const allObjects: ObjectList = []; const responses = []; let continuationToken: string | undefined; diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts index 90cc10f77b..85a9c78cf7 100644 --- a/packages/integration/config.d.ts +++ b/packages/integration/config.d.ts @@ -166,11 +166,6 @@ export interface Config { /** Integration configuration for AWS S3 Service */ awsS3?: Array<{ - /** - * The host of the target that this matches on, e.g. "amazonaws.com". - * @visibility frontend - */ - host: string; /** * Account access key used to authenticate requests. * @visibility backend diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts index 8c8c057e58..ccbc09a45a 100644 --- a/packages/integration/src/awsS3/config.ts +++ b/packages/integration/src/awsS3/config.ts @@ -15,7 +15,6 @@ */ import { Config } from '@backstage/config'; -import { isValidHost } from '../helpers'; export const AMAZON_AWS_HOST = 'amazonaws.com'; @@ -26,38 +25,40 @@ export const AMAZON_AWS_HOST = 'amazonaws.com'; */ export type AwsS3IntegrationConfig = { /** - * The host of the target that this matches on, e.g. "amazonaws.com" - * - * If validateHost is true, "amazonaws.com" host is enforced. To test with localstack or similar AWS S3 emulators, - * setting validateHost to false allows hosts like "localhost:4566". Set ssl to false to access emulated S3 - * endpoint over http (vs https). In that case, S3 urls would look like "http://.localhost:4566/". + * Host, derived from endpoint, and defaults to amazonaws.com */ host: string; /** - * accessKeyId + * (Optional) AWS Endpoint. + * The endpoint URI to send requests to. The default endpoint is built from the configured region. + * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property + * + * Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566 + */ + endpoint?: string; + + /** + * (Optional) Whether to use path style URLs when communicating with S3. + * Defaults to false. + * This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used. + */ + s3ForcePathStyle?: boolean; + + /** + * (Optional) User access key id */ accessKeyId?: string; /** - * secretAccessKey + * (Optional) User secret access key */ secretAccessKey?: string; /** - * roleArn + * (Optional) ARN of role to be assumed */ roleArn?: string; - - /** - * validateHost - */ - validateHost: boolean; - - /** - * ssl - */ - ssl: boolean; }; /** @@ -70,20 +71,42 @@ export type AwsS3IntegrationConfig = { export function readAwsS3IntegrationConfig( config: Config, ): AwsS3IntegrationConfig { - const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST; + const endpoint = config.getOptionalString('endpoint'); + const s3ForcePathStyle = + config.getOptionalBoolean('s3ForcePathStyle') ?? false; + let host; + let pathname; + if (endpoint) { + try { + const url = new URL(endpoint); + host = url.host; + pathname = url.pathname; + } catch { + throw new Error( + `Invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`, + ); + } + if (pathname !== '/') { + throw new Error( + `Invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`, + ); + } + } else { + host = AMAZON_AWS_HOST; + } + const accessKeyId = config.getOptionalString('accessKeyId'); const secretAccessKey = config.getOptionalString('secretAccessKey'); const roleArn = config.getOptionalString('roleArn'); - const validateHost = config.getOptionalBoolean('validateHost') ?? true; - const ssl = config.getOptionalBoolean('ssl') ?? true; - if (!isValidHost(host)) { - throw new Error( - `Invalid awsS3 integration config, '${host}' is not a valid host`, - ); - } - - return { host, accessKeyId, secretAccessKey, roleArn, validateHost, ssl }; + return { + host, + endpoint, + s3ForcePathStyle, + accessKeyId, + secretAccessKey, + roleArn, + }; } /** @@ -102,11 +125,7 @@ export function readAwsS3IntegrationConfigs( // If no explicit amazonaws.com integration was added, put one in the list as // a convenience if (!result.some(c => c.host === AMAZON_AWS_HOST)) { - result.push({ - host: AMAZON_AWS_HOST, - validateHost: true, - ssl: true, - }); + result.push({ host: AMAZON_AWS_HOST }); } return result; }