From e74854410d81ec7c1439fcf9840054ff9654152d Mon Sep 17 00:00:00 2001 From: Leon Stein Date: Thu, 23 Dec 2021 16:36:35 -0500 Subject: [PATCH] initial Signed-off-by: Leon Stein --- .../src/reading/AwsS3UrlReader.test.ts | 43 +++++++++++++++++ .../src/reading/AwsS3UrlReader.ts | 47 +++++++++++++++---- packages/integration/src/awsS3/config.ts | 22 +++++++-- packages/integration/src/awsS3/index.ts | 1 + 4 files changed, 100 insertions(+), 13 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index a3ae8b647f..81d34c29a4 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -332,4 +332,47 @@ describe('AwsS3UrlReader', () => { expect(body.toString().trim()).toBe('site_name: Test'); }); }); + + describe('readNonAwsHost', () => { + let awsS3UrlReader: AwsS3UrlReader; + + beforeAll(() => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'localhost:4566', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + validateHost: false, + ssl: false, + }), + ), + ), + { s3, treeResponseFactory }, + ); + }); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.read( + 'http://test-bucket.localhost:4566/awsS3-mock-object.yaml', + ); + expect(response.toString().trim()).toBe('site_name: Test'); + }); + }); }); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 0d806fcfaa..055dd9384e 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -27,12 +27,17 @@ import { UrlReader, } from './types'; import getRawBody from 'raw-body'; -import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { + AwsS3Integration, + ScmIntegrations, + AMAZON_AWS_HOST, +} from '@backstage/integration'; import { ForwardedError, NotModifiedError } from '@backstage/errors'; import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( url: string, + validateHost: boolean, ): { path: string; bucket: string; region: string } => { let { host, pathname } = new URL(url); @@ -47,15 +52,23 @@ const parseURL = ( * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname */ const validHost = new RegExp( - /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, + /^[a-z\d][a-z\d.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, ); - if (!validHost.test(host)) { - throw new Error(`not a valid AWS S3 URL: ${url}`); - } - const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); - host = host.substring(bucket.length); - const [, , region, ,] = host.split('.'); + let bucket; + let region; + + if (!validHost.test(host)) { + if (validateHost) { + throw new Error(`not a valid AWS S3 URL: ${url}`); + } + [bucket] = host.split('.'); + region = 'none'; + } else { + [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); + host = host.substring(bucket.length); + [, , region, ,] = host.split('.'); + } return { path: pathname, @@ -70,9 +83,17 @@ export class AwsS3UrlReader implements UrlReader { return integrations.awsS3.list().map(integration => { const creds = AwsS3UrlReader.buildCredentials(integration); + const defaultHost = integration.config.host === AMAZON_AWS_HOST; + const s3 = new S3({ apiVersion: '2006-03-01', credentials: creds, + endpoint: defaultHost + ? undefined + : `${integration.config.ssl ? 'https://' : 'http://'}${ + integration.config.host + }`, + s3ForcePathStyle: !defaultHost, }); const reader = new AwsS3UrlReader(integration, { s3, @@ -138,7 +159,10 @@ export class AwsS3UrlReader implements UrlReader { options?: ReadUrlOptions, ): Promise { try { - const { path, bucket, region } = parseURL(url); + const { path, bucket, region } = parseURL( + url, + this.integration.config.validateHost, + ); aws.config.update({ region: region }); let params; @@ -178,7 +202,10 @@ export class AwsS3UrlReader implements UrlReader { options?: ReadTreeOptions, ): Promise { try { - const { path, bucket, region } = parseURL(url); + const { path, bucket, region } = parseURL( + url, + this.integration.config.validateHost, + ); const allObjects: ObjectList = []; const responses = []; let continuationToken: string | undefined; diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts index 6b5a23c85c..8c8c057e58 100644 --- a/packages/integration/src/awsS3/config.ts +++ b/packages/integration/src/awsS3/config.ts @@ -17,7 +17,7 @@ import { Config } from '@backstage/config'; import { isValidHost } from '../helpers'; -const AMAZON_AWS_HOST = 'amazonaws.com'; +export const AMAZON_AWS_HOST = 'amazonaws.com'; /** * The configuration parameters for a single AWS S3 provider. @@ -28,7 +28,9 @@ export type AwsS3IntegrationConfig = { /** * The host of the target that this matches on, e.g. "amazonaws.com" * - * Currently only "amazonaws.com" is supported. + * If validateHost is true, "amazonaws.com" host is enforced. To test with localstack or similar AWS S3 emulators, + * setting validateHost to false allows hosts like "localhost:4566". Set ssl to false to access emulated S3 + * endpoint over http (vs https). In that case, S3 urls would look like "http://.localhost:4566/". */ host: string; @@ -46,6 +48,16 @@ export type AwsS3IntegrationConfig = { * roleArn */ roleArn?: string; + + /** + * validateHost + */ + validateHost: boolean; + + /** + * ssl + */ + ssl: boolean; }; /** @@ -62,6 +74,8 @@ export function readAwsS3IntegrationConfig( const accessKeyId = config.getOptionalString('accessKeyId'); const secretAccessKey = config.getOptionalString('secretAccessKey'); const roleArn = config.getOptionalString('roleArn'); + const validateHost = config.getOptionalBoolean('validateHost') ?? true; + const ssl = config.getOptionalBoolean('ssl') ?? true; if (!isValidHost(host)) { throw new Error( @@ -69,7 +83,7 @@ export function readAwsS3IntegrationConfig( ); } - return { host, accessKeyId, secretAccessKey, roleArn }; + return { host, accessKeyId, secretAccessKey, roleArn, validateHost, ssl }; } /** @@ -90,6 +104,8 @@ export function readAwsS3IntegrationConfigs( if (!result.some(c => c.host === AMAZON_AWS_HOST)) { result.push({ host: AMAZON_AWS_HOST, + validateHost: true, + ssl: true, }); } return result; diff --git a/packages/integration/src/awsS3/index.ts b/packages/integration/src/awsS3/index.ts index 1ae3964ec1..de665e2af0 100644 --- a/packages/integration/src/awsS3/index.ts +++ b/packages/integration/src/awsS3/index.ts @@ -17,5 +17,6 @@ export { AwsS3Integration } from './AwsS3Integration'; export { readAwsS3IntegrationConfig, readAwsS3IntegrationConfigs, + AMAZON_AWS_HOST, } from './config'; export type { AwsS3IntegrationConfig } from './config';