diff --git a/.changeset/proud-rocks-lick.md b/.changeset/proud-rocks-lick.md new file mode 100644 index 0000000000..ad33c410e4 --- /dev/null +++ b/.changeset/proud-rocks-lick.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Added support for non-"amazonaws.com" hosts (for example when testing with LocalStack) in AwsS3UrlReader. diff --git a/.changeset/red-chefs-travel.md b/.changeset/red-chefs-travel.md new file mode 100644 index 0000000000..1323b4afca --- /dev/null +++ b/.changeset/red-chefs-travel.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Added `endpoint` and `s3ForcePathStyle` as optional configuration for AWS S3 integrations. diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index a3ae8b647f..2911699a5d 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -178,7 +178,7 @@ describe('AwsS3UrlReader', () => { ), ).rejects.toThrow( Error( - `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, ), ); }); @@ -234,7 +234,7 @@ describe('AwsS3UrlReader', () => { ), ).rejects.toThrow( Error( - `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, ), ); }); @@ -332,4 +332,47 @@ describe('AwsS3UrlReader', () => { expect(body.toString().trim()).toBe('site_name: Test'); }); }); + + describe('readNonAwsHost', () => { + let awsS3UrlReader: AwsS3UrlReader; + + beforeAll(() => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'localhost:4566', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + endpoint: 'http://localhost:4566', + s3ForcePathStyle: true, + }), + ), + ), + { s3, treeResponseFactory }, + ); + }); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.read( + 'http://localhost:4566/test-bucket/awsS3-mock-object.yaml', + ); + expect(response.toString().trim()).toBe('site_name: Test'); + }); + }); }); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 0d806fcfaa..1fca72e91c 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -27,12 +27,17 @@ import { UrlReader, } from './types'; import getRawBody from 'raw-body'; -import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { + AwsS3Integration, + ScmIntegrations, + AwsS3IntegrationConfig, +} from '@backstage/integration'; import { ForwardedError, NotModifiedError } from '@backstage/errors'; import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( url: string, + config: AwsS3IntegrationConfig, ): { path: string; bucket: string; region: string } => { let { host, pathname } = new URL(url); @@ -42,20 +47,45 @@ const parseURL = ( */ pathname = pathname.substr(1); + let bucket; + let region; + /** - * Checks that the given URL is a valid S3 object url. - * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname + * Path style URLs: https://s3.Region.amazonaws.com/bucket-name/key-name + * Virtual hosted style URLs: https://bucket-name.s3.Region.amazonaws.com/key-name + * See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access */ - const validHost = new RegExp( - /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, - ); - if (!validHost.test(host)) { - throw new Error(`not a valid AWS S3 URL: ${url}`); + if (config.s3ForcePathStyle) { + if (pathname.indexOf('/') < 0) { + throw new Error( + `invalid path-style AWS S3 URL, ${url} does not contain bucket in the path`, + ); + } + [bucket] = pathname.split('/'); + pathname = pathname.substr(bucket.length + 1); + } else { + if (host.indexOf('.') < 0) { + throw new Error( + `invalid virtual hosted-style AWS S3 URL, ${url} does not contain bucket prefix in the host`, + ); + } + [bucket] = host.split('.'); + host = host.substr(bucket.length + 1); } - const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); - host = host.substring(bucket.length); - const [, , region, ,] = host.split('.'); + // Only extract region from *.amazonaws.com hosts + if (config.host === 'amazonaws.com') { + // At this point bucket prefix is removed from host for virtual hosted URLs + const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws\.com$/); + if (!match) { + throw new Error( + `invalid AWS S3 URL, cannot parse region from host in ${url}`, + ); + } + region = match[1]; + } else { + region = ''; + } return { path: pathname, @@ -70,9 +100,12 @@ export class AwsS3UrlReader implements UrlReader { return integrations.awsS3.list().map(integration => { const creds = AwsS3UrlReader.buildCredentials(integration); + const s3 = new S3({ apiVersion: '2006-03-01', credentials: creds, + endpoint: integration.config.endpoint, + s3ForcePathStyle: integration.config.s3ForcePathStyle, }); const reader = new AwsS3UrlReader(integration, { s3, @@ -138,7 +171,7 @@ export class AwsS3UrlReader implements UrlReader { options?: ReadUrlOptions, ): Promise { try { - const { path, bucket, region } = parseURL(url); + const { path, bucket, region } = parseURL(url, this.integration.config); aws.config.update({ region: region }); let params; @@ -178,7 +211,7 @@ export class AwsS3UrlReader implements UrlReader { options?: ReadTreeOptions, ): Promise { try { - const { path, bucket, region } = parseURL(url); + const { path, bucket, region } = parseURL(url, this.integration.config); const allObjects: ObjectList = []; const responses = []; let continuationToken: string | undefined; diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 00623dc917..8e74470973 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -30,6 +30,8 @@ export class AwsS3Integration implements ScmIntegration { // @public export type AwsS3IntegrationConfig = { host: string; + endpoint?: string; + s3ForcePathStyle?: boolean; accessKeyId?: string; secretAccessKey?: string; roleArn?: string; diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts index 90cc10f77b..8873f0f220 100644 --- a/packages/integration/config.d.ts +++ b/packages/integration/config.d.ts @@ -167,10 +167,23 @@ export interface Config { /** Integration configuration for AWS S3 Service */ awsS3?: Array<{ /** - * The host of the target that this matches on, e.g. "amazonaws.com". + * AWS Endpoint. + * The endpoint URI to send requests to. The default endpoint is built from the configured region. + * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property + * + * Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566 * @visibility frontend */ - host: string; + endpoint?: string; + + /** + * Whether to use path style URLs when communicating with S3. + * Defaults to false. + * This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used. + * @visibility frontend + */ + s3ForcePathStyle?: boolean; + /** * Account access key used to authenticate requests. * @visibility backend diff --git a/packages/integration/src/awsS3/AwsS3Integration.test.ts b/packages/integration/src/awsS3/AwsS3Integration.test.ts index 481f3c1951..2987f46a8a 100644 --- a/packages/integration/src/awsS3/AwsS3Integration.test.ts +++ b/packages/integration/src/awsS3/AwsS3Integration.test.ts @@ -24,7 +24,7 @@ describe('AwsS3Integration', () => { integrations: { awsS3: [ { - host: 'a.com', + endpoint: 'https://a.com', accessKeyId: 'access key', secretAccessKey: 'secret key', }, diff --git a/packages/integration/src/awsS3/config.test.ts b/packages/integration/src/awsS3/config.test.ts index 1c7d7d1647..4885a41a2a 100644 --- a/packages/integration/src/awsS3/config.test.ts +++ b/packages/integration/src/awsS3/config.test.ts @@ -26,16 +26,33 @@ describe('readAwsS3IntegrationConfig', () => { return new ConfigReader(data); } - it('reads all values', () => { + it('reads all values (default)', () => { const output = readAwsS3IntegrationConfig( buildConfig({ - host: 'amazonaws.com', accessKeyId: 'fake-key', secretAccessKey: 'fake-secret-key', }), ); expect(output).toEqual({ host: 'amazonaws.com', + s3ForcePathStyle: false, + accessKeyId: 'fake-key', + secretAccessKey: 'fake-secret-key', + }); + }); + it('reads all values (endpoint)', () => { + const output = readAwsS3IntegrationConfig( + buildConfig({ + endpoint: 'http://localhost:4566', + s3ForcePathStyle: true, + accessKeyId: 'fake-key', + secretAccessKey: 'fake-secret-key', + }), + ); + expect(output).toEqual({ + host: 'localhost:4566', + endpoint: 'http://localhost:4566', + s3ForcePathStyle: true, accessKeyId: 'fake-key', secretAccessKey: 'fake-secret-key', }); @@ -59,6 +76,7 @@ describe('readAwsS3IntegrationConfigs', () => { ); expect(output).toContainEqual({ host: 'amazonaws.com', + s3ForcePathStyle: false, accessKeyId: 'key', secretAccessKey: 'secret', }); diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts index 6b5a23c85c..45cbdbe210 100644 --- a/packages/integration/src/awsS3/config.ts +++ b/packages/integration/src/awsS3/config.ts @@ -15,7 +15,6 @@ */ import { Config } from '@backstage/config'; -import { isValidHost } from '../helpers'; const AMAZON_AWS_HOST = 'amazonaws.com'; @@ -26,24 +25,38 @@ const AMAZON_AWS_HOST = 'amazonaws.com'; */ export type AwsS3IntegrationConfig = { /** - * The host of the target that this matches on, e.g. "amazonaws.com" - * - * Currently only "amazonaws.com" is supported. + * Host, derived from endpoint, and defaults to amazonaws.com */ host: string; /** - * accessKeyId + * (Optional) AWS Endpoint. + * The endpoint URI to send requests to. The default endpoint is built from the configured region. + * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property + * + * Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566 + */ + endpoint?: string; + + /** + * (Optional) Whether to use path style URLs when communicating with S3. + * Defaults to false. + * This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used. + */ + s3ForcePathStyle?: boolean; + + /** + * (Optional) User access key id */ accessKeyId?: string; /** - * secretAccessKey + * (Optional) User secret access key */ secretAccessKey?: string; /** - * roleArn + * (Optional) ARN of role to be assumed */ roleArn?: string; }; @@ -58,18 +71,42 @@ export type AwsS3IntegrationConfig = { export function readAwsS3IntegrationConfig( config: Config, ): AwsS3IntegrationConfig { - const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST; + const endpoint = config.getOptionalString('endpoint'); + const s3ForcePathStyle = + config.getOptionalBoolean('s3ForcePathStyle') ?? false; + let host; + let pathname; + if (endpoint) { + try { + const url = new URL(endpoint); + host = url.host; + pathname = url.pathname; + } catch { + throw new Error( + `invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`, + ); + } + if (pathname !== '/') { + throw new Error( + `invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`, + ); + } + } else { + host = AMAZON_AWS_HOST; + } + const accessKeyId = config.getOptionalString('accessKeyId'); const secretAccessKey = config.getOptionalString('secretAccessKey'); const roleArn = config.getOptionalString('roleArn'); - if (!isValidHost(host)) { - throw new Error( - `Invalid awsS3 integration config, '${host}' is not a valid host`, - ); - } - - return { host, accessKeyId, secretAccessKey, roleArn }; + return { + host, + endpoint, + s3ForcePathStyle, + accessKeyId, + secretAccessKey, + roleArn, + }; } /**