paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 35 Packages Projects Releases Wiki Activity

use endpoint/s3ForcePathStyle instead of validateHost/ssl per PR feedback

Browse Source 
Signed-off-by: Leon Stein <leons727@gmail.com>
This commit is contained in:
Leon Stein
2021-12-28 10:23:53 -05:00
parent 995e4c7d9d
commit bc78cc3dca
4 changed files with 99 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+5 -5
View File
@@ -178,7 +178,7 @@ describe('AwsS3UrlReader', () => {
),
).rejects.toThrow(
Error(
`Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
`Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
),
);
});
@@ -234,7 +234,7 @@ describe('AwsS3UrlReader', () => {
),
).rejects.toThrow(
Error(
`Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
`Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
),
);
});
@@ -359,8 +359,8 @@ describe('AwsS3UrlReader', () => {
host: 'localhost:4566',
accessKeyId: 'fake-access-key',
secretAccessKey: 'fake-secret-key',
validateHost: false,
ssl: false,
endpoint: 'http://localhost:4566',
s3ForcePathStyle: true,
}),
),
),
@@ -370,7 +370,7 @@ describe('AwsS3UrlReader', () => {
it('returns contents of an object in a bucket', async () => {
const response = await awsS3UrlReader.read(
'http://test-bucket.localhost:4566/awsS3-mock-object.yaml',
'http://localhost:4566/test-bucket/awsS3-mock-object.yaml',
);
expect(response.toString().trim()).toBe('site_name: Test');
});
packages/backend-common/src/reading/AwsS3UrlReader.ts
+41 -31
View File
@@ -31,13 +31,14 @@ import {
AwsS3Integration,
ScmIntegrations,
AMAZON_AWS_HOST,
AwsS3IntegrationConfig,
} from '@backstage/integration';
import { ForwardedError, NotModifiedError } from '@backstage/errors';
import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
const parseURL = (
url: string,
validateHost: boolean,
config: AwsS3IntegrationConfig,
): { path: string; bucket: string; region: string } => {
let { host, pathname } = new URL(url);
@@ -47,27 +48,44 @@ const parseURL = (
*/
pathname = pathname.substr(1);
/**
* Checks that the given URL is a valid S3 object url.
* Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
*/
const validHost = new RegExp(
/^[a-z\d][a-z\d.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
);
let bucket;
let region;
if (!validHost.test(host)) {
if (validateHost) {
throw new Error(`not a valid AWS S3 URL: ${url}`);
/**
* Path style URLs: https://s3.Region.amazonaws.com/bucket-name/key-name
* Virtual hosted style URLs: https://bucket-name.s3.Region.amazonaws.com/key-name
* See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access
*/
if (config.s3ForcePathStyle) {
if (pathname.indexOf('/') < 0) {
throw new Error(
`invalid path-style AWS S3 URL, ${url} does not contain bucket in the path`,
);
}
[bucket] = pathname.split('/');
pathname = pathname.substr(bucket.length + 1);
} else {
if (host.indexOf('.') < 0) {
throw new Error(
`invalid virtual hosted-style AWS S3 URL, ${url} does not contain bucket prefix in the host`,
);
}
[bucket] = host.split('.');
region = 'none';
host = host.substr(bucket.length + 1);
}
// Only extract region from *.amazonaws.com hosts
if (config.host === AMAZON_AWS_HOST) {
// At this point bucket prefix is removed from host for virtual hosted URLs
const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws.com$/);
if (!match) {
throw new Error(
`invalid AWS S3 URL, cannot parse region from host in ${url}`,
);
}
region = match[1];
} else {
[bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
host = host.substring(bucket.length);
[, , region, ,] = host.split('.');
region = '';
}
return {
@@ -83,17 +101,15 @@ export class AwsS3UrlReader implements UrlReader {
return integrations.awsS3.list().map(integration => {
const creds = AwsS3UrlReader.buildCredentials(integration);
const defaultHost = integration.config.host === AMAZON_AWS_HOST;
const s3 = new S3({
apiVersion: '2006-03-01',
credentials: creds,
endpoint: defaultHost
? undefined
: `${integration.config.ssl ? 'https://' : 'http://'}${
integration.config.host
}`,
s3ForcePathStyle: !defaultHost,
endpoint:
integration.config.host === AMAZON_AWS_HOST
? undefined
: integration.config.endpoint,
s3ForcePathStyle: integration.config.s3ForcePathStyle,
});
const reader = new AwsS3UrlReader(integration, {
s3,
@@ -159,10 +175,7 @@ export class AwsS3UrlReader implements UrlReader {
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
try {
const { path, bucket, region } = parseURL(
url,
this.integration.config.validateHost,
);
const { path, bucket, region } = parseURL(url, this.integration.config);
aws.config.update({ region: region });
let params;
@@ -202,10 +215,7 @@ export class AwsS3UrlReader implements UrlReader {
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
try {
const { path, bucket, region } = parseURL(
url,
this.integration.config.validateHost,
);
const { path, bucket, region } = parseURL(url, this.integration.config);
const allObjects: ObjectList = [];
const responses = [];
let continuationToken: string | undefined;
packages/integration/config.d.ts
Vendored
-5
View File
@@ -166,11 +166,6 @@ export interface Config {
/** Integration configuration for AWS S3 Service */
awsS3?: Array<{
/**
* The host of the target that this matches on, e.g. "amazonaws.com".
* @visibility frontend
*/
host: string;
/**
* Account access key used to authenticate requests.
* @visibility backend
packages/integration/src/awsS3/config.ts
+53 -34
View File
@@ -15,7 +15,6 @@
*/
import { Config } from '@backstage/config';
import { isValidHost } from '../helpers';
export const AMAZON_AWS_HOST = 'amazonaws.com';
@@ -26,38 +25,40 @@ export const AMAZON_AWS_HOST = 'amazonaws.com';
*/
export type AwsS3IntegrationConfig = {
/**
* The host of the target that this matches on, e.g. "amazonaws.com"
*
* If validateHost is true, "amazonaws.com" host is enforced. To test with localstack or similar AWS S3 emulators,
* setting validateHost to false allows hosts like "localhost:4566". Set ssl to false to access emulated S3
* endpoint over http (vs https). In that case, S3 urls would look like "http://<bucket>.localhost:4566/<path>".
* Host, derived from endpoint, and defaults to amazonaws.com
*/
host: string;
/**
* accessKeyId
* (Optional) AWS Endpoint.
* The endpoint URI to send requests to. The default endpoint is built from the configured region.
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
*
* Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566
*/
endpoint?: string;
/**
* (Optional) Whether to use path style URLs when communicating with S3.
* Defaults to false.
* This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used.
*/
s3ForcePathStyle?: boolean;
/**
* (Optional) User access key id
*/
accessKeyId?: string;
/**
* secretAccessKey
* (Optional) User secret access key
*/
secretAccessKey?: string;
/**
* roleArn
* (Optional) ARN of role to be assumed
*/
roleArn?: string;
/**
* validateHost
*/
validateHost: boolean;
/**
* ssl
*/
ssl: boolean;
};
/**
@@ -70,20 +71,42 @@ export type AwsS3IntegrationConfig = {
export function readAwsS3IntegrationConfig(
config: Config,
): AwsS3IntegrationConfig {
const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST;
const endpoint = config.getOptionalString('endpoint');
const s3ForcePathStyle =
config.getOptionalBoolean('s3ForcePathStyle') ?? false;
let host;
let pathname;
if (endpoint) {
try {
const url = new URL(endpoint);
host = url.host;
pathname = url.pathname;
} catch {
throw new Error(
`Invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`,
);
}
if (pathname !== '/') {
throw new Error(
`Invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`,
);
}
} else {
host = AMAZON_AWS_HOST;
}
const accessKeyId = config.getOptionalString('accessKeyId');
const secretAccessKey = config.getOptionalString('secretAccessKey');
const roleArn = config.getOptionalString('roleArn');
const validateHost = config.getOptionalBoolean('validateHost') ?? true;
const ssl = config.getOptionalBoolean('ssl') ?? true;
if (!isValidHost(host)) {
throw new Error(
`Invalid awsS3 integration config, '${host}' is not a valid host`,
);
}
return { host, accessKeyId, secretAccessKey, roleArn, validateHost, ssl };
return {
host,
endpoint,
s3ForcePathStyle,
accessKeyId,
secretAccessKey,
roleArn,
};
}
/**
@@ -102,11 +125,7 @@ export function readAwsS3IntegrationConfigs(
// If no explicit amazonaws.com integration was added, put one in the list as
// a convenience
if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
result.push({
host: AMAZON_AWS_HOST,
validateHost: true,
ssl: true,
});
result.push({ host: AMAZON_AWS_HOST });
}
return result;
}