paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #8636 from leons727/aws-s3-localstack

Browse Source 
Support for non-amazonaws.com hosts in AwsS3UrlReader
This commit is contained in:
Patrik Oldsberg
2021-12-30 12:01:09 +01:00
committed by GitHub
parent 6335b5a9ea cf2e20a792
commit f3784edd89
9 changed files with 191 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/proud-rocks-lick.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-common': patch
---
Added support for non-"amazonaws.com" hosts (for example when testing with LocalStack) in AwsS3UrlReader.
.changeset/red-chefs-travel.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/integration': patch
---
Added `endpoint` and `s3ForcePathStyle` as optional configuration for AWS S3 integrations.
packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+45 -2
View File
@@ -178,7 +178,7 @@ describe('AwsS3UrlReader', () => {
),
).rejects.toThrow(
Error(
`Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
`Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
),
);
});
@@ -234,7 +234,7 @@ describe('AwsS3UrlReader', () => {
),
).rejects.toThrow(
Error(
`Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
`Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
),
);
});
@@ -332,4 +332,47 @@ describe('AwsS3UrlReader', () => {
expect(body.toString().trim()).toBe('site_name: Test');
});
});
describe('readNonAwsHost', () => {
let awsS3UrlReader: AwsS3UrlReader;
beforeAll(() => {
AWSMock.setSDKInstance(aws);
AWSMock.mock(
'S3',
'getObject',
Buffer.from(
require('fs').readFileSync(
path.resolve(
__dirname,
'__fixtures__/awsS3/awsS3-mock-object.yaml',
),
),
),
);
const s3 = new aws.S3();
awsS3UrlReader = new AwsS3UrlReader(
new AwsS3Integration(
readAwsS3IntegrationConfig(
new ConfigReader({
host: 'localhost:4566',
accessKeyId: 'fake-access-key',
secretAccessKey: 'fake-secret-key',
endpoint: 'http://localhost:4566',
s3ForcePathStyle: true,
}),
),
),
{ s3, treeResponseFactory },
);
});
it('returns contents of an object in a bucket', async () => {
const response = await awsS3UrlReader.read(
'http://localhost:4566/test-bucket/awsS3-mock-object.yaml',
);
expect(response.toString().trim()).toBe('site_name: Test');
});
});
});
packages/backend-common/src/reading/AwsS3UrlReader.ts
+46 -13
View File
@@ -27,12 +27,17 @@ import {
UrlReader,
} from './types';
import getRawBody from 'raw-body';
import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
import {
AwsS3Integration,
ScmIntegrations,
AwsS3IntegrationConfig,
} from '@backstage/integration';
import { ForwardedError, NotModifiedError } from '@backstage/errors';
import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
const parseURL = (
url: string,
config: AwsS3IntegrationConfig,
): { path: string; bucket: string; region: string } => {
let { host, pathname } = new URL(url);
@@ -42,20 +47,45 @@ const parseURL = (
*/
pathname = pathname.substr(1);
let bucket;
let region;
/**
* Checks that the given URL is a valid S3 object url.
* Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
* Path style URLs: https://s3.Region.amazonaws.com/bucket-name/key-name
* Virtual hosted style URLs: https://bucket-name.s3.Region.amazonaws.com/key-name
* See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access
*/
const validHost = new RegExp(
/^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
);
if (!validHost.test(host)) {
throw new Error(`not a valid AWS S3 URL: ${url}`);
if (config.s3ForcePathStyle) {
if (pathname.indexOf('/') < 0) {
throw new Error(
`invalid path-style AWS S3 URL, ${url} does not contain bucket in the path`,
);
}
[bucket] = pathname.split('/');
pathname = pathname.substr(bucket.length + 1);
} else {
if (host.indexOf('.') < 0) {
throw new Error(
`invalid virtual hosted-style AWS S3 URL, ${url} does not contain bucket prefix in the host`,
);
}
[bucket] = host.split('.');
host = host.substr(bucket.length + 1);
}
const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
host = host.substring(bucket.length);
const [, , region, ,] = host.split('.');
// Only extract region from *.amazonaws.com hosts
if (config.host === 'amazonaws.com') {
// At this point bucket prefix is removed from host for virtual hosted URLs
const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws\.com$/);
if (!match) {
throw new Error(
`invalid AWS S3 URL, cannot parse region from host in ${url}`,
);
}
region = match[1];
} else {
region = '';
}
return {
path: pathname,
@@ -70,9 +100,12 @@ export class AwsS3UrlReader implements UrlReader {
return integrations.awsS3.list().map(integration => {
const creds = AwsS3UrlReader.buildCredentials(integration);
const s3 = new S3({
apiVersion: '2006-03-01',
credentials: creds,
endpoint: integration.config.endpoint,
s3ForcePathStyle: integration.config.s3ForcePathStyle,
});
const reader = new AwsS3UrlReader(integration, {
s3,
@@ -138,7 +171,7 @@ export class AwsS3UrlReader implements UrlReader {
options?: ReadUrlOptions,
): Promise<ReadUrlResponse> {
try {
const { path, bucket, region } = parseURL(url);
const { path, bucket, region } = parseURL(url, this.integration.config);
aws.config.update({ region: region });
let params;
@@ -178,7 +211,7 @@ export class AwsS3UrlReader implements UrlReader {
options?: ReadTreeOptions,
): Promise<ReadTreeResponse> {
try {
const { path, bucket, region } = parseURL(url);
const { path, bucket, region } = parseURL(url, this.integration.config);
const allObjects: ObjectList = [];
const responses = [];
let continuationToken: string | undefined;
packages/integration/api-report.md
+2
View File
@@ -30,6 +30,8 @@ export class AwsS3Integration implements ScmIntegration {
// @public
export type AwsS3IntegrationConfig = {
host: string;
endpoint?: string;
s3ForcePathStyle?: boolean;
accessKeyId?: string;
secretAccessKey?: string;
roleArn?: string;
packages/integration/config.d.ts
Vendored
+15 -2
View File
@@ -167,10 +167,23 @@ export interface Config {
/** Integration configuration for AWS S3 Service */
awsS3?: Array<{
/**
* The host of the target that this matches on, e.g. "amazonaws.com".
* AWS Endpoint.
* The endpoint URI to send requests to. The default endpoint is built from the configured region.
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
*
* Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566
* @visibility frontend
*/
host: string;
endpoint?: string;
/**
* Whether to use path style URLs when communicating with S3.
* Defaults to false.
* This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used.
* @visibility frontend
*/
s3ForcePathStyle?: boolean;
/**
* Account access key used to authenticate requests.
* @visibility backend
packages/integration/src/awsS3/AwsS3Integration.test.ts
+1 -1
View File
@@ -24,7 +24,7 @@ describe('AwsS3Integration', () => {
integrations: {
awsS3: [
{
host: 'a.com',
endpoint: 'https://a.com',
accessKeyId: 'access key',
secretAccessKey: 'secret key',
},
packages/integration/src/awsS3/config.test.ts
+20 -2
View File
@@ -26,16 +26,33 @@ describe('readAwsS3IntegrationConfig', () => {
return new ConfigReader(data);
}
it('reads all values', () => {
it('reads all values (default)', () => {
const output = readAwsS3IntegrationConfig(
buildConfig({
host: 'amazonaws.com',
accessKeyId: 'fake-key',
secretAccessKey: 'fake-secret-key',
}),
);
expect(output).toEqual({
host: 'amazonaws.com',
s3ForcePathStyle: false,
accessKeyId: 'fake-key',
secretAccessKey: 'fake-secret-key',
});
});
it('reads all values (endpoint)', () => {
const output = readAwsS3IntegrationConfig(
buildConfig({
endpoint: 'http://localhost:4566',
s3ForcePathStyle: true,
accessKeyId: 'fake-key',
secretAccessKey: 'fake-secret-key',
}),
);
expect(output).toEqual({
host: 'localhost:4566',
endpoint: 'http://localhost:4566',
s3ForcePathStyle: true,
accessKeyId: 'fake-key',
secretAccessKey: 'fake-secret-key',
});
@@ -59,6 +76,7 @@ describe('readAwsS3IntegrationConfigs', () => {
);
expect(output).toContainEqual({
host: 'amazonaws.com',
s3ForcePathStyle: false,
accessKeyId: 'key',
secretAccessKey: 'secret',
});
packages/integration/src/awsS3/config.ts
+52 -15
View File
@@ -15,7 +15,6 @@
*/
import { Config } from '@backstage/config';
import { isValidHost } from '../helpers';
const AMAZON_AWS_HOST = 'amazonaws.com';
@@ -26,24 +25,38 @@ const AMAZON_AWS_HOST = 'amazonaws.com';
*/
export type AwsS3IntegrationConfig = {
/**
* The host of the target that this matches on, e.g. "amazonaws.com"
*
* Currently only "amazonaws.com" is supported.
* Host, derived from endpoint, and defaults to amazonaws.com
*/
host: string;
/**
* accessKeyId
* (Optional) AWS Endpoint.
* The endpoint URI to send requests to. The default endpoint is built from the configured region.
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
*
* Supports non-AWS providers, e.g. for LocalStack, endpoint may look like http://localhost:4566
*/
endpoint?: string;
/**
* (Optional) Whether to use path style URLs when communicating with S3.
* Defaults to false.
* This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used.
*/
s3ForcePathStyle?: boolean;
/**
* (Optional) User access key id
*/
accessKeyId?: string;
/**
* secretAccessKey
* (Optional) User secret access key
*/
secretAccessKey?: string;
/**
* roleArn
* (Optional) ARN of role to be assumed
*/
roleArn?: string;
};
@@ -58,18 +71,42 @@ export type AwsS3IntegrationConfig = {
export function readAwsS3IntegrationConfig(
config: Config,
): AwsS3IntegrationConfig {
const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST;
const endpoint = config.getOptionalString('endpoint');
const s3ForcePathStyle =
config.getOptionalBoolean('s3ForcePathStyle') ?? false;
let host;
let pathname;
if (endpoint) {
try {
const url = new URL(endpoint);
host = url.host;
pathname = url.pathname;
} catch {
throw new Error(
`invalid awsS3 integration config, endpoint '${endpoint}' is not a valid URL`,
);
}
if (pathname !== '/') {
throw new Error(
`invalid awsS3 integration config, endpoints cannot contain path, got '${endpoint}'`,
);
}
} else {
host = AMAZON_AWS_HOST;
}
const accessKeyId = config.getOptionalString('accessKeyId');
const secretAccessKey = config.getOptionalString('secretAccessKey');
const roleArn = config.getOptionalString('roleArn');
if (!isValidHost(host)) {
throw new Error(
`Invalid awsS3 integration config, '${host}' is not a valid host`,
);
}
return { host, accessKeyId, secretAccessKey, roleArn };
return {
host,
endpoint,
s3ForcePathStyle,
accessKeyId,
secretAccessKey,
roleArn,
};
}
/**