diff --git a/app-config.yaml b/app-config.yaml index c1d0121dff..7488cd8648 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -160,9 +160,12 @@ integrations: azure: - host: dev.azure.com token: ${AZURE_TOKEN} -# googleGcs: -# clientEmail: 'example@example.com' -# privateKey: ${GCS_PRIVATE_KEY} + # googleGcs: + # clientEmail: 'example@example.com' + # privateKey: ${GCS_PRIVATE_KEY} + # awsS3: + # accessKeyId: ${AWS_ACCESS_KEY_ID} + # secretAccessKey: ${AWS_SECRET_ACCESS_KEY} catalog: rules: diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 4c7aacdf96..c4fe498cda 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -40,6 +40,7 @@ "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "archiver": "^5.0.2", + "aws-sdk": "^2.952.0", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts new file mode 100644 index 0000000000..7ab43cd8cc --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader, JsonObject } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { DefaultReadTreeResponseFactory } from './tree'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; +import { UrlReaderPredicateTuple } from './types'; + +describe('AwsS3UrlReader', () => { + const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { + return AwsS3UrlReader.factory({ + config: new ConfigReader(config), + logger: getVoidLogger(), + treeResponseFactory: DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), + }), + }); + }; + + it('does not create a reader without the awsS3 field', () => { + const entries = createReader({ + integrations: {}, + }); + expect(entries).toHaveLength(0); + }); + + it('creates a reader with credentials correctly configured', () => { + const entries = createReader({ + integrations: { + awsS3: { + accessKeyId: 'fakekey', + secretAccessKey: 'fakekey', + }, + }, + }); + expect(entries).toHaveLength(1); + }); + + it('creates a reader with default credentials provider', () => { + const entries = createReader({ + integrations: { + awsS3: {}, + }, + }); + expect(entries).toHaveLength(1); + }); + + describe('predicates', () => { + const readers = createReader({ + integrations: { + awsS3: {}, + }, + }); + const predicate = readers[0].predicate; + + it('returns true for the correct aws s3 storage host', () => { + expect( + predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')), + ).toBe(true); + }); + + it('returns true for a url with the full path and the correct host', () => { + expect( + predicate( + new URL( + 'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + + it('returns false for an incorrect host', () => { + expect(predicate(new URL('https://amazon.com'))).toBe(false); + }); + + it('returns false for a completely different host', () => { + expect(predicate(new URL('https://storage.cloud.google.com'))).toBe( + false, + ); + }); + + it("returns true for a url with a bucket with '.'", () => { + expect( + predicate( + new URL( + 'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + }); +}); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts new file mode 100644 index 0000000000..b0b9dcf322 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import aws, { Credentials, S3 } from 'aws-sdk'; +import { + ReaderFactory, + ReadTreeResponse, + SearchResponse, + UrlReader, +} from './types'; +import getRawBody from 'raw-body'; +import { + AwsS3IntegrationConfig, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; + +const AMAZON_AWS_HOST = '.amazonaws.com'; + +const parseURL = ( + url: string, +): { path: string; bucket: string; region: string } => { + let { host, pathname } = new URL(url); + + pathname = pathname.substr(1); + + const validHost = new RegExp( + /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, + ); + if (!validHost.test(host)) { + throw new Error(`not a valid AWS S3 URL: ${url}`); + } + + const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); + host = host.substring(bucket.length); + const [, , region, ,] = host.split('.'); + + return { + path: pathname, + bucket: bucket, + region: region, + }; +}; + +export class AwsS3UrlReader implements UrlReader { + static factory: ReaderFactory = ({ config, logger }) => { + if (!config.has('integrations.awsS3')) { + return []; + } + const awsS3Config = readAwsS3IntegrationConfig( + config.getConfig('integrations.awsS3'), + ); + let s3: S3; + if (!awsS3Config.accessKeyId || !awsS3Config.secretAccessKey) { + logger.info( + 'awsS3 credentials not found in config. Using default credentials provider.', + ); + s3 = new S3({}); + } else { + const creds = new Credentials({ + accessKeyId: awsS3Config.accessKeyId, + secretAccessKey: awsS3Config.secretAccessKey, + }); + s3 = new S3({ + apiVersion: '2006-03-01', + credentials: creds, + }); + } + const reader = new AwsS3UrlReader(awsS3Config, s3); + const predicate = (url: URL) => url.host.endsWith(AMAZON_AWS_HOST); + return [{ reader, predicate }]; + }; + + constructor( + private readonly integration: AwsS3IntegrationConfig, + private readonly s3: S3, + ) {} + + async read(url: string): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + const params = { + Bucket: bucket, + Key: path, + }; + return await getRawBody(this.s3.getObject(params).createReadStream()); + } catch (e) { + throw new Error(`Could not retrieve file from S3: ${e.message}`); + } + } + + async readTree(): Promise { + throw new Error('AwsS3Reader does not implement search'); + } + + async search(): Promise { + throw new Error('AwsS3Reader does not implement search'); + } + + toString() { + const secretAccessKey = this.integration.secretAccessKey; + return `awsS3{host=${AMAZON_AWS_HOST},authed=${Boolean(secretAccessKey)}}`; + } +} diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2b3a2f166c..f9a4865717 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -25,6 +25,7 @@ import { GitlabUrlReader } from './GitlabUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import { FetchUrlReader } from './FetchUrlReader'; import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; type CreateOptions = { /** Root config object */ @@ -74,6 +75,7 @@ export class UrlReaders { GithubUrlReader.factory, GitlabUrlReader.factory, GoogleGcsUrlReader.factory, + AwsS3UrlReader.factory, FetchUrlReader.factory, ]), }); diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 52663f55fa..88b679e7bc 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -6,6 +6,14 @@ import { Config } from '@backstage/config'; import { RestEndpointMethodTypes } from '@octokit/rest'; +// Warning: (ae-missing-release-tag) "AwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type AwsS3IntegrationConfig = { + accessKeyId?: string; + secretAccessKey?: string; +}; + // Warning: (ae-missing-release-tag) "AzureIntegration" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -314,6 +322,14 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; +// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function readAwsS3IntegrationConfig( + config: Config, +): AwsS3IntegrationConfig; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAzureIntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/packages/integration/src/awsS3/config.tests.ts b/packages/integration/src/awsS3/config.tests.ts new file mode 100644 index 0000000000..f3fd2bf7b1 --- /dev/null +++ b/packages/integration/src/awsS3/config.tests.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { AwsS3IntegrationConfig, readAwsS3IntegrationConfig } from './config'; + +describe('readAwsS3IntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return new ConfigReader(data); + } + + it('reads all values', () => { + const output = readAwsS3IntegrationConfig( + buildConfig({ + accessKeyId: 'fake-key', + secretAccessKey: 'fake-secret-key', + }), + ); + expect(output).toEqual({ + accessKeyId: 'fake-key', + secretAccessKey: 'fake-secret-key', + }); + }); + + it('does not fail when config is not set', () => { + const output = readAwsS3IntegrationConfig(buildConfig({})); + expect(output).toEqual({}); + }); +}); diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts new file mode 100644 index 0000000000..f896bbbef2 --- /dev/null +++ b/packages/integration/src/awsS3/config.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +/** + * The configuration parameters for a single AWS S3 provider. + */ + +export type AwsS3IntegrationConfig = { + /** + * accessKeyId + */ + accessKeyId?: string; + + /** + * secretAccessKey + */ + secretAccessKey?: string; +}; + +/** + * Reads a single Aws S3 integration config. + * + * @param config The config object of a single integration + */ + +export function readAwsS3IntegrationConfig( + config: Config, +): AwsS3IntegrationConfig { + if (!config) { + return {}; + } + + if (!config.has('accessKeyId') && !config.has('secretAccessKey')) { + return {}; + } + + const accessKeyId = config.getString('accessKeyId'); + + const secretAccessKey = config.getString('secretAccessKey'); + + return { accessKeyId: accessKeyId, secretAccessKey: secretAccessKey }; +} diff --git a/packages/integration/src/awsS3/index.ts b/packages/integration/src/awsS3/index.ts new file mode 100644 index 0000000000..27991a8327 --- /dev/null +++ b/packages/integration/src/awsS3/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { readAwsS3IntegrationConfig } from './config'; +export type { AwsS3IntegrationConfig } from './config'; diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index bf88cf9751..9bbbab9fce 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -19,6 +19,7 @@ export * from './bitbucket'; export * from './github'; export * from './gitlab'; export * from './googleGcs'; +export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; diff --git a/yarn.lock b/yarn.lock index d14fac46fa..95ef2e6f38 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9004,6 +9004,21 @@ aws-sdk@^2.948.0: uuid "3.3.2" xml2js "0.4.19" +aws-sdk@^2.952.0: + version "2.952.0" + resolved "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.952.0.tgz#cfbdf2c4685aed17165f4df8760759cd7659a597" + integrity sha512-FZkmOWAyDSQMeD8iioeoSW873ZjPXLfGejr0gNi8kQB7JrllOayPaexpq70aT+7n5bAzArjSIH8OAB+BoHYijA== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"