diff --git a/.changeset/wicked-spoons-perform.md b/.changeset/wicked-spoons-perform.md new file mode 100644 index 0000000000..61c489df13 --- /dev/null +++ b/.changeset/wicked-spoons-perform.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Add AWS S3 URL Reader diff --git a/app-config.yaml b/app-config.yaml index c1d0121dff..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -160,9 +160,13 @@ integrations: azure: - host: dev.azure.com token: ${AZURE_TOKEN} -# googleGcs: -# clientEmail: 'example@example.com' -# privateKey: ${GCS_PRIVATE_KEY} + # googleGcs: + # clientEmail: 'example@example.com' + # privateKey: ${GCS_PRIVATE_KEY} + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} catalog: rules: diff --git a/docs/integrations/aws-s3/locations.md b/docs/integrations/aws-s3/locations.md new file mode 100644 index 0000000000..fe1c6050c2 --- /dev/null +++ b/docs/integrations/aws-s3/locations.md @@ -0,0 +1,42 @@ +--- +id: locations +sidebar_label: Locations +title: Amazon Web Services S3 Locations +# prettier-ignore +description: Setting up an integration with Amazon Web Services S3 +--- + +The AWS S3 integration supports loading catalog entities from an S3 Bucket. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +To use this integration, add configuration to your `app-config.yaml`: + +```yaml +integrations: + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} +``` + +Then make sure the environment variables `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` are set when you run Backstage. + +Users with multiple AWS accounts may want to use a role for S3 storage that is +in a different AWS account. Using the `roleArn` parameter as seen below, you can +instruct the AWS S3 reader to assume a role before accessing S3: + +```yaml +integrations: + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} + roleArn: 'arn:aws:iam::xxxxxxxxxxxx:role/example-role' +``` diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c22a7e961d..d787fb28aa 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -104,6 +104,11 @@ ], "Integrations": [ "integrations/index", + { + "type": "subcategory", + "label": "AWS S3", + "ids": ["integrations/aws-s3/locations"] + }, { "type": "subcategory", "label": "Azure", diff --git a/mkdocs.yml b/mkdocs.yml index bd70104abc..73ad52341e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,8 @@ nav: - FAQ: 'features/techdocs/FAQ.md' - Integrations: - Overview: 'integrations/index.md' + - AWS S3: + - Locations: 'integrations/aws-s3/locations.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 4c7aacdf96..6d25ece793 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -40,6 +40,7 @@ "@types/dockerode": "^3.2.1", "@types/express": "^4.17.6", "archiver": "^5.0.2", + "aws-sdk": "^2.840.0", "compression": "^1.7.4", "concat-stream": "^2.0.0", "cors": "^2.8.5", @@ -92,6 +93,7 @@ "@types/tar": "^4.0.3", "@types/unzipper": "^0.10.3", "@types/webpack-env": "^1.15.2", + "aws-sdk-mock": "^5.2.1", "get-port": "^5.1.1", "http-errors": "^1.7.3", "jest": "^26.0.1", diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts new file mode 100644 index 0000000000..fe45931d53 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -0,0 +1,232 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ConfigReader, JsonObject } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { DefaultReadTreeResponseFactory } from './tree'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; +import { + AwsS3Integration, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; +import { UrlReaderPredicateTuple } from './types'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +describe('AwsS3UrlReader', () => { + const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { + return AwsS3UrlReader.factory({ + config: new ConfigReader(config), + logger: getVoidLogger(), + treeResponseFactory: DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), + }), + }); + }; + + afterEach(() => { + AWSMock.restore(); + }); + + it('creates a dummy reader without the awsS3 field', () => { + const entries = createReader({ + integrations: {}, + }); + + expect(entries).toHaveLength(1); + }); + + it('creates a reader with credentials correctly configured', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + accessKeyId: 'fakekey', + secretAccessKey: 'fakekey', + }); + + const entries = createReader({ + integrations: { + awsS3: awsS3Integrations, + }, + }); + + expect(entries).toHaveLength(1); + }); + + it('creates a reader with default credentials provider', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + }); + + const entries = createReader({ + integrations: { + awsS3: awsS3Integrations, + }, + }); + + expect(entries).toHaveLength(1); + }); + + describe('predicates', () => { + const readers = createReader({ + integrations: { + awsS3: [{}], + }, + }); + const predicate = readers[0].predicate; + + it('returns true for the correct aws s3 storage host', () => { + expect( + predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')), + ).toBe(true); + }); + + it('returns true for a url with the full path and the correct host', () => { + expect( + predicate( + new URL( + 'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + + it('returns false for an incorrect host', () => { + expect(predicate(new URL('https://amazon.com'))).toBe(false); + }); + + it('returns false for a completely different host', () => { + expect(predicate(new URL('https://storage.cloud.google.com'))).toBe( + false, + ); + }); + + it("returns true for a url with a bucket with '.'", () => { + expect( + predicate( + new URL( + 'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + }); + + describe('read', () => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + ); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.read( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + expect(response.toString()).toBe('site_name: Test\n'); + }); + + it('rejects unknown targets', async () => { + await expect( + awsS3UrlReader.read( + 'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml', + ), + ).rejects.toThrow( + Error( + `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + ), + ); + }); + }); + + describe('readUrl', () => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + ); + + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + ); + const buffer = await response.buffer(); + expect(buffer.toString()).toBe('site_name: Test\n'); + }); + + it('rejects unknown targets', async () => { + await expect( + awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml', + ), + ).rejects.toThrow( + Error( + `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`, + ), + ); + }); + }); +}); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts new file mode 100644 index 0000000000..d119115930 --- /dev/null +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -0,0 +1,175 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import aws, { Credentials, S3 } from 'aws-sdk'; +import { CredentialsOptions } from 'aws-sdk/lib/credentials'; +import { + ReaderFactory, + ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, + SearchResponse, + UrlReader, +} from './types'; +import getRawBody from 'raw-body'; +import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; + +const parseURL = ( + url: string, +): { path: string; bucket: string; region: string } => { + let { host, pathname } = new URL(url); + + /** + * Removes the leading '/' from the pathname to be processed + * as a parameter by AWS S3 SDK getObject method. + */ + pathname = pathname.substr(1); + + /** + * Checks that the given URL is a valid S3 object url. + * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname + */ + const validHost = new RegExp( + /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/, + ); + if (!validHost.test(host)) { + throw new Error(`not a valid AWS S3 URL: ${url}`); + } + + const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/); + host = host.substring(bucket.length); + const [, , region, ,] = host.split('.'); + + return { + path: pathname, + bucket: bucket, + region: region, + }; +}; + +export class AwsS3UrlReader implements UrlReader { + static factory: ReaderFactory = ({ config }) => { + const integrations = ScmIntegrations.fromConfig(config); + + return integrations.awsS3.list().map(integration => { + const creds = AwsS3UrlReader.buildCredentials(integration); + const s3 = new S3({ + apiVersion: '2006-03-01', + credentials: creds, + }); + const reader = new AwsS3UrlReader(integration, s3); + const predicate = (url: URL) => + url.host.endsWith(integration.config.host); + return { reader, predicate }; + }); + }; + + constructor( + private readonly integration: AwsS3Integration, + private readonly s3: S3, + ) {} + + /** + * If accesKeyId and secretAccessKey are missing, the standard credentials provider chain will be used: + * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html + */ + private static buildCredentials( + integration?: AwsS3Integration, + ): Credentials | CredentialsOptions | undefined { + if (!integration) { + return undefined; + } + + const accessKeyId = integration.config.accessKeyId; + const secretAccessKey = integration.config.secretAccessKey; + let explicitCredentials: Credentials | undefined; + + if (accessKeyId && secretAccessKey) { + explicitCredentials = new Credentials({ + accessKeyId, + secretAccessKey, + }); + } + + const roleArn = integration.config.roleArn; + if (roleArn) { + return new aws.ChainableTemporaryCredentials({ + masterCredentials: explicitCredentials, + params: { + RoleSessionName: 'backstage-aws-s3-url-reader', + RoleArn: roleArn, + }, + }); + } + + return explicitCredentials; + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + let params; + if (options?.etag) { + params = { + Bucket: bucket, + Key: path, + IfNoneMatch: options.etag, + }; + } else { + params = { + Bucket: bucket, + Key: path, + }; + } + + const response = this.s3.getObject(params); + const buffer = await getRawBody(response.createReadStream()); + const etag = (await response.promise()).ETag; + + return { + buffer: async () => buffer, + etag: etag, + }; + } catch (e) { + throw new Error(`Could not retrieve file from S3: ${e.message}`); + } + } + + async readTree(): Promise { + throw new Error('AwsS3Reader does not implement readTree'); + } + + async search(): Promise { + throw new Error('AwsS3Reader does not implement search'); + } + + toString() { + const secretAccessKey = this.integration.config.secretAccessKey; + return `awsS3{host=${this.integration.config.host},authed=${Boolean( + secretAccessKey, + )}}`; + } +} diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2b3a2f166c..f9a4865717 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -25,6 +25,7 @@ import { GitlabUrlReader } from './GitlabUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import { FetchUrlReader } from './FetchUrlReader'; import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; +import { AwsS3UrlReader } from './AwsS3UrlReader'; type CreateOptions = { /** Root config object */ @@ -74,6 +75,7 @@ export class UrlReaders { GithubUrlReader.factory, GitlabUrlReader.factory, GoogleGcsUrlReader.factory, + AwsS3UrlReader.factory, FetchUrlReader.factory, ]), }); diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml @@ -0,0 +1 @@ +site_name: Test diff --git a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts index 869ef83a86..aa291c2815 100644 --- a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts +++ b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts @@ -26,6 +26,6 @@ describe('scmIntegrationsApiRef', () => { it('should be instantiated', () => { const i = ScmIntegrationsApi.fromConfig(new ConfigReader({})); - expect(i.list().length).toBe(4); // The default ones + expect(i.list().length).toBe(5); // The default ones }); }); diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 52663f55fa..832064eebc 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -6,6 +6,41 @@ import { Config } from '@backstage/config'; import { RestEndpointMethodTypes } from '@octokit/rest'; +// Warning: (ae-missing-release-tag) "AwsS3Integration" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3Integration implements ScmIntegration { + constructor(integrationConfig: AwsS3IntegrationConfig); + // (undocumented) + get config(): AwsS3IntegrationConfig; + // Warning: (ae-forgotten-export) The symbol "ScmIntegrationsFactory" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static factory: ScmIntegrationsFactory; + // (undocumented) + resolveEditUrl(url: string): string; + // (undocumented) + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string; + // (undocumented) + get title(): string; + // (undocumented) + get type(): string; +} + +// Warning: (ae-missing-release-tag) "AwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type AwsS3IntegrationConfig = { + host: string; + accessKeyId?: string; + secretAccessKey?: string; + roleArn?: string; +}; + // Warning: (ae-missing-release-tag) "AzureIntegration" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -13,8 +48,6 @@ export class AzureIntegration implements ScmIntegration { constructor(integrationConfig: AzureIntegrationConfig); // (undocumented) get config(): AzureIntegrationConfig; - // Warning: (ae-forgotten-export) The symbol "ScmIntegrationsFactory" needs to be exported by the entry point index.d.ts - // // (undocumented) static factory: ScmIntegrationsFactory; // (undocumented) @@ -314,6 +347,21 @@ export type GoogleGcsIntegrationConfig = { privateKey?: string; }; +// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen +// Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export function readAwsS3IntegrationConfig( + config: Config, +): AwsS3IntegrationConfig; + +// Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfigs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export function readAwsS3IntegrationConfigs( + configs: Config[], +): AwsS3IntegrationConfig[]; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // Warning: (ae-missing-release-tag) "readAzureIntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -410,6 +458,8 @@ export interface ScmIntegration { export interface ScmIntegrationRegistry extends ScmIntegrationsGroup { // (undocumented) + awsS3: ScmIntegrationsGroup; + // (undocumented) azure: ScmIntegrationsGroup; // (undocumented) bitbucket: ScmIntegrationsGroup; @@ -436,6 +486,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry { // Warning: (ae-forgotten-export) The symbol "IntegrationsByType" needs to be exported by the entry point index.d.ts constructor(integrationsByType: IntegrationsByType); // (undocumented) + get awsS3(): ScmIntegrationsGroup; + // (undocumented) get azure(): ScmIntegrationsGroup; // (undocumented) get bitbucket(): ScmIntegrationsGroup; diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts index 501373de53..90cc10f77b 100644 --- a/packages/integration/config.d.ts +++ b/packages/integration/config.d.ts @@ -163,5 +163,30 @@ export interface Config { */ privateKey?: string; }; + + /** Integration configuration for AWS S3 Service */ + awsS3?: Array<{ + /** + * The host of the target that this matches on, e.g. "amazonaws.com". + * @visibility frontend + */ + host: string; + /** + * Account access key used to authenticate requests. + * @visibility backend + */ + accessKeyId?: string; + /** + * Account secret key used to authenticate requests. + * @visibility secret + */ + secretAccessKey?: string; + + /** + * ARN of the role to be assumed + * @visibility backend + */ + roleArn?: string; + }>; }; } diff --git a/packages/integration/src/ScmIntegrations.test.ts b/packages/integration/src/ScmIntegrations.test.ts index a21337b45d..f0a6e5d72d 100644 --- a/packages/integration/src/ScmIntegrations.test.ts +++ b/packages/integration/src/ScmIntegrations.test.ts @@ -13,7 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { AwsS3IntegrationConfig } from './awsS3'; +import { AwsS3Integration } from './awsS3/AwsS3Integration'; import { AzureIntegrationConfig } from './azure'; import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegrationConfig } from './bitbucket'; @@ -26,6 +27,10 @@ import { basicIntegrations } from './helpers'; import { ScmIntegrations } from './ScmIntegrations'; describe('ScmIntegrations', () => { + const awsS3 = new AwsS3Integration({ + host: 'awss3.local', + } as AwsS3IntegrationConfig); + const azure = new AzureIntegration({ host: 'azure.local', } as AzureIntegrationConfig); @@ -43,6 +48,7 @@ describe('ScmIntegrations', () => { } as GitLabIntegrationConfig); const i = new ScmIntegrations({ + awsS3: basicIntegrations([awsS3], item => item.config.host), azure: basicIntegrations([azure], item => item.config.host), bitbucket: basicIntegrations([bitbucket], item => item.config.host), github: basicIntegrations([github], item => item.config.host), @@ -50,6 +56,7 @@ describe('ScmIntegrations', () => { }); it('can get the specifics', () => { + expect(i.awsS3.byUrl('https://awss3.local')).toBe(awsS3); expect(i.azure.byUrl('https://azure.local')).toBe(azure); expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.github.byUrl('https://github.local')).toBe(github); @@ -58,16 +65,18 @@ describe('ScmIntegrations', () => { it('can list', () => { expect(i.list()).toEqual( - expect.arrayContaining([azure, bitbucket, github, gitlab]), + expect.arrayContaining([awsS3, azure, bitbucket, github, gitlab]), ); }); it('can select by url and host', () => { + expect(i.byUrl('https://awss3.local')).toBe(awsS3); expect(i.byUrl('https://azure.local')).toBe(azure); expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.byUrl('https://github.local')).toBe(github); expect(i.byUrl('https://gitlab.local')).toBe(gitlab); + expect(i.byHost('awss3.local')).toBe(awsS3); expect(i.byHost('azure.local')).toBe(azure); expect(i.byHost('bitbucket.local')).toBe(bitbucket); expect(i.byHost('github.local')).toBe(github); diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index d5548b0ce8..983ab7d24b 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -15,6 +15,7 @@ */ import { Config } from '@backstage/config'; +import { AwsS3Integration } from './awsS3/AwsS3Integration'; import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegration } from './bitbucket/BitbucketIntegration'; import { GitHubIntegration } from './github/GitHubIntegration'; @@ -24,6 +25,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types'; import { ScmIntegrationRegistry } from './registry'; type IntegrationsByType = { + awsS3: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; bitbucket: ScmIntegrationsGroup; github: ScmIntegrationsGroup; @@ -35,6 +37,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry { static fromConfig(config: Config): ScmIntegrations { return new ScmIntegrations({ + awsS3: AwsS3Integration.factory({ config }), azure: AzureIntegration.factory({ config }), bitbucket: BitbucketIntegration.factory({ config }), github: GitHubIntegration.factory({ config }), @@ -46,6 +49,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry { this.byType = integrationsByType; } + get awsS3(): ScmIntegrationsGroup { + return this.byType.awsS3; + } + get azure(): ScmIntegrationsGroup { return this.byType.azure; } diff --git a/packages/integration/src/awsS3/AwsS3Integration.test.ts b/packages/integration/src/awsS3/AwsS3Integration.test.ts new file mode 100644 index 0000000000..481f3c1951 --- /dev/null +++ b/packages/integration/src/awsS3/AwsS3Integration.test.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { AwsS3Integration } from './AwsS3Integration'; + +describe('AwsS3Integration', () => { + it('has a working factory', () => { + const integrations = AwsS3Integration.factory({ + config: new ConfigReader({ + integrations: { + awsS3: [ + { + host: 'a.com', + accessKeyId: 'access key', + secretAccessKey: 'secret key', + }, + ], + }, + }), + }); + expect(integrations.list().length).toBe(2); // including default + expect(integrations.list()[0].config.host).toBe('a.com'); + expect(integrations.list()[1].config.host).toBe('amazonaws.com'); + }); + + it('returns the basics', () => { + const integration = new AwsS3Integration({ host: 'a.com' } as any); + expect(integration.type).toBe('awsS3'); + expect(integration.title).toBe('a.com'); + }); + + describe('resolveUrl', () => { + it('works for valid urls', () => { + const integration = new AwsS3Integration({ + host: 'amazonaws.com', + } as any); + + expect( + integration.resolveUrl({ + url: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml', + base: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml', + }), + ).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml'); + }); + }); + + it('resolve edit URL', () => { + const integration = new AwsS3Integration({ host: 'a.com' } as any); + + // TODO: The Aws S3 integration doesn't support resolving an edit URL, + // instead we keep the input URL. + expect( + integration.resolveEditUrl( + 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml', + ), + ).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml'); + }); +}); diff --git a/packages/integration/src/awsS3/AwsS3Integration.ts b/packages/integration/src/awsS3/AwsS3Integration.ts new file mode 100644 index 0000000000..e987b8f7a3 --- /dev/null +++ b/packages/integration/src/awsS3/AwsS3Integration.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; +import { ScmIntegration, ScmIntegrationsFactory } from '../types'; +import { AwsS3IntegrationConfig, readAwsS3IntegrationConfigs } from './config'; + +export class AwsS3Integration implements ScmIntegration { + static factory: ScmIntegrationsFactory = ({ config }) => { + const configs = readAwsS3IntegrationConfigs( + config.getOptionalConfigArray('integrations.awsS3') ?? [], + ); + return basicIntegrations( + configs.map(c => new AwsS3Integration(c)), + i => i.config.host, + ); + }; + + get type(): string { + return 'awsS3'; + } + + get title(): string { + return this.integrationConfig.host; + } + + get config(): AwsS3IntegrationConfig { + return this.integrationConfig; + } + + constructor(private readonly integrationConfig: AwsS3IntegrationConfig) {} + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string { + const resolved = defaultScmResolveUrl(options); + return resolved; + } + + resolveEditUrl(url: string): string { + // TODO: Implement edit URL for awsS3 + return url; + } +} diff --git a/packages/integration/src/awsS3/config.test.ts b/packages/integration/src/awsS3/config.test.ts new file mode 100644 index 0000000000..1c7d7d1647 --- /dev/null +++ b/packages/integration/src/awsS3/config.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + AwsS3IntegrationConfig, + readAwsS3IntegrationConfig, + readAwsS3IntegrationConfigs, +} from './config'; + +describe('readAwsS3IntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return new ConfigReader(data); + } + + it('reads all values', () => { + const output = readAwsS3IntegrationConfig( + buildConfig({ + host: 'amazonaws.com', + accessKeyId: 'fake-key', + secretAccessKey: 'fake-secret-key', + }), + ); + expect(output).toEqual({ + host: 'amazonaws.com', + accessKeyId: 'fake-key', + secretAccessKey: 'fake-secret-key', + }); + }); +}); + +describe('readAwsS3IntegrationConfigs', () => { + function buildConfig(data: Partial[]): Config[] { + return data.map(item => new ConfigReader(item)); + } + + it('reads all values', () => { + const output = readAwsS3IntegrationConfigs( + buildConfig([ + { + host: 'amazonaws.com', + accessKeyId: 'key', + secretAccessKey: 'secret', + }, + ]), + ); + expect(output).toContainEqual({ + host: 'amazonaws.com', + accessKeyId: 'key', + secretAccessKey: 'secret', + }); + }); + + it('adds a default entry when missing', () => { + const output = readAwsS3IntegrationConfigs(buildConfig([])); + expect(output).toEqual([ + { + host: 'amazonaws.com', + }, + ]); + }); +}); diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts new file mode 100644 index 0000000000..5f9cd46a22 --- /dev/null +++ b/packages/integration/src/awsS3/config.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { isValidHost } from '../helpers'; + +const AMAZON_AWS_HOST = 'amazonaws.com'; + +/** + * The configuration parameters for a single AWS S3 provider. + */ + +export type AwsS3IntegrationConfig = { + /** + * The host of the target that this matches on, e.g. "amazonaws.com" + * + * Currently only "amazonaws.com" is supported. + */ + host: string; + + /** + * accessKeyId + */ + accessKeyId?: string; + + /** + * secretAccessKey + */ + secretAccessKey?: string; + + /** + * roleArn + */ + roleArn?: string; +}; + +/** + * Reads a single Aws S3 integration config. + * + * @param config The config object of a single integration + */ + +export function readAwsS3IntegrationConfig( + config: Config, +): AwsS3IntegrationConfig { + const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST; + const accessKeyId = config.getOptionalString('accessKeyId'); + const secretAccessKey = config.getOptionalString('secretAccessKey'); + const roleArn = config.getOptionalString('roleArn'); + + if (!isValidHost(host)) { + throw new Error( + `Invalid awsS3 integration config, '${host}' is not a valid host`, + ); + } + + return { host, accessKeyId, secretAccessKey, roleArn }; +} + +export function readAwsS3IntegrationConfigs( + configs: Config[], +): AwsS3IntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readAwsS3IntegrationConfig); + + // If no explicit amazonaws.com integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === AMAZON_AWS_HOST)) { + result.push({ + host: AMAZON_AWS_HOST, + }); + } + return result; +} diff --git a/packages/integration/src/awsS3/index.ts b/packages/integration/src/awsS3/index.ts new file mode 100644 index 0000000000..1ae3964ec1 --- /dev/null +++ b/packages/integration/src/awsS3/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { AwsS3Integration } from './AwsS3Integration'; +export { + readAwsS3IntegrationConfig, + readAwsS3IntegrationConfigs, +} from './config'; +export type { AwsS3IntegrationConfig } from './config'; diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index bf88cf9751..9bbbab9fce 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -19,6 +19,7 @@ export * from './bitbucket'; export * from './github'; export * from './gitlab'; export * from './googleGcs'; +export * from './awsS3'; export { defaultScmResolveUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { ScmIntegration, ScmIntegrationsGroup } from './types'; diff --git a/packages/integration/src/registry.ts b/packages/integration/src/registry.ts index da205ae1e7..21ca472ccb 100644 --- a/packages/integration/src/registry.ts +++ b/packages/integration/src/registry.ts @@ -15,6 +15,7 @@ */ import { ScmIntegration, ScmIntegrationsGroup } from './types'; +import { AwsS3Integration } from './awsS3/AwsS3Integration'; import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegration } from './bitbucket/BitbucketIntegration'; import { GitHubIntegration } from './github/GitHubIntegration'; @@ -25,6 +26,7 @@ import { GitLabIntegration } from './gitlab/GitLabIntegration'; */ export interface ScmIntegrationRegistry extends ScmIntegrationsGroup { + awsS3: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; bitbucket: ScmIntegrationsGroup; github: ScmIntegrationsGroup;