diff --git a/app-config.yaml b/app-config.yaml index 7488cd8648..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -163,9 +163,10 @@ integrations: # googleGcs: # clientEmail: 'example@example.com' # privateKey: ${GCS_PRIVATE_KEY} - # awsS3: - # accessKeyId: ${AWS_ACCESS_KEY_ID} - # secretAccessKey: ${AWS_SECRET_ACCESS_KEY} + awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} + secretAccessKey: ${AWS_SECRET_ACCESS_KEY} catalog: rules: diff --git a/docs/integrations/aws-s3/locations.md b/docs/integrations/aws-s3/locations.md index bd352e71fb..39ceb9e7aa 100644 --- a/docs/integrations/aws-s3/locations.md +++ b/docs/integrations/aws-s3/locations.md @@ -18,9 +18,9 @@ plugin. To use this integration, add configuration to your root `app-config.yaml`: ```yaml -integrations: - awsS3: - accessKeyID: ${AWS_ACCESS_KEY_ID} +awsS3: + - host: amazonaws.com + accessKeyId: ${AWS_ACCESS_KEY_ID} secretAccessKey: ${AWS_SECRET_ACCESS_KEY} ``` diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 7ab43cd8cc..0977e560da 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -30,38 +30,45 @@ describe('AwsS3UrlReader', () => { }); }; - it('does not create a reader without the awsS3 field', () => { + it('creates a dummy reader without the awsS3 field', () => { const entries = createReader({ integrations: {}, }); - expect(entries).toHaveLength(0); + expect(entries).toHaveLength(1); }); it('creates a reader with credentials correctly configured', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + accessKeyId: 'fakekey', + secretAccessKey: 'fakekey', + }); const entries = createReader({ integrations: { - awsS3: { - accessKeyId: 'fakekey', - secretAccessKey: 'fakekey', - }, + awsS3: awsS3Integrations, }, }); - expect(entries).toHaveLength(1); + expect(entries).toHaveLength(2); }); it('creates a reader with default credentials provider', () => { + const awsS3Integrations = []; + awsS3Integrations.push({ + host: 'amazonaws.com', + }); const entries = createReader({ integrations: { - awsS3: {}, + awsS3: awsS3Integrations, }, }); - expect(entries).toHaveLength(1); + expect(entries).toHaveLength(2); }); describe('predicates', () => { const readers = createReader({ integrations: { - awsS3: {}, + awsS3: [{}], }, }); const predicate = readers[0].predicate; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 0c85a3ede9..113ebeb587 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -24,12 +24,7 @@ import { UrlReader, } from './types'; import getRawBody from 'raw-body'; -import { - AwsS3IntegrationConfig, - readAwsS3IntegrationConfig, -} from '@backstage/integration'; - -const AMAZON_AWS_HOST = '.amazonaws.com'; +import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; const parseURL = ( url: string, @@ -66,35 +61,37 @@ const parseURL = ( export class AwsS3UrlReader implements UrlReader { static factory: ReaderFactory = ({ config, logger }) => { - if (!config.has('integrations.awsS3')) { - return []; - } - const awsS3Config = readAwsS3IntegrationConfig( - config.getConfig('integrations.awsS3'), - ); - let s3: S3; - if (!awsS3Config.accessKeyId || !awsS3Config.secretAccessKey) { - logger.debug( - 'integrations.awsS3 not found in app config. AWS S3 integration will use default AWS credentials if set in environment.', - ); - s3 = new S3({}); - } else { - const creds = new Credentials({ - accessKeyId: awsS3Config.accessKeyId, - secretAccessKey: awsS3Config.secretAccessKey, - }); - s3 = new S3({ - apiVersion: '2006-03-01', - credentials: creds, - }); - } - const reader = new AwsS3UrlReader(awsS3Config, s3); - const predicate = (url: URL) => url.host.endsWith(AMAZON_AWS_HOST); - return [{ reader, predicate }]; + const integrations = ScmIntegrations.fromConfig(config); + + return integrations.awsS3.list().map(integration => { + let s3: S3; + if ( + !integration.config.accessKeyId || + !integration.config.secretAccessKey + ) { + logger.info( + 'awsS3 credentials not found in config. Using default credentials provider.', + ); + s3 = new S3({}); + } else { + const creds = new Credentials({ + accessKeyId: integration.config.accessKeyId, + secretAccessKey: integration.config.secretAccessKey, + }); + s3 = new S3({ + apiVersion: '2006-03-01', + credentials: creds, + }); + } + const reader = new AwsS3UrlReader(integration, s3); + const predicate = (url: URL) => + url.host.endsWith(integration.config.host); + return { reader, predicate }; + }); }; constructor( - private readonly integration: AwsS3IntegrationConfig, + private readonly integration: AwsS3Integration, private readonly s3: S3, ) {} @@ -147,7 +144,9 @@ export class AwsS3UrlReader implements UrlReader { } toString() { - const secretAccessKey = this.integration.secretAccessKey; - return `awsS3{host=${AMAZON_AWS_HOST},authed=${Boolean(secretAccessKey)}}`; + const secretAccessKey = this.integration.config.secretAccessKey; + return `awsS3{host=${this.integration.config.host},authed=${Boolean( + secretAccessKey, + )}}`; } } diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts index d4721776d7..e2ae193deb 100644 --- a/packages/integration/config.d.ts +++ b/packages/integration/config.d.ts @@ -165,7 +165,14 @@ export interface Config { }; /** Integration configuration for AWS S3 Service */ - awsS3?: { + awsS3?: Array<{ + /** + * The host of the target that this matches on, e.g. "amazonaws.com". + * + * @visibility frontend + */ + host: string; + /** /** * Account access key used to authenticate requests. * @visibility backend @@ -176,6 +183,6 @@ export interface Config { * @visibility secret */ secretAccessKey?: string; - }; + }>; }; } diff --git a/packages/integration/src/ScmIntegrations.test.ts b/packages/integration/src/ScmIntegrations.test.ts index a21337b45d..f0a6e5d72d 100644 --- a/packages/integration/src/ScmIntegrations.test.ts +++ b/packages/integration/src/ScmIntegrations.test.ts @@ -13,7 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { AwsS3IntegrationConfig } from './awsS3'; +import { AwsS3Integration } from './awsS3/AwsS3Integration'; import { AzureIntegrationConfig } from './azure'; import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegrationConfig } from './bitbucket'; @@ -26,6 +27,10 @@ import { basicIntegrations } from './helpers'; import { ScmIntegrations } from './ScmIntegrations'; describe('ScmIntegrations', () => { + const awsS3 = new AwsS3Integration({ + host: 'awss3.local', + } as AwsS3IntegrationConfig); + const azure = new AzureIntegration({ host: 'azure.local', } as AzureIntegrationConfig); @@ -43,6 +48,7 @@ describe('ScmIntegrations', () => { } as GitLabIntegrationConfig); const i = new ScmIntegrations({ + awsS3: basicIntegrations([awsS3], item => item.config.host), azure: basicIntegrations([azure], item => item.config.host), bitbucket: basicIntegrations([bitbucket], item => item.config.host), github: basicIntegrations([github], item => item.config.host), @@ -50,6 +56,7 @@ describe('ScmIntegrations', () => { }); it('can get the specifics', () => { + expect(i.awsS3.byUrl('https://awss3.local')).toBe(awsS3); expect(i.azure.byUrl('https://azure.local')).toBe(azure); expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.github.byUrl('https://github.local')).toBe(github); @@ -58,16 +65,18 @@ describe('ScmIntegrations', () => { it('can list', () => { expect(i.list()).toEqual( - expect.arrayContaining([azure, bitbucket, github, gitlab]), + expect.arrayContaining([awsS3, azure, bitbucket, github, gitlab]), ); }); it('can select by url and host', () => { + expect(i.byUrl('https://awss3.local')).toBe(awsS3); expect(i.byUrl('https://azure.local')).toBe(azure); expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.byUrl('https://github.local')).toBe(github); expect(i.byUrl('https://gitlab.local')).toBe(gitlab); + expect(i.byHost('awss3.local')).toBe(awsS3); expect(i.byHost('azure.local')).toBe(azure); expect(i.byHost('bitbucket.local')).toBe(bitbucket); expect(i.byHost('github.local')).toBe(github); diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index d5548b0ce8..983ab7d24b 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -15,6 +15,7 @@ */ import { Config } from '@backstage/config'; +import { AwsS3Integration } from './awsS3/AwsS3Integration'; import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegration } from './bitbucket/BitbucketIntegration'; import { GitHubIntegration } from './github/GitHubIntegration'; @@ -24,6 +25,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types'; import { ScmIntegrationRegistry } from './registry'; type IntegrationsByType = { + awsS3: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; bitbucket: ScmIntegrationsGroup; github: ScmIntegrationsGroup; @@ -35,6 +37,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry { static fromConfig(config: Config): ScmIntegrations { return new ScmIntegrations({ + awsS3: AwsS3Integration.factory({ config }), azure: AzureIntegration.factory({ config }), bitbucket: BitbucketIntegration.factory({ config }), github: GitHubIntegration.factory({ config }), @@ -46,6 +49,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry { this.byType = integrationsByType; } + get awsS3(): ScmIntegrationsGroup { + return this.byType.awsS3; + } + get azure(): ScmIntegrationsGroup { return this.byType.azure; } diff --git a/packages/integration/src/awsS3/AwsS3Integration.test.ts b/packages/integration/src/awsS3/AwsS3Integration.test.ts new file mode 100644 index 0000000000..617e75666e --- /dev/null +++ b/packages/integration/src/awsS3/AwsS3Integration.test.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { AwsS3Integration } from './AwsS3Integration'; + +describe('AwsS3Integration', () => { + it('has a working factory', () => { + const integrations = AwsS3Integration.factory({ + config: new ConfigReader({ + integrations: { + awsS3: [ + { + host: 'a.com', + accessKeyId: 'access key', + secretAccessKey: 'secret key', + }, + ], + }, + }), + }); + expect(integrations.list().length).toBe(2); // including default + expect(integrations.list()[0].config.host).toBe('a.com'); + expect(integrations.list()[1].config.host).toBe('.amazonaws.com'); + }); + + it('returns the basics', () => { + const integration = new AwsS3Integration({ host: 'a.com' } as any); + expect(integration.type).toBe('awsS3'); + expect(integration.title).toBe('a.com'); + }); + + describe('resolveUrl', () => { + it('works for valid urls', () => { + const integration = new AwsS3Integration({ + host: 'amazonaws.com', + } as any); + + expect( + integration.resolveUrl({ + url: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml', + base: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml', + }), + ).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml'); + }); + }); + + it('resolve edit URL', () => { + const integration = new AwsS3Integration({ host: 'a.com' } as any); + + // TODO: The Aws S3 integration doesn't support resolving an edit URL, + // instead we keep the input URL. + expect( + integration.resolveEditUrl( + 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml', + ), + ).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml'); + }); +}); diff --git a/packages/integration/src/awsS3/AwsS3Integration.ts b/packages/integration/src/awsS3/AwsS3Integration.ts new file mode 100644 index 0000000000..e987b8f7a3 --- /dev/null +++ b/packages/integration/src/awsS3/AwsS3Integration.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; +import { ScmIntegration, ScmIntegrationsFactory } from '../types'; +import { AwsS3IntegrationConfig, readAwsS3IntegrationConfigs } from './config'; + +export class AwsS3Integration implements ScmIntegration { + static factory: ScmIntegrationsFactory = ({ config }) => { + const configs = readAwsS3IntegrationConfigs( + config.getOptionalConfigArray('integrations.awsS3') ?? [], + ); + return basicIntegrations( + configs.map(c => new AwsS3Integration(c)), + i => i.config.host, + ); + }; + + get type(): string { + return 'awsS3'; + } + + get title(): string { + return this.integrationConfig.host; + } + + get config(): AwsS3IntegrationConfig { + return this.integrationConfig; + } + + constructor(private readonly integrationConfig: AwsS3IntegrationConfig) {} + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string { + const resolved = defaultScmResolveUrl(options); + return resolved; + } + + resolveEditUrl(url: string): string { + // TODO: Implement edit URL for awsS3 + return url; + } +} diff --git a/packages/integration/src/awsS3/config.tests.ts b/packages/integration/src/awsS3/config.test.ts similarity index 54% rename from packages/integration/src/awsS3/config.tests.ts rename to packages/integration/src/awsS3/config.test.ts index f3fd2bf7b1..d5b9e104ee 100644 --- a/packages/integration/src/awsS3/config.tests.ts +++ b/packages/integration/src/awsS3/config.test.ts @@ -15,7 +15,11 @@ */ import { Config, ConfigReader } from '@backstage/config'; -import { AwsS3IntegrationConfig, readAwsS3IntegrationConfig } from './config'; +import { + AwsS3IntegrationConfig, + readAwsS3IntegrationConfig, + readAwsS3IntegrationConfigs, +} from './config'; describe('readAwsS3IntegrationConfig', () => { function buildConfig(data: Partial): Config { @@ -25,18 +29,47 @@ describe('readAwsS3IntegrationConfig', () => { it('reads all values', () => { const output = readAwsS3IntegrationConfig( buildConfig({ + host: '.amazonaws.com', accessKeyId: 'fake-key', secretAccessKey: 'fake-secret-key', }), ); expect(output).toEqual({ + host: '.amazonaws.com', accessKeyId: 'fake-key', secretAccessKey: 'fake-secret-key', }); }); +}); - it('does not fail when config is not set', () => { - const output = readAwsS3IntegrationConfig(buildConfig({})); - expect(output).toEqual({}); +describe('readAwsS3IntegrationConfigs', () => { + function buildConfig(data: Partial[]): Config[] { + return data.map(item => new ConfigReader(item)); + } + + it('reads all values', () => { + const output = readAwsS3IntegrationConfigs( + buildConfig([ + { + host: '.amazonaws.com', + accessKeyId: 'key', + secretAccessKey: 'secret', + }, + ]), + ); + expect(output).toContainEqual({ + host: '.amazonaws.com', + accessKeyId: 'key', + secretAccessKey: 'secret', + }); + }); + + it('adds a default entry when missing', () => { + const output = readAwsS3IntegrationConfigs(buildConfig([])); + expect(output).toEqual([ + { + host: '.amazonaws.com', + }, + ]); }); }); diff --git a/packages/integration/src/awsS3/config.ts b/packages/integration/src/awsS3/config.ts index 3e52ccb694..e92d1162e2 100644 --- a/packages/integration/src/awsS3/config.ts +++ b/packages/integration/src/awsS3/config.ts @@ -15,12 +15,22 @@ */ import { Config } from '@backstage/config'; +import { isValidHost } from '../helpers'; + +const AMAZON_AWS_HOST = '.amazonaws.com'; /** * The configuration parameters for a single AWS S3 provider. */ export type AwsS3IntegrationConfig = { + /** + * The host of the target that this matches on, e.g. "amazonaws.com" + * + * Currently only "amazonaws.com" is supported. + */ + host: string; + /** * accessKeyId */ @@ -41,17 +51,31 @@ export type AwsS3IntegrationConfig = { export function readAwsS3IntegrationConfig( config: Config, ): AwsS3IntegrationConfig { - if (!config) { - return {}; + const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST; + const accessKeyId = config.getOptionalString('accessKeyId'); + const secretAccessKey = config.getOptionalString('secretAccessKey'); + + if (!isValidHost(host)) { + throw new Error( + `Invalid awsS3 integration config, '${host}' is not a valid host`, + ); } - if (!config.has('accessKeyId') && !config.has('secretAccessKey')) { - return {}; - } - - const accessKeyId = config.getString('accessKeyId'); - - const secretAccessKey = config.getString('secretAccessKey'); - - return { accessKeyId, secretAccessKey }; + return { host, accessKeyId, secretAccessKey }; +} + +export function readAwsS3IntegrationConfigs( + configs: Config[], +): AwsS3IntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readAwsS3IntegrationConfig); + + // If no explicit amazonaws.com integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === AMAZON_AWS_HOST)) { + result.push({ + host: AMAZON_AWS_HOST, + }); + } + return result; } diff --git a/packages/integration/src/awsS3/index.ts b/packages/integration/src/awsS3/index.ts index 27991a8327..1ae3964ec1 100644 --- a/packages/integration/src/awsS3/index.ts +++ b/packages/integration/src/awsS3/index.ts @@ -13,6 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -export { readAwsS3IntegrationConfig } from './config'; +export { AwsS3Integration } from './AwsS3Integration'; +export { + readAwsS3IntegrationConfig, + readAwsS3IntegrationConfigs, +} from './config'; export type { AwsS3IntegrationConfig } from './config'; diff --git a/packages/integration/src/registry.ts b/packages/integration/src/registry.ts index da205ae1e7..21ca472ccb 100644 --- a/packages/integration/src/registry.ts +++ b/packages/integration/src/registry.ts @@ -15,6 +15,7 @@ */ import { ScmIntegration, ScmIntegrationsGroup } from './types'; +import { AwsS3Integration } from './awsS3/AwsS3Integration'; import { AzureIntegration } from './azure/AzureIntegration'; import { BitbucketIntegration } from './bitbucket/BitbucketIntegration'; import { GitHubIntegration } from './github/GitHubIntegration'; @@ -25,6 +26,7 @@ import { GitLabIntegration } from './gitlab/GitLabIntegration'; */ export interface ScmIntegrationRegistry extends ScmIntegrationsGroup { + awsS3: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; bitbucket: ScmIntegrationsGroup; github: ScmIntegrationsGroup;