Merge pull request #6600 from splunk/seant-splunk/awsS3_url_reader

AWS S3 Url Reader

.changeset/wicked-spoons-perform.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+'@backstage/integration': patch
+---
+
+Add AWS S3 URL Reader

app-config.yaml

@@ -160,9 +160,13 @@ integrations:
   azure:
     - host: dev.azure.com
       token: ${AZURE_TOKEN}
-#  googleGcs:
-#    clientEmail: 'example@example.com'
-#    privateKey: ${GCS_PRIVATE_KEY}
+  #  googleGcs:
+  #    clientEmail: 'example@example.com'
+  #    privateKey: ${GCS_PRIVATE_KEY}
+  awsS3:
+    - host: amazonaws.com
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
 
 catalog:
   rules:

docs/integrations/aws-s3/locations.md

@@ -0,0 +1,42 @@
+---
+id: locations
+sidebar_label: Locations
+title: Amazon Web Services S3 Locations
+# prettier-ignore
+description: Setting up an integration with Amazon Web Services S3
+---
+
+The AWS S3 integration supports loading catalog entities from an S3 Bucket.
+Entities can be added to
+[static catalog configuration](../../features/software-catalog/configuration.md),
+or registered with the
+[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import)
+plugin.
+
+## Configuration
+
+To use this integration, add configuration to your `app-config.yaml`:
+
+```yaml
+integrations:
+  awsS3:
+    - host: amazonaws.com
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
+```
+
+Then make sure the environment variables `AWS_ACCESS_KEY_ID` and
+`AWS_SECRET_ACCESS_KEY` are set when you run Backstage.
+
+Users with multiple AWS accounts may want to use a role for S3 storage that is
+in a different AWS account. Using the `roleArn` parameter as seen below, you can
+instruct the AWS S3 reader to assume a role before accessing S3:
+
+```yaml
+integrations:
+  awsS3:
+    - host: amazonaws.com
+      accessKeyId: ${AWS_ACCESS_KEY_ID}
+      secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
+      roleArn: 'arn:aws:iam::xxxxxxxxxxxx:role/example-role'
+```

microsite/sidebars.json

@@ -104,6 +104,11 @@
     ],
     "Integrations": [
       "integrations/index",
+      {
+        "type": "subcategory",
+        "label": "AWS S3",
+        "ids": ["integrations/aws-s3/locations"]
+      },
       {
         "type": "subcategory",
         "label": "Azure",

mkdocs.yml

@@ -79,6 +79,8 @@ nav:
           - FAQ: 'features/techdocs/FAQ.md'
   - Integrations:
       - Overview: 'integrations/index.md'
+      - AWS S3:
+          - Locations: 'integrations/aws-s3/locations.md'
       - Azure:
           - Locations: 'integrations/azure/locations.md'
           - Org Data: 'integrations/azure/org.md'

packages/backend-common/package.json

@@ -40,6 +40,7 @@
     "@types/dockerode": "^3.2.1",
     "@types/express": "^4.17.6",
     "archiver": "^5.0.2",
+    "aws-sdk": "^2.840.0",
     "compression": "^1.7.4",
     "concat-stream": "^2.0.0",
     "cors": "^2.8.5",
@@ -92,6 +93,7 @@
     "@types/tar": "^4.0.3",
     "@types/unzipper": "^0.10.3",
     "@types/webpack-env": "^1.15.2",
+    "aws-sdk-mock": "^5.2.1",
     "get-port": "^5.1.1",
     "http-errors": "^1.7.3",
     "jest": "^26.0.1",

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -0,0 +1,232 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ConfigReader, JsonObject } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { DefaultReadTreeResponseFactory } from './tree';
+import { AwsS3UrlReader } from './AwsS3UrlReader';
+import {
+  AwsS3Integration,
+  readAwsS3IntegrationConfig,
+} from '@backstage/integration';
+import { UrlReaderPredicateTuple } from './types';
+import AWSMock from 'aws-sdk-mock';
+import aws from 'aws-sdk';
+import path from 'path';
+
+describe('AwsS3UrlReader', () => {
+  const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
+    return AwsS3UrlReader.factory({
+      config: new ConfigReader(config),
+      logger: getVoidLogger(),
+      treeResponseFactory: DefaultReadTreeResponseFactory.create({
+        config: new ConfigReader({}),
+      }),
+    });
+  };
+
+  afterEach(() => {
+    AWSMock.restore();
+  });
+
+  it('creates a dummy reader without the awsS3 field', () => {
+    const entries = createReader({
+      integrations: {},
+    });
+
+    expect(entries).toHaveLength(1);
+  });
+
+  it('creates a reader with credentials correctly configured', () => {
+    const awsS3Integrations = [];
+    awsS3Integrations.push({
+      host: 'amazonaws.com',
+      accessKeyId: 'fakekey',
+      secretAccessKey: 'fakekey',
+    });
+
+    const entries = createReader({
+      integrations: {
+        awsS3: awsS3Integrations,
+      },
+    });
+
+    expect(entries).toHaveLength(1);
+  });
+
+  it('creates a reader with default credentials provider', () => {
+    const awsS3Integrations = [];
+    awsS3Integrations.push({
+      host: 'amazonaws.com',
+    });
+
+    const entries = createReader({
+      integrations: {
+        awsS3: awsS3Integrations,
+      },
+    });
+
+    expect(entries).toHaveLength(1);
+  });
+
+  describe('predicates', () => {
+    const readers = createReader({
+      integrations: {
+        awsS3: [{}],
+      },
+    });
+    const predicate = readers[0].predicate;
+
+    it('returns true for the correct aws s3 storage host', () => {
+      expect(
+        predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')),
+      ).toBe(true);
+    });
+
+    it('returns true for a url with the full path and the correct host', () => {
+      expect(
+        predicate(
+          new URL(
+            'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml',
+          ),
+        ),
+      ).toBe(true);
+    });
+
+    it('returns false for an incorrect host', () => {
+      expect(predicate(new URL('https://amazon.com'))).toBe(false);
+    });
+
+    it('returns false for a completely different host', () => {
+      expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(
+        false,
+      );
+    });
+
+    it("returns true for a url with a bucket with '.'", () => {
+      expect(
+        predicate(
+          new URL(
+            'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml',
+          ),
+        ),
+      ).toBe(true);
+    });
+  });
+
+  describe('read', () => {
+    AWSMock.setSDKInstance(aws);
+    AWSMock.mock(
+      'S3',
+      'getObject',
+      Buffer.from(
+        require('fs').readFileSync(
+          path.resolve(
+            'src',
+            'reading',
+            '__fixtures__',
+            'awsS3-mock-object.yaml',
+          ),
+        ),
+      ),
+    );
+    const s3 = new aws.S3();
+    const awsS3UrlReader = new AwsS3UrlReader(
+      new AwsS3Integration(
+        readAwsS3IntegrationConfig(
+          new ConfigReader({
+            host: 'amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          }),
+        ),
+      ),
+      s3,
+    );
+
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.read(
+        'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
+      );
+      expect(response.toString()).toBe('site_name: Test\n');
+    });
+
+    it('rejects unknown targets', async () => {
+      await expect(
+        awsS3UrlReader.read(
+          'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml',
+        ),
+      ).rejects.toThrow(
+        Error(
+          `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
+        ),
+      );
+    });
+  });
+
+  describe('readUrl', () => {
+    AWSMock.setSDKInstance(aws);
+
+    AWSMock.mock(
+      'S3',
+      'getObject',
+      Buffer.from(
+        require('fs').readFileSync(
+          path.resolve(
+            'src',
+            'reading',
+            '__fixtures__',
+            'awsS3-mock-object.yaml',
+          ),
+        ),
+      ),
+    );
+
+    const s3 = new aws.S3();
+
+    const awsS3UrlReader = new AwsS3UrlReader(
+      new AwsS3Integration(
+        readAwsS3IntegrationConfig(
+          new ConfigReader({
+            host: 'amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          }),
+        ),
+      ),
+      s3,
+    );
+
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.readUrl(
+        'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
+      );
+      const buffer = await response.buffer();
+      expect(buffer.toString()).toBe('site_name: Test\n');
+    });
+
+    it('rejects unknown targets', async () => {
+      await expect(
+        awsS3UrlReader.readUrl(
+          'https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml',
+        ),
+      ).rejects.toThrow(
+        Error(
+          `Could not retrieve file from S3: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
+        ),
+      );
+    });
+  });
+});

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import aws, { Credentials, S3 } from 'aws-sdk';
+import { CredentialsOptions } from 'aws-sdk/lib/credentials';
+import {
+  ReaderFactory,
+  ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
+  SearchResponse,
+  UrlReader,
+} from './types';
+import getRawBody from 'raw-body';
+import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+
+const parseURL = (
+  url: string,
+): { path: string; bucket: string; region: string } => {
+  let { host, pathname } = new URL(url);
+
+  /**
+   * Removes the leading '/' from the pathname to be processed
+   * as a parameter by AWS S3 SDK getObject method.
+   */
+  pathname = pathname.substr(1);
+
+  /**
+   * Checks that the given URL is a valid S3 object url.
+   * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
+   */
+  const validHost = new RegExp(
+    /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
+  );
+  if (!validHost.test(host)) {
+    throw new Error(`not a valid AWS S3 URL: ${url}`);
+  }
+
+  const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
+  host = host.substring(bucket.length);
+  const [, , region, ,] = host.split('.');
+
+  return {
+    path: pathname,
+    bucket: bucket,
+    region: region,
+  };
+};
+
+export class AwsS3UrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config }) => {
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return integrations.awsS3.list().map(integration => {
+      const creds = AwsS3UrlReader.buildCredentials(integration);
+      const s3 = new S3({
+        apiVersion: '2006-03-01',
+        credentials: creds,
+      });
+      const reader = new AwsS3UrlReader(integration, s3);
+      const predicate = (url: URL) =>
+        url.host.endsWith(integration.config.host);
+      return { reader, predicate };
+    });
+  };
+
+  constructor(
+    private readonly integration: AwsS3Integration,
+    private readonly s3: S3,
+  ) {}
+
+  /**
+   * If accesKeyId and secretAccessKey are missing, the standard credentials provider chain will be used:
+   * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
+   */
+  private static buildCredentials(
+    integration?: AwsS3Integration,
+  ): Credentials | CredentialsOptions | undefined {
+    if (!integration) {
+      return undefined;
+    }
+
+    const accessKeyId = integration.config.accessKeyId;
+    const secretAccessKey = integration.config.secretAccessKey;
+    let explicitCredentials: Credentials | undefined;
+
+    if (accessKeyId && secretAccessKey) {
+      explicitCredentials = new Credentials({
+        accessKeyId,
+        secretAccessKey,
+      });
+    }
+
+    const roleArn = integration.config.roleArn;
+    if (roleArn) {
+      return new aws.ChainableTemporaryCredentials({
+        masterCredentials: explicitCredentials,
+        params: {
+          RoleSessionName: 'backstage-aws-s3-url-reader',
+          RoleArn: roleArn,
+        },
+      });
+    }
+
+    return explicitCredentials;
+  }
+
+  async read(url: string): Promise<Buffer> {
+    const response = await this.readUrl(url);
+    return response.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    try {
+      const { path, bucket, region } = parseURL(url);
+      aws.config.update({ region: region });
+
+      let params;
+      if (options?.etag) {
+        params = {
+          Bucket: bucket,
+          Key: path,
+          IfNoneMatch: options.etag,
+        };
+      } else {
+        params = {
+          Bucket: bucket,
+          Key: path,
+        };
+      }
+
+      const response = this.s3.getObject(params);
+      const buffer = await getRawBody(response.createReadStream());
+      const etag = (await response.promise()).ETag;
+
+      return {
+        buffer: async () => buffer,
+        etag: etag,
+      };
+    } catch (e) {
+      throw new Error(`Could not retrieve file from S3: ${e.message}`);
+    }
+  }
+
+  async readTree(): Promise<ReadTreeResponse> {
+    throw new Error('AwsS3Reader does not implement readTree');
+  }
+
+  async search(): Promise<SearchResponse> {
+    throw new Error('AwsS3Reader does not implement search');
+  }
+
+  toString() {
+    const secretAccessKey = this.integration.config.secretAccessKey;
+    return `awsS3{host=${this.integration.config.host},authed=${Boolean(
+      secretAccessKey,
+    )}}`;
+  }
+}

packages/backend-common/src/reading/UrlReaders.ts

@@ -25,6 +25,7 @@ import { GitlabUrlReader } from './GitlabUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
 import { FetchUrlReader } from './FetchUrlReader';
 import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
+import { AwsS3UrlReader } from './AwsS3UrlReader';
 
 type CreateOptions = {
   /** Root config object */
@@ -74,6 +75,7 @@ export class UrlReaders {
         GithubUrlReader.factory,
         GitlabUrlReader.factory,
         GoogleGcsUrlReader.factory,
+        AwsS3UrlReader.factory,
         FetchUrlReader.factory,
       ]),
     });

packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml

@@ -0,0 +1 @@
+site_name: Test

packages/integration-react/src/api/ScmIntegrationsApi.test.ts

@@ -26,6 +26,6 @@ describe('scmIntegrationsApiRef', () => {
 
   it('should be instantiated', () => {
     const i = ScmIntegrationsApi.fromConfig(new ConfigReader({}));
-    expect(i.list().length).toBe(4); // The default ones
+    expect(i.list().length).toBe(5); // The default ones
   });
 });

packages/integration/api-report.md

@@ -6,6 +6,41 @@
 import { Config } from '@backstage/config';
 import { RestEndpointMethodTypes } from '@octokit/rest';
 
+// Warning: (ae-missing-release-tag) "AwsS3Integration" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class AwsS3Integration implements ScmIntegration {
+  constructor(integrationConfig: AwsS3IntegrationConfig);
+  // (undocumented)
+  get config(): AwsS3IntegrationConfig;
+  // Warning: (ae-forgotten-export) The symbol "ScmIntegrationsFactory" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  static factory: ScmIntegrationsFactory<AwsS3Integration>;
+  // (undocumented)
+  resolveEditUrl(url: string): string;
+  // (undocumented)
+  resolveUrl(options: {
+    url: string;
+    base: string;
+    lineNumber?: number | undefined;
+  }): string;
+  // (undocumented)
+  get title(): string;
+  // (undocumented)
+  get type(): string;
+}
+
+// Warning: (ae-missing-release-tag) "AwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export type AwsS3IntegrationConfig = {
+  host: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  roleArn?: string;
+};
+
 // Warning: (ae-missing-release-tag) "AzureIntegration" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -13,8 +48,6 @@ export class AzureIntegration implements ScmIntegration {
   constructor(integrationConfig: AzureIntegrationConfig);
   // (undocumented)
   get config(): AzureIntegrationConfig;
-  // Warning: (ae-forgotten-export) The symbol "ScmIntegrationsFactory" needs to be exported by the entry point index.d.ts
-  //
   // (undocumented)
   static factory: ScmIntegrationsFactory<AzureIntegration>;
   // (undocumented)
@@ -314,6 +347,21 @@ export type GoogleGcsIntegrationConfig = {
   privateKey?: string;
 };
 
+// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export function readAwsS3IntegrationConfig(
+  config: Config,
+): AwsS3IntegrationConfig;
+
+// Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfigs" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export function readAwsS3IntegrationConfigs(
+  configs: Config[],
+): AwsS3IntegrationConfig[];
+
 // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // Warning: (ae-missing-release-tag) "readAzureIntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
@@ -410,6 +458,8 @@ export interface ScmIntegration {
 export interface ScmIntegrationRegistry
   extends ScmIntegrationsGroup<ScmIntegration> {
   // (undocumented)
+  awsS3: ScmIntegrationsGroup<AwsS3Integration>;
+  // (undocumented)
   azure: ScmIntegrationsGroup<AzureIntegration>;
   // (undocumented)
   bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
@@ -436,6 +486,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
   // Warning: (ae-forgotten-export) The symbol "IntegrationsByType" needs to be exported by the entry point index.d.ts
   constructor(integrationsByType: IntegrationsByType);
   // (undocumented)
+  get awsS3(): ScmIntegrationsGroup<AwsS3Integration>;
+  // (undocumented)
   get azure(): ScmIntegrationsGroup<AzureIntegration>;
   // (undocumented)
   get bitbucket(): ScmIntegrationsGroup<BitbucketIntegration>;

packages/integration/config.d.ts

@@ -163,5 +163,30 @@ export interface Config {
        */
       privateKey?: string;
     };
+
+    /** Integration configuration for AWS S3 Service */
+    awsS3?: Array<{
+      /**
+       * The host of the target that this matches on, e.g. "amazonaws.com".
+       * @visibility frontend
+       */
+      host: string;
+      /**
+       * Account access key used to authenticate requests.
+       * @visibility backend
+       */
+      accessKeyId?: string;
+      /**
+       * Account secret key used to authenticate requests.
+       * @visibility secret
+       */
+      secretAccessKey?: string;
+
+      /**
+       * ARN of the role to be assumed
+       * @visibility backend
+       */
+      roleArn?: string;
+    }>;
   };
 }

packages/integration/src/ScmIntegrations.test.ts

@@ -13,7 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import { AwsS3IntegrationConfig } from './awsS3';
+import { AwsS3Integration } from './awsS3/AwsS3Integration';
 import { AzureIntegrationConfig } from './azure';
 import { AzureIntegration } from './azure/AzureIntegration';
 import { BitbucketIntegrationConfig } from './bitbucket';
@@ -26,6 +27,10 @@ import { basicIntegrations } from './helpers';
 import { ScmIntegrations } from './ScmIntegrations';
 
 describe('ScmIntegrations', () => {
+  const awsS3 = new AwsS3Integration({
+    host: 'awss3.local',
+  } as AwsS3IntegrationConfig);
+
   const azure = new AzureIntegration({
     host: 'azure.local',
   } as AzureIntegrationConfig);
@@ -43,6 +48,7 @@ describe('ScmIntegrations', () => {
   } as GitLabIntegrationConfig);
 
   const i = new ScmIntegrations({
+    awsS3: basicIntegrations([awsS3], item => item.config.host),
     azure: basicIntegrations([azure], item => item.config.host),
     bitbucket: basicIntegrations([bitbucket], item => item.config.host),
     github: basicIntegrations([github], item => item.config.host),
@@ -50,6 +56,7 @@ describe('ScmIntegrations', () => {
   });
 
   it('can get the specifics', () => {
+    expect(i.awsS3.byUrl('https://awss3.local')).toBe(awsS3);
     expect(i.azure.byUrl('https://azure.local')).toBe(azure);
     expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket);
     expect(i.github.byUrl('https://github.local')).toBe(github);
@@ -58,16 +65,18 @@ describe('ScmIntegrations', () => {
 
   it('can list', () => {
     expect(i.list()).toEqual(
-      expect.arrayContaining([azure, bitbucket, github, gitlab]),
+      expect.arrayContaining([awsS3, azure, bitbucket, github, gitlab]),
     );
   });
 
   it('can select by url and host', () => {
+    expect(i.byUrl('https://awss3.local')).toBe(awsS3);
     expect(i.byUrl('https://azure.local')).toBe(azure);
     expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket);
     expect(i.byUrl('https://github.local')).toBe(github);
     expect(i.byUrl('https://gitlab.local')).toBe(gitlab);
 
+    expect(i.byHost('awss3.local')).toBe(awsS3);
     expect(i.byHost('azure.local')).toBe(azure);
     expect(i.byHost('bitbucket.local')).toBe(bitbucket);
     expect(i.byHost('github.local')).toBe(github);

packages/integration/src/ScmIntegrations.ts

@@ -15,6 +15,7 @@
  */
 
 import { Config } from '@backstage/config';
+import { AwsS3Integration } from './awsS3/AwsS3Integration';
 import { AzureIntegration } from './azure/AzureIntegration';
 import { BitbucketIntegration } from './bitbucket/BitbucketIntegration';
 import { GitHubIntegration } from './github/GitHubIntegration';
@@ -24,6 +25,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types';
 import { ScmIntegrationRegistry } from './registry';
 
 type IntegrationsByType = {
+  awsS3: ScmIntegrationsGroup<AwsS3Integration>;
   azure: ScmIntegrationsGroup<AzureIntegration>;
   bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
   github: ScmIntegrationsGroup<GitHubIntegration>;
@@ -35,6 +37,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
 
   static fromConfig(config: Config): ScmIntegrations {
     return new ScmIntegrations({
+      awsS3: AwsS3Integration.factory({ config }),
       azure: AzureIntegration.factory({ config }),
       bitbucket: BitbucketIntegration.factory({ config }),
       github: GitHubIntegration.factory({ config }),
@@ -46,6 +49,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
     this.byType = integrationsByType;
   }
 
+  get awsS3(): ScmIntegrationsGroup<AwsS3Integration> {
+    return this.byType.awsS3;
+  }
+
   get azure(): ScmIntegrationsGroup<AzureIntegration> {
     return this.byType.azure;
   }

packages/integration/src/awsS3/AwsS3Integration.test.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { AwsS3Integration } from './AwsS3Integration';
+
+describe('AwsS3Integration', () => {
+  it('has a working factory', () => {
+    const integrations = AwsS3Integration.factory({
+      config: new ConfigReader({
+        integrations: {
+          awsS3: [
+            {
+              host: 'a.com',
+              accessKeyId: 'access key',
+              secretAccessKey: 'secret key',
+            },
+          ],
+        },
+      }),
+    });
+    expect(integrations.list().length).toBe(2); // including default
+    expect(integrations.list()[0].config.host).toBe('a.com');
+    expect(integrations.list()[1].config.host).toBe('amazonaws.com');
+  });
+
+  it('returns the basics', () => {
+    const integration = new AwsS3Integration({ host: 'a.com' } as any);
+    expect(integration.type).toBe('awsS3');
+    expect(integration.title).toBe('a.com');
+  });
+
+  describe('resolveUrl', () => {
+    it('works for valid urls', () => {
+      const integration = new AwsS3Integration({
+        host: 'amazonaws.com',
+      } as any);
+
+      expect(
+        integration.resolveUrl({
+          url: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml',
+          base: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml',
+        }),
+      ).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml');
+    });
+  });
+
+  it('resolve edit URL', () => {
+    const integration = new AwsS3Integration({ host: 'a.com' } as any);
+
+    // TODO: The Aws S3 integration doesn't support resolving an edit URL,
+    // instead we keep the input URL.
+    expect(
+      integration.resolveEditUrl(
+        'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml',
+      ),
+    ).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml');
+  });
+});

packages/integration/src/awsS3/AwsS3Integration.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
+import { ScmIntegration, ScmIntegrationsFactory } from '../types';
+import { AwsS3IntegrationConfig, readAwsS3IntegrationConfigs } from './config';
+
+export class AwsS3Integration implements ScmIntegration {
+  static factory: ScmIntegrationsFactory<AwsS3Integration> = ({ config }) => {
+    const configs = readAwsS3IntegrationConfigs(
+      config.getOptionalConfigArray('integrations.awsS3') ?? [],
+    );
+    return basicIntegrations(
+      configs.map(c => new AwsS3Integration(c)),
+      i => i.config.host,
+    );
+  };
+
+  get type(): string {
+    return 'awsS3';
+  }
+
+  get title(): string {
+    return this.integrationConfig.host;
+  }
+
+  get config(): AwsS3IntegrationConfig {
+    return this.integrationConfig;
+  }
+
+  constructor(private readonly integrationConfig: AwsS3IntegrationConfig) {}
+  resolveUrl(options: {
+    url: string;
+    base: string;
+    lineNumber?: number | undefined;
+  }): string {
+    const resolved = defaultScmResolveUrl(options);
+    return resolved;
+  }
+
+  resolveEditUrl(url: string): string {
+    // TODO: Implement edit URL for awsS3
+    return url;
+  }
+}

packages/integration/src/awsS3/config.test.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config, ConfigReader } from '@backstage/config';
+import {
+  AwsS3IntegrationConfig,
+  readAwsS3IntegrationConfig,
+  readAwsS3IntegrationConfigs,
+} from './config';
+
+describe('readAwsS3IntegrationConfig', () => {
+  function buildConfig(data: Partial<AwsS3IntegrationConfig>): Config {
+    return new ConfigReader(data);
+  }
+
+  it('reads all values', () => {
+    const output = readAwsS3IntegrationConfig(
+      buildConfig({
+        host: 'amazonaws.com',
+        accessKeyId: 'fake-key',
+        secretAccessKey: 'fake-secret-key',
+      }),
+    );
+    expect(output).toEqual({
+      host: 'amazonaws.com',
+      accessKeyId: 'fake-key',
+      secretAccessKey: 'fake-secret-key',
+    });
+  });
+});
+
+describe('readAwsS3IntegrationConfigs', () => {
+  function buildConfig(data: Partial<AwsS3IntegrationConfig>[]): Config[] {
+    return data.map(item => new ConfigReader(item));
+  }
+
+  it('reads all values', () => {
+    const output = readAwsS3IntegrationConfigs(
+      buildConfig([
+        {
+          host: 'amazonaws.com',
+          accessKeyId: 'key',
+          secretAccessKey: 'secret',
+        },
+      ]),
+    );
+    expect(output).toContainEqual({
+      host: 'amazonaws.com',
+      accessKeyId: 'key',
+      secretAccessKey: 'secret',
+    });
+  });
+
+  it('adds a default entry when missing', () => {
+    const output = readAwsS3IntegrationConfigs(buildConfig([]));
+    expect(output).toEqual([
+      {
+        host: 'amazonaws.com',
+      },
+    ]);
+  });
+});

packages/integration/src/awsS3/config.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import { isValidHost } from '../helpers';
+
+const AMAZON_AWS_HOST = 'amazonaws.com';
+
+/**
+ * The configuration parameters for a single AWS S3 provider.
+ */
+
+export type AwsS3IntegrationConfig = {
+  /**
+   * The host of the target that this matches on, e.g. "amazonaws.com"
+   *
+   * Currently only "amazonaws.com" is supported.
+   */
+  host: string;
+
+  /**
+   * accessKeyId
+   */
+  accessKeyId?: string;
+
+  /**
+   * secretAccessKey
+   */
+  secretAccessKey?: string;
+
+  /**
+   * roleArn
+   */
+  roleArn?: string;
+};
+
+/**
+ * Reads a single Aws S3 integration config.
+ *
+ * @param config The config object of a single integration
+ */
+
+export function readAwsS3IntegrationConfig(
+  config: Config,
+): AwsS3IntegrationConfig {
+  const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST;
+  const accessKeyId = config.getOptionalString('accessKeyId');
+  const secretAccessKey = config.getOptionalString('secretAccessKey');
+  const roleArn = config.getOptionalString('roleArn');
+
+  if (!isValidHost(host)) {
+    throw new Error(
+      `Invalid awsS3 integration config, '${host}' is not a valid host`,
+    );
+  }
+
+  return { host, accessKeyId, secretAccessKey, roleArn };
+}
+
+export function readAwsS3IntegrationConfigs(
+  configs: Config[],
+): AwsS3IntegrationConfig[] {
+  // First read all the explicit integrations
+  const result = configs.map(readAwsS3IntegrationConfig);
+
+  // If no explicit amazonaws.com integration was added, put one in the list as
+  // a convenience
+  if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
+    result.push({
+      host: AMAZON_AWS_HOST,
+    });
+  }
+  return result;
+}

packages/integration/src/awsS3/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { AwsS3Integration } from './AwsS3Integration';
+export {
+  readAwsS3IntegrationConfig,
+  readAwsS3IntegrationConfigs,
+} from './config';
+export type { AwsS3IntegrationConfig } from './config';

packages/integration/src/index.ts

@@ -19,6 +19,7 @@ export * from './bitbucket';
 export * from './github';
 export * from './gitlab';
 export * from './googleGcs';
+export * from './awsS3';
 export { defaultScmResolveUrl } from './helpers';
 export { ScmIntegrations } from './ScmIntegrations';
 export type { ScmIntegration, ScmIntegrationsGroup } from './types';

packages/integration/src/registry.ts

@@ -15,6 +15,7 @@
  */
 
 import { ScmIntegration, ScmIntegrationsGroup } from './types';
+import { AwsS3Integration } from './awsS3/AwsS3Integration';
 import { AzureIntegration } from './azure/AzureIntegration';
 import { BitbucketIntegration } from './bitbucket/BitbucketIntegration';
 import { GitHubIntegration } from './github/GitHubIntegration';
@@ -25,6 +26,7 @@ import { GitLabIntegration } from './gitlab/GitLabIntegration';
  */
 export interface ScmIntegrationRegistry
   extends ScmIntegrationsGroup<ScmIntegration> {
+  awsS3: ScmIntegrationsGroup<AwsS3Integration>;
   azure: ScmIntegrationsGroup<AzureIntegration>;
   bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
   github: ScmIntegrationsGroup<GitHubIntegration>;