Add AWS S3 url reader

Signed-off-by: Kiera Jost <kjost@splunk.com>

packages/backend-common/package.json

@@ -40,6 +40,7 @@
     "@types/dockerode": "^3.2.1",
     "@types/express": "^4.17.6",
     "archiver": "^5.0.2",
+    "aws-sdk": "^2.952.0",
     "compression": "^1.7.4",
     "concat-stream": "^2.0.0",
     "cors": "^2.8.5",

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ConfigReader, JsonObject } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { DefaultReadTreeResponseFactory } from './tree';
+import { AwsS3UrlReader } from './AwsS3UrlReader';
+import { UrlReaderPredicateTuple } from './types';
+
+describe('AwsS3UrlReader', () => {
+  const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
+    return AwsS3UrlReader.factory({
+      config: new ConfigReader(config),
+      logger: getVoidLogger(),
+      treeResponseFactory: DefaultReadTreeResponseFactory.create({
+        config: new ConfigReader({}),
+      }),
+    });
+  };
+
+  it('does not create a reader without the awsS3 field', () => {
+    const entries = createReader({
+      integrations: {},
+    });
+    expect(entries).toHaveLength(0);
+  });
+
+  it('creates a reader with credentials correctly configured', () => {
+    const entries = createReader({
+      integrations: {
+        awsS3: {
+          accessKeyId: 'fakekey',
+          secretAccessKey: 'fakekey',
+        },
+      },
+    });
+    expect(entries).toHaveLength(1);
+  });
+
+  it('creates a reader with default credentials provider', () => {
+    const entries = createReader({
+      integrations: {
+        awsS3: {},
+      },
+    });
+    expect(entries).toHaveLength(1);
+  });
+
+  describe('predicates', () => {
+    const readers = createReader({
+      integrations: {
+        awsS3: {},
+      },
+    });
+    const predicate = readers[0].predicate;
+
+    it('returns true for the correct aws s3 storage host', () => {
+      expect(
+        predicate(new URL('https://test-bucket.s3.us-east-2.amazonaws.com')),
+      ).toBe(true);
+    });
+
+    it('returns true for a url with the full path and the correct host', () => {
+      expect(
+        predicate(
+          new URL(
+            'https://test-bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml',
+          ),
+        ),
+      ).toBe(true);
+    });
+
+    it('returns false for an incorrect host', () => {
+      expect(predicate(new URL('https://amazon.com'))).toBe(false);
+    });
+
+    it('returns false for a completely different host', () => {
+      expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(
+        false,
+      );
+    });
+
+    it("returns true for a url with a bucket with '.'", () => {
+      expect(
+        predicate(
+          new URL(
+            'https://test.bucket.s3.us-east-2.amazonaws.com/team/service/catalog-info.yaml',
+          ),
+        ),
+      ).toBe(true);
+    });
+  });
+});

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import aws, { Credentials, S3 } from 'aws-sdk';
+import {
+  ReaderFactory,
+  ReadTreeResponse,
+  SearchResponse,
+  UrlReader,
+} from './types';
+import getRawBody from 'raw-body';
+import {
+  AwsS3IntegrationConfig,
+  readAwsS3IntegrationConfig,
+} from '@backstage/integration';
+
+const AMAZON_AWS_HOST = '.amazonaws.com';
+
+const parseURL = (
+  url: string,
+): { path: string; bucket: string; region: string } => {
+  let { host, pathname } = new URL(url);
+
+  pathname = pathname.substr(1);
+
+  const validHost = new RegExp(
+    /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
+  );
+  if (!validHost.test(host)) {
+    throw new Error(`not a valid AWS S3 URL: ${url}`);
+  }
+
+  const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
+  host = host.substring(bucket.length);
+  const [, , region, ,] = host.split('.');
+
+  return {
+    path: pathname,
+    bucket: bucket,
+    region: region,
+  };
+};
+
+export class AwsS3UrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config, logger }) => {
+    if (!config.has('integrations.awsS3')) {
+      return [];
+    }
+    const awsS3Config = readAwsS3IntegrationConfig(
+      config.getConfig('integrations.awsS3'),
+    );
+    let s3: S3;
+    if (!awsS3Config.accessKeyId || !awsS3Config.secretAccessKey) {
+      logger.info(
+        'awsS3 credentials not found in config. Using default credentials provider.',
+      );
+      s3 = new S3({});
+    } else {
+      const creds = new Credentials({
+        accessKeyId: awsS3Config.accessKeyId,
+        secretAccessKey: awsS3Config.secretAccessKey,
+      });
+      s3 = new S3({
+        apiVersion: '2006-03-01',
+        credentials: creds,
+      });
+    }
+    const reader = new AwsS3UrlReader(awsS3Config, s3);
+    const predicate = (url: URL) => url.host.endsWith(AMAZON_AWS_HOST);
+    return [{ reader, predicate }];
+  };
+
+  constructor(
+    private readonly integration: AwsS3IntegrationConfig,
+    private readonly s3: S3,
+  ) {}
+
+  async read(url: string): Promise<Buffer> {
+    try {
+      const { path, bucket, region } = parseURL(url);
+      aws.config.update({ region: region });
+
+      const params = {
+        Bucket: bucket,
+        Key: path,
+      };
+      return await getRawBody(this.s3.getObject(params).createReadStream());
+    } catch (e) {
+      throw new Error(`Could not retrieve file from S3: ${e.message}`);
+    }
+  }
+
+  async readTree(): Promise<ReadTreeResponse> {
+    throw new Error('AwsS3Reader does not implement search');
+  }
+
+  async search(): Promise<SearchResponse> {
+    throw new Error('AwsS3Reader does not implement search');
+  }
+
+  toString() {
+    const secretAccessKey = this.integration.secretAccessKey;
+    return `awsS3{host=${AMAZON_AWS_HOST},authed=${Boolean(secretAccessKey)}}`;
+  }
+}

packages/backend-common/src/reading/UrlReaders.ts

@@ -25,6 +25,7 @@ import { GitlabUrlReader } from './GitlabUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
 import { FetchUrlReader } from './FetchUrlReader';
 import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
+import { AwsS3UrlReader } from './AwsS3UrlReader';
 
 type CreateOptions = {
   /** Root config object */
@@ -74,6 +75,7 @@ export class UrlReaders {
         GithubUrlReader.factory,
         GitlabUrlReader.factory,
         GoogleGcsUrlReader.factory,
+        AwsS3UrlReader.factory,
         FetchUrlReader.factory,
       ]),
     });

packages/integration/api-report.md

@@ -6,6 +6,14 @@
 import { Config } from '@backstage/config';
 import { RestEndpointMethodTypes } from '@octokit/rest';
 
+// Warning: (ae-missing-release-tag) "AwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export type AwsS3IntegrationConfig = {
+  accessKeyId?: string;
+  secretAccessKey?: string;
+};
+
 // Warning: (ae-missing-release-tag) "AzureIntegration" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -314,6 +322,14 @@ export type GoogleGcsIntegrationConfig = {
   privateKey?: string;
 };
 
+// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
+// Warning: (ae-missing-release-tag) "readAwsS3IntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export function readAwsS3IntegrationConfig(
+  config: Config,
+): AwsS3IntegrationConfig;
+
 // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // Warning: (ae-missing-release-tag) "readAzureIntegrationConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //

packages/integration/src/awsS3/config.tests.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config, ConfigReader } from '@backstage/config';
+import { AwsS3IntegrationConfig, readAwsS3IntegrationConfig } from './config';
+
+describe('readAwsS3IntegrationConfig', () => {
+  function buildConfig(data: Partial<AwsS3IntegrationConfig>): Config {
+    return new ConfigReader(data);
+  }
+
+  it('reads all values', () => {
+    const output = readAwsS3IntegrationConfig(
+      buildConfig({
+        accessKeyId: 'fake-key',
+        secretAccessKey: 'fake-secret-key',
+      }),
+    );
+    expect(output).toEqual({
+      accessKeyId: 'fake-key',
+      secretAccessKey: 'fake-secret-key',
+    });
+  });
+
+  it('does not fail when config is not set', () => {
+    const output = readAwsS3IntegrationConfig(buildConfig({}));
+    expect(output).toEqual({});
+  });
+});

packages/integration/src/awsS3/config.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+/**
+ * The configuration parameters for a single AWS S3 provider.
+ */
+
+export type AwsS3IntegrationConfig = {
+  /**
+   * accessKeyId
+   */
+  accessKeyId?: string;
+
+  /**
+   * secretAccessKey
+   */
+  secretAccessKey?: string;
+};
+
+/**
+ * Reads a single Aws S3 integration config.
+ *
+ * @param config The config object of a single integration
+ */
+
+export function readAwsS3IntegrationConfig(
+  config: Config,
+): AwsS3IntegrationConfig {
+  if (!config) {
+    return {};
+  }
+
+  if (!config.has('accessKeyId') && !config.has('secretAccessKey')) {
+    return {};
+  }
+
+  const accessKeyId = config.getString('accessKeyId');
+
+  const secretAccessKey = config.getString('secretAccessKey');
+
+  return { accessKeyId: accessKeyId, secretAccessKey: secretAccessKey };
+}

packages/integration/src/awsS3/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { readAwsS3IntegrationConfig } from './config';
+export type { AwsS3IntegrationConfig } from './config';

packages/integration/src/index.ts

@@ -19,6 +19,7 @@ export * from './bitbucket';
 export * from './github';
 export * from './gitlab';
 export * from './googleGcs';
+export * from './awsS3';
 export { defaultScmResolveUrl } from './helpers';
 export { ScmIntegrations } from './ScmIntegrations';
 export type { ScmIntegration, ScmIntegrationsGroup } from './types';