Add GcsUrlReader

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
2021-02-26 18:01:06 +01:00
parent f91ee28deb
commit eafb4f9f34
5 changed files with 182 additions and 0 deletions
@@ -122,6 +122,13 @@ kafka:
        - localhost:9092

 integrations:
+  gcs:
+    - host: 'storage.cloud.google.com'
+      clientEmail:
+        $env: GCS_CLIENT_EMAIL
+      privateKey:
+        $env: GCS_PRIVATE_KEY
+
  github:
    - host: github.com
      token:
@@ -34,6 +34,7 @@
    "@backstage/config-loader": "^0.5.1",
    "@backstage/errors": "^0.1.1",
    "@backstage/integration": "^0.5.1",
+    "@google-cloud/storage": "^5.8.0",
    "@octokit/rest": "^18.0.12",
    "@types/cors": "^2.8.6",
    "@types/dockerode": "^3.2.1",
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Storage } from '@google-cloud/storage';
+import {
+  ReaderFactory,
+  ReadTreeResponse,
+  SearchResponse,
+  UrlReader,
+} from './types';
+import getRawBody from 'raw-body';
+import { Logger } from 'winston';
+
+const parseURL = (
+  url: string,
+): { host: string; bucket: string; key: string } => {
+  const { host, pathname } = new URL(url);
+
+  if (host !== 'storage.cloud.google.com') {
+    throw new Error(`not a valid GCS URL: ${url}`);
+  }
+
+  const [, bucket, ...key] = pathname.split('/');
+  return {
+    host: host,
+    bucket,
+    key: key.join('/'),
+  };
+};
+
+export class GcsUrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config, logger }) => {
+    if (!config.has('integrations.gcs')) {
+      return [];
+    }
+    return config
+      .getConfigArray('integrations.gcs')
+      .filter(integration => {
+        if (!integration.has('clientEmail') || !integration.has('privateKey')) {
+          logger.warn(
+            "Skipping gcs integration, Missing required config value at 'integration.gcs.clientEmail' or 'integration.gcs.privateKey'",
+          );
+          return false;
+        }
+        return true;
+      })
+      .map(integration => {
+        const privKey = integration
+          .getOptionalString('privateKey')
+          ?.split('\\n')
+          .join('\n');
+
+        const storage = new Storage({
+          credentials: {
+            client_email: integration.getOptionalString('clientEmail'),
+            private_key: privKey,
+          },
+        });
+        const reader = new GcsUrlReader(storage, logger);
+        const host =
+          integration.getOptionalString('host') || 'storage.cloud.google.com';
+
+        logger.info('Configuring integration, gcs');
+        const predicate = (url: URL) => url.host === host;
+        return { reader, predicate };
+      });
+  };
+
+  constructor(
+    private readonly storage: Storage,
+    private readonly logger: Logger,
+  ) {}
+
+  async read(url: string): Promise<Buffer> {
+    try {
+      const { bucket, key } = parseURL(url);
+
+      this.logger.info('Reading GCS Location');
+      return await getRawBody(
+        this.storage.bucket(bucket).file(key).createReadStream(),
+      );
+    } catch (error) {
+      this.logger.warn(error.stack);
+      throw new Error(`unable to read gcs file from ${url}`);
+    }
+  }
+
+  async readTree(): Promise<ReadTreeResponse> {
+    throw new Error('GcsUrlReader does not implement readTree');
+  }
+
+  async search(): Promise<SearchResponse> {
+    throw new Error('GcsUrlReader does not implement readTree');
+  }
+
+  toString() {
+    return `gcs{host=storage.cloud.google.com,authed=true}}`;
+  }
+}
@@ -24,6 +24,7 @@ import { GithubUrlReader } from './GithubUrlReader';
 import { GitlabUrlReader } from './GitlabUrlReader';
 import { ReadTreeResponseFactory } from './tree';
 import { FetchUrlReader } from './FetchUrlReader';
+import { GcsUrlReader } from './GcsUrlReader';

 type CreateOptions = {
  /** Root config object */
@@ -70,6 +71,7 @@ export class UrlReaders {
        BitbucketUrlReader.factory,
        GithubUrlReader.factory,
        GitlabUrlReader.factory,
+        GcsUrlReader.factory,
        FetchUrlReader.factory,
      ]),
    });
@@ -2471,6 +2471,21 @@
    retry-request "^4.1.1"
    teeny-request "^7.0.0"

+"@google-cloud/common@^3.6.0":
+  version "3.6.0"
+  resolved "https://registry.npmjs.org/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b"
+  integrity sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q==
+  dependencies:
+    "@google-cloud/projectify" "^2.0.0"
+    "@google-cloud/promisify" "^2.0.0"
+    arrify "^2.0.1"
+    duplexify "^4.1.1"
+    ent "^2.2.0"
+    extend "^3.0.2"
+    google-auth-library "^7.0.2"
+    retry-request "^4.1.1"
+    teeny-request "^7.0.0"
+
 "@google-cloud/container@^2.2.0":
  version "2.2.0"
  resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.2.0.tgz#e97ae1cee9040b6af09cc8199ed0aa2d4ae6238e"
@@ -2570,6 +2585,33 @@
    stream-events "^1.0.1"
    xdg-basedir "^4.0.0"

+"@google-cloud/storage@^5.8.0":
+  version "5.8.0"
+  resolved "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz#1f580e276f1d453790b382156421d1bcc4bd3f4b"
+  integrity sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g==
+  dependencies:
+    "@google-cloud/common" "^3.6.0"
+    "@google-cloud/paginator" "^3.0.0"
+    "@google-cloud/promisify" "^2.0.0"
+    arrify "^2.0.0"
+    async-retry "^1.3.1"
+    compressible "^2.0.12"
+    date-and-time "^0.14.2"
+    duplexify "^4.0.0"
+    extend "^3.0.2"
+    gaxios "^4.0.0"
+    gcs-resumable-upload "^3.1.3"
+    get-stream "^6.0.0"
+    hash-stream-validation "^0.2.2"
+    mime "^2.2.0"
+    mime-types "^2.0.8"
+    onetime "^5.1.0"
+    p-limit "^3.0.1"
+    pumpify "^2.0.0"
+    snakeize "^0.1.0"
+    stream-events "^1.0.1"
+    xdg-basedir "^4.0.0"
+
 "@graphiql/toolkit@^0.1.0":
  version "0.1.1"
  resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.1.1.tgz#a7da3ba460ceae27bcdc8f03831ca4f88f90f3d7"
@@ -11393,6 +11435,11 @@ date-and-time@^0.14.2:
  resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e"
  integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==

+date-and-time@^0.14.2:
+  version "0.14.2"
+  resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e"
+  integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==
+
 date-and-time@^0.6.3:
  version "0.6.3"
  resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.6.3.tgz#2daee52df67c28bd93bce862756ac86b68cf4237"
@@ -13979,6 +14026,19 @@ gcs-resumable-upload@^3.1.3:
    pumpify "^2.0.0"
    stream-events "^1.0.4"

+gcs-resumable-upload@^3.1.3:
+  version "3.1.3"
+  resolved "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.3.tgz#1e38e1339600b85812e6430a5ab455453c64cce3"
+  integrity sha512-LjVrv6YVH0XqBr/iBW0JgRA1ndxhK6zfEFFJR4im51QVTj/4sInOXimY2evDZuSZ75D3bHxTaQAdXRukMc1y+w==
+  dependencies:
+    abort-controller "^3.0.0"
+    configstore "^5.0.0"
+    extend "^3.0.2"
+    gaxios "^4.0.0"
+    google-auth-library "^7.0.0"
+    pumpify "^2.0.0"
+    stream-events "^1.0.4"
+
 generic-names@^2.0.1:
  version "2.0.1"
  resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872"