From eafb4f9f34a75c2bb43b8fa706e8501eb86f5879 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Fri, 26 Feb 2021 18:01:06 +0100
Subject: [PATCH 01/11] Add GcsUrlReader

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 app-config.yaml                               |   7 ++
 packages/backend-common/package.json          |   1 +
 .../src/reading/GcsUrlReader.ts               | 112 ++++++++++++++++++
 .../backend-common/src/reading/UrlReaders.ts  |   2 +
 yarn.lock                                     |  60 ++++++++++
 5 files changed, 182 insertions(+)
 create mode 100644 packages/backend-common/src/reading/GcsUrlReader.ts

diff --git a/app-config.yaml b/app-config.yaml
index 465c7b0049..4d12e3e474 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -122,6 +122,13 @@ kafka:
         - localhost:9092
 
 integrations:
+  gcs:
+    - host: 'storage.cloud.google.com'
+      clientEmail:
+        $env: GCS_CLIENT_EMAIL
+      privateKey:
+        $env: GCS_PRIVATE_KEY
+
   github:
     - host: github.com
       token:
diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json
index b860336d31..a720f9e3ab 100644
--- a/packages/backend-common/package.json
+++ b/packages/backend-common/package.json
@@ -34,6 +34,7 @@
     "@backstage/config-loader": "^0.5.1",
     "@backstage/errors": "^0.1.1",
     "@backstage/integration": "^0.5.1",
+    "@google-cloud/storage": "^5.8.0",
     "@octokit/rest": "^18.0.12",
     "@types/cors": "^2.8.6",
     "@types/dockerode": "^3.2.1",
diff --git a/packages/backend-common/src/reading/GcsUrlReader.ts b/packages/backend-common/src/reading/GcsUrlReader.ts
new file mode 100644
index 0000000000..23c8f45fe7
--- /dev/null
+++ b/packages/backend-common/src/reading/GcsUrlReader.ts
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Storage } from '@google-cloud/storage';
+import {
+  ReaderFactory,
+  ReadTreeResponse,
+  SearchResponse,
+  UrlReader,
+} from './types';
+import getRawBody from 'raw-body';
+import { Logger } from 'winston';
+
+const parseURL = (
+  url: string,
+): { host: string; bucket: string; key: string } => {
+  const { host, pathname } = new URL(url);
+
+  if (host !== 'storage.cloud.google.com') {
+    throw new Error(`not a valid GCS URL: ${url}`);
+  }
+
+  const [, bucket, ...key] = pathname.split('/');
+  return {
+    host: host,
+    bucket,
+    key: key.join('/'),
+  };
+};
+
+export class GcsUrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config, logger }) => {
+    if (!config.has('integrations.gcs')) {
+      return [];
+    }
+    return config
+      .getConfigArray('integrations.gcs')
+      .filter(integration => {
+        if (!integration.has('clientEmail') || !integration.has('privateKey')) {
+          logger.warn(
+            "Skipping gcs integration, Missing required config value at 'integration.gcs.clientEmail' or 'integration.gcs.privateKey'",
+          );
+          return false;
+        }
+        return true;
+      })
+      .map(integration => {
+        const privKey = integration
+          .getOptionalString('privateKey')
+          ?.split('\\n')
+          .join('\n');
+
+        const storage = new Storage({
+          credentials: {
+            client_email: integration.getOptionalString('clientEmail'),
+            private_key: privKey,
+          },
+        });
+        const reader = new GcsUrlReader(storage, logger);
+        const host =
+          integration.getOptionalString('host') || 'storage.cloud.google.com';
+
+        logger.info('Configuring integration, gcs');
+        const predicate = (url: URL) => url.host === host;
+        return { reader, predicate };
+      });
+  };
+
+  constructor(
+    private readonly storage: Storage,
+    private readonly logger: Logger,
+  ) {}
+
+  async read(url: string): Promise<Buffer> {
+    try {
+      const { bucket, key } = parseURL(url);
+
+      this.logger.info('Reading GCS Location');
+      return await getRawBody(
+        this.storage.bucket(bucket).file(key).createReadStream(),
+      );
+    } catch (error) {
+      this.logger.warn(error.stack);
+      throw new Error(`unable to read gcs file from ${url}`);
+    }
+  }
+
+  async readTree(): Promise<ReadTreeResponse> {
+    throw new Error('GcsUrlReader does not implement readTree');
+  }
+
+  async search(): Promise<SearchResponse> {
+    throw new Error('GcsUrlReader does not implement readTree');
+  }
+
+  toString() {
+    return `gcs{host=storage.cloud.google.com,authed=true}}`;
+  }
+}
diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts
index 2f233463bb..785325faa7 100644
--- a/packages/backend-common/src/reading/UrlReaders.ts
+++ b/packages/backend-common/src/reading/UrlReaders.ts
@@ -24,6 +24,7 @@ import { GithubUrlReader } from './GithubUrlReader';
 import { GitlabUrlReader } from './GitlabUrlReader';
 import { ReadTreeResponseFactory } from './tree';
 import { FetchUrlReader } from './FetchUrlReader';
+import { GcsUrlReader } from './GcsUrlReader';
 
 type CreateOptions = {
   /** Root config object */
@@ -70,6 +71,7 @@ export class UrlReaders {
         BitbucketUrlReader.factory,
         GithubUrlReader.factory,
         GitlabUrlReader.factory,
+        GcsUrlReader.factory,
         FetchUrlReader.factory,
       ]),
     });
diff --git a/yarn.lock b/yarn.lock
index b0b8c04313..d9dcd3f5b1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2471,6 +2471,21 @@
     retry-request "^4.1.1"
     teeny-request "^7.0.0"
 
+"@google-cloud/common@^3.6.0":
+  version "3.6.0"
+  resolved "https://registry.npmjs.org/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b"
+  integrity sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q==
+  dependencies:
+    "@google-cloud/projectify" "^2.0.0"
+    "@google-cloud/promisify" "^2.0.0"
+    arrify "^2.0.1"
+    duplexify "^4.1.1"
+    ent "^2.2.0"
+    extend "^3.0.2"
+    google-auth-library "^7.0.2"
+    retry-request "^4.1.1"
+    teeny-request "^7.0.0"
+
 "@google-cloud/container@^2.2.0":
   version "2.2.0"
   resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.2.0.tgz#e97ae1cee9040b6af09cc8199ed0aa2d4ae6238e"
@@ -2570,6 +2585,33 @@
     stream-events "^1.0.1"
     xdg-basedir "^4.0.0"
 
+"@google-cloud/storage@^5.8.0":
+  version "5.8.0"
+  resolved "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz#1f580e276f1d453790b382156421d1bcc4bd3f4b"
+  integrity sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g==
+  dependencies:
+    "@google-cloud/common" "^3.6.0"
+    "@google-cloud/paginator" "^3.0.0"
+    "@google-cloud/promisify" "^2.0.0"
+    arrify "^2.0.0"
+    async-retry "^1.3.1"
+    compressible "^2.0.12"
+    date-and-time "^0.14.2"
+    duplexify "^4.0.0"
+    extend "^3.0.2"
+    gaxios "^4.0.0"
+    gcs-resumable-upload "^3.1.3"
+    get-stream "^6.0.0"
+    hash-stream-validation "^0.2.2"
+    mime "^2.2.0"
+    mime-types "^2.0.8"
+    onetime "^5.1.0"
+    p-limit "^3.0.1"
+    pumpify "^2.0.0"
+    snakeize "^0.1.0"
+    stream-events "^1.0.1"
+    xdg-basedir "^4.0.0"
+
 "@graphiql/toolkit@^0.1.0":
   version "0.1.1"
   resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.1.1.tgz#a7da3ba460ceae27bcdc8f03831ca4f88f90f3d7"
@@ -11393,6 +11435,11 @@ date-and-time@^0.14.2:
   resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e"
   integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==
 
+date-and-time@^0.14.2:
+  version "0.14.2"
+  resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e"
+  integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==
+
 date-and-time@^0.6.3:
   version "0.6.3"
   resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.6.3.tgz#2daee52df67c28bd93bce862756ac86b68cf4237"
@@ -13979,6 +14026,19 @@ gcs-resumable-upload@^3.1.3:
     pumpify "^2.0.0"
     stream-events "^1.0.4"
 
+gcs-resumable-upload@^3.1.3:
+  version "3.1.3"
+  resolved "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.3.tgz#1e38e1339600b85812e6430a5ab455453c64cce3"
+  integrity sha512-LjVrv6YVH0XqBr/iBW0JgRA1ndxhK6zfEFFJR4im51QVTj/4sInOXimY2evDZuSZ75D3bHxTaQAdXRukMc1y+w==
+  dependencies:
+    abort-controller "^3.0.0"
+    configstore "^5.0.0"
+    extend "^3.0.2"
+    gaxios "^4.0.0"
+    google-auth-library "^7.0.0"
+    pumpify "^2.0.0"
+    stream-events "^1.0.4"
+
 generic-names@^2.0.1:
   version "2.0.1"
   resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872"

From 7abb50d59ef22859bf7f3581195e0e635195d45d Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Fri, 12 Mar 2021 17:11:37 +0100
Subject: [PATCH 02/11] Add tests for GCSUrlReader

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 .../src/reading/GcsUrlReader.test.ts          | 113 ++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 packages/backend-common/src/reading/GcsUrlReader.test.ts

diff --git a/packages/backend-common/src/reading/GcsUrlReader.test.ts b/packages/backend-common/src/reading/GcsUrlReader.test.ts
new file mode 100644
index 0000000000..abd48d5b36
--- /dev/null
+++ b/packages/backend-common/src/reading/GcsUrlReader.test.ts
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader, JsonObject } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { ReadTreeResponseFactory } from './tree';
+import { GcsUrlReader } from './GcsUrlReader';
+import { UrlReaderPredicateTuple } from './types';
+
+describe('GcsUrlReader', () => {
+  const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
+    return GcsUrlReader.factory({
+      config: new ConfigReader(config),
+      logger: getVoidLogger(),
+      treeResponseFactory: ReadTreeResponseFactory.create({
+        config: new ConfigReader({}),
+      }),
+    });
+  };
+
+  it('does not create a reader without the gcs field', () => {
+    const entries = createReader({
+      integrations: {},
+    });
+    expect(entries).toHaveLength(0);
+  });
+
+  it('creates a reader with credentials correctly configured', () => {
+    const entries = createReader({
+      integrations: {
+        gcs: [
+          {
+            privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---',
+            clientEmail: 'someone@example.com',
+          },
+          {
+            host: 'proxy.storage.cloud.google.com',
+            privateKey: '--- BEGIN KEY ---- fakekey2 --- END KEY ---',
+            clientEmail: 'someone2@example.com',
+          },
+        ],
+      },
+    });
+    expect(entries).toHaveLength(2);
+  });
+
+  it('does not create a reader if the privateKey is missing', () => {
+    const entries = createReader({
+      integrations: {
+        gcs: [
+          {
+            clientEmail: 'someone@example.com',
+          },
+        ],
+      },
+    });
+    expect(entries).toHaveLength(0);
+  });
+
+  it('does not create a reader if the clientEmail is missing', () => {
+    const entries = createReader({
+      integrations: {
+        gcs: [
+          {
+            privateKey:
+              '-----BEGIN PRIVATE KEY----- fakekey -----END PRIVATE KEY-----',
+          },
+        ],
+      },
+    });
+    expect(entries).toHaveLength(0);
+  });
+
+  it('predicates', () => {
+    const readers = createReader({
+      integrations: {
+        gcs: [
+          {
+            privateKey:
+              '-----BEGIN PRIVATE KEY----- fakekey -----END PRIVATE KEY-----',
+            clientEmail: 'someone@example.com',
+          },
+        ],
+      },
+    });
+    const predicate = readers[0].predicate;
+    expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(true);
+    expect(
+      predicate(
+        new URL(
+          'https://storage.cloud.google.com/team1/service1/catalog-info.yaml',
+        ),
+      ),
+    ).toBe(true);
+    expect(predicate(new URL('https://storage2.cloud.google.com'))).toBe(false);
+    expect(predicate(new URL('https://cloud.google.com'))).toBe(false);
+    expect(predicate(new URL('https://google.com'))).toBe(false);
+    expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
+  });
+});

From b779b5fee035455d2c75687e4275ba23a1bfc8ae Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Mon, 15 Mar 2021 09:02:03 +0100
Subject: [PATCH 03/11] Add changeset

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 .changeset/sour-clocks-pay.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/sour-clocks-pay.md

diff --git a/.changeset/sour-clocks-pay.md b/.changeset/sour-clocks-pay.md
new file mode 100644
index 0000000000..79b7c95be3
--- /dev/null
+++ b/.changeset/sour-clocks-pay.md
@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': minor
+---
+
+Add UrlReader for Google Cloud Storage

From 9c4cedf268a0ef95d75f3781a2a74b7527c4539b Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Thu, 18 Mar 2021 19:15:25 +0100
Subject: [PATCH 04/11] Fix some comments

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 .changeset/sour-clocks-pay.md                 |  2 +-
 app-config.yaml                               |  5 +-
 packages/backend-common/package.json          |  1 +
 .../src/reading/GcsUrlReader.test.ts          | 46 ++++++++-----
 .../src/reading/GcsUrlReader.ts               | 21 +++---
 yarn.lock                                     | 65 ++++++-------------
 6 files changed, 63 insertions(+), 77 deletions(-)

diff --git a/.changeset/sour-clocks-pay.md b/.changeset/sour-clocks-pay.md
index 79b7c95be3..11196aeebe 100644
--- a/.changeset/sour-clocks-pay.md
+++ b/.changeset/sour-clocks-pay.md
@@ -1,5 +1,5 @@
 ---
-'@backstage/backend-common': minor
+'@backstage/backend-common': patch
 ---
 
 Add UrlReader for Google Cloud Storage
diff --git a/app-config.yaml b/app-config.yaml
index 4d12e3e474..49ac0b0d00 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -122,9 +122,8 @@ kafka:
         - localhost:9092
 
 integrations:
-  gcs:
-    - host: 'storage.cloud.google.com'
-      clientEmail:
+  googleGcs:
+    - clientEmail:
         $env: GCS_CLIENT_EMAIL
       privateKey:
         $env: GCS_PRIVATE_KEY
diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json
index a720f9e3ab..ba846e0f5f 100644
--- a/packages/backend-common/package.json
+++ b/packages/backend-common/package.json
@@ -57,6 +57,7 @@
     "minimatch": "^3.0.4",
     "minimist": "^1.2.5",
     "morgan": "^1.10.0",
+    "raw-body": "^2.4.1",
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
     "tar": "^6.0.5",
diff --git a/packages/backend-common/src/reading/GcsUrlReader.test.ts b/packages/backend-common/src/reading/GcsUrlReader.test.ts
index abd48d5b36..16102d2812 100644
--- a/packages/backend-common/src/reading/GcsUrlReader.test.ts
+++ b/packages/backend-common/src/reading/GcsUrlReader.test.ts
@@ -31,7 +31,7 @@ describe('GcsUrlReader', () => {
     });
   };
 
-  it('does not create a reader without the gcs field', () => {
+  it('does not create a reader without the googleGcs field', () => {
     const entries = createReader({
       integrations: {},
     });
@@ -41,7 +41,7 @@ describe('GcsUrlReader', () => {
   it('creates a reader with credentials correctly configured', () => {
     const entries = createReader({
       integrations: {
-        gcs: [
+        googleGcs: [
           {
             privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---',
             clientEmail: 'someone@example.com',
@@ -60,7 +60,7 @@ describe('GcsUrlReader', () => {
   it('does not create a reader if the privateKey is missing', () => {
     const entries = createReader({
       integrations: {
-        gcs: [
+        googleGcs: [
           {
             clientEmail: 'someone@example.com',
           },
@@ -73,7 +73,7 @@ describe('GcsUrlReader', () => {
   it('does not create a reader if the clientEmail is missing', () => {
     const entries = createReader({
       integrations: {
-        gcs: [
+        googleGcs: [
           {
             privateKey:
               '-----BEGIN PRIVATE KEY----- fakekey -----END PRIVATE KEY-----',
@@ -84,10 +84,10 @@ describe('GcsUrlReader', () => {
     expect(entries).toHaveLength(0);
   });
 
-  it('predicates', () => {
+  describe('predicates', () => {
     const readers = createReader({
       integrations: {
-        gcs: [
+        googleGcs: [
           {
             privateKey:
               '-----BEGIN PRIVATE KEY----- fakekey -----END PRIVATE KEY-----',
@@ -97,17 +97,29 @@ describe('GcsUrlReader', () => {
       },
     });
     const predicate = readers[0].predicate;
-    expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(true);
-    expect(
-      predicate(
-        new URL(
-          'https://storage.cloud.google.com/team1/service1/catalog-info.yaml',
+
+    it('returns true for the correct google cloud storage host', () => {
+      expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(true);
+    });
+    it('returns true for a url with the full path and the correct host', () => {
+      expect(
+        predicate(
+          new URL(
+            'https://storage.cloud.google.com/team1/service1/catalog-info.yaml',
+          ),
         ),
-      ),
-    ).toBe(true);
-    expect(predicate(new URL('https://storage2.cloud.google.com'))).toBe(false);
-    expect(predicate(new URL('https://cloud.google.com'))).toBe(false);
-    expect(predicate(new URL('https://google.com'))).toBe(false);
-    expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
+      ).toBe(true);
+    });
+    it('returns false for the wrong hostname under cloud.google.com', () => {
+      expect(predicate(new URL('https://storage2.cloud.google.com'))).toBe(
+        false,
+      );
+    });
+    it('returns false for a partially correct host', () => {
+      expect(predicate(new URL('https://cloud.google.com'))).toBe(false);
+    });
+    it('returns false for a completely different host', () => {
+      expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
+    });
   });
 });
diff --git a/packages/backend-common/src/reading/GcsUrlReader.ts b/packages/backend-common/src/reading/GcsUrlReader.ts
index 23c8f45fe7..08891d7b49 100644
--- a/packages/backend-common/src/reading/GcsUrlReader.ts
+++ b/packages/backend-common/src/reading/GcsUrlReader.ts
@@ -22,7 +22,6 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
-import { Logger } from 'winston';
 
 const parseURL = (
   url: string,
@@ -43,11 +42,14 @@ const parseURL = (
 
 export class GcsUrlReader implements UrlReader {
   static factory: ReaderFactory = ({ config, logger }) => {
-    if (!config.has('integrations.gcs')) {
+    if (!config.has('integrations.googleGcs')) {
       return [];
     }
-    return config
-      .getConfigArray('integrations.gcs')
+    const configs = config.getOptionalConfigArray('integrations.googleGcs');
+    if (!configs) {
+      return [];
+    }
+    return configs
       .filter(integration => {
         if (!integration.has('clientEmail') || !integration.has('privateKey')) {
           logger.warn(
@@ -69,7 +71,7 @@ export class GcsUrlReader implements UrlReader {
             private_key: privKey,
           },
         });
-        const reader = new GcsUrlReader(storage, logger);
+        const reader = new GcsUrlReader(storage);
         const host =
           integration.getOptionalString('host') || 'storage.cloud.google.com';
 
@@ -79,21 +81,16 @@ export class GcsUrlReader implements UrlReader {
       });
   };
 
-  constructor(
-    private readonly storage: Storage,
-    private readonly logger: Logger,
-  ) {}
+  constructor(private readonly storage: Storage) {}
 
   async read(url: string): Promise<Buffer> {
     try {
       const { bucket, key } = parseURL(url);
 
-      this.logger.info('Reading GCS Location');
       return await getRawBody(
         this.storage.bucket(bucket).file(key).createReadStream(),
       );
     } catch (error) {
-      this.logger.warn(error.stack);
       throw new Error(`unable to read gcs file from ${url}`);
     }
   }
@@ -103,7 +100,7 @@ export class GcsUrlReader implements UrlReader {
   }
 
   async search(): Promise<SearchResponse> {
-    throw new Error('GcsUrlReader does not implement readTree');
+    throw new Error('GcsUrlReader does not implement search');
   }
 
   toString() {
diff --git a/yarn.lock b/yarn.lock
index d9dcd3f5b1..1cb153d4e1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2471,21 +2471,6 @@
     retry-request "^4.1.1"
     teeny-request "^7.0.0"
 
-"@google-cloud/common@^3.6.0":
-  version "3.6.0"
-  resolved "https://registry.npmjs.org/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b"
-  integrity sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q==
-  dependencies:
-    "@google-cloud/projectify" "^2.0.0"
-    "@google-cloud/promisify" "^2.0.0"
-    arrify "^2.0.1"
-    duplexify "^4.1.1"
-    ent "^2.2.0"
-    extend "^3.0.2"
-    google-auth-library "^7.0.2"
-    retry-request "^4.1.1"
-    teeny-request "^7.0.0"
-
 "@google-cloud/container@^2.2.0":
   version "2.2.0"
   resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.2.0.tgz#e97ae1cee9040b6af09cc8199ed0aa2d4ae6238e"
@@ -11435,11 +11420,6 @@ date-and-time@^0.14.2:
   resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e"
   integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==
 
-date-and-time@^0.14.2:
-  version "0.14.2"
-  resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e"
-  integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA==
-
 date-and-time@^0.6.3:
   version "0.6.3"
   resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.6.3.tgz#2daee52df67c28bd93bce862756ac86b68cf4237"
@@ -14026,19 +14006,6 @@ gcs-resumable-upload@^3.1.3:
     pumpify "^2.0.0"
     stream-events "^1.0.4"
 
-gcs-resumable-upload@^3.1.3:
-  version "3.1.3"
-  resolved "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.3.tgz#1e38e1339600b85812e6430a5ab455453c64cce3"
-  integrity sha512-LjVrv6YVH0XqBr/iBW0JgRA1ndxhK6zfEFFJR4im51QVTj/4sInOXimY2evDZuSZ75D3bHxTaQAdXRukMc1y+w==
-  dependencies:
-    abort-controller "^3.0.0"
-    configstore "^5.0.0"
-    extend "^3.0.2"
-    gaxios "^4.0.0"
-    google-auth-library "^7.0.0"
-    pumpify "^2.0.0"
-    stream-events "^1.0.4"
-
 generic-names@^2.0.1:
   version "2.0.1"
   resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872"
@@ -15159,6 +15126,17 @@ http-errors@1.7.2:
     statuses ">= 1.5.0 < 2"
     toidentifier "1.0.0"
 
+http-errors@1.7.3, http-errors@~1.7.2:
+  version "1.7.3"
+  resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06"
+  integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==
+  dependencies:
+    depd "~1.1.2"
+    inherits "2.0.4"
+    setprototypeof "1.1.1"
+    statuses ">= 1.5.0 < 2"
+    toidentifier "1.0.0"
+
 http-errors@^1.7.3:
   version "1.8.0"
   resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.8.0.tgz#75d1bbe497e1044f51e4ee9e704a62f28d336507"
@@ -15180,17 +15158,6 @@ http-errors@~1.6.2:
     setprototypeof "1.1.0"
     statuses ">= 1.4.0 < 2"
 
-http-errors@~1.7.2:
-  version "1.7.3"
-  resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06"
-  integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==
-  dependencies:
-    depd "~1.1.2"
-    inherits "2.0.4"
-    setprototypeof "1.1.1"
-    statuses ">= 1.5.0 < 2"
-    toidentifier "1.0.0"
-
 "http-parser-js@>=0.4.0 <0.4.11":
   version "0.4.10"
   resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.10.tgz#92c9c1374c35085f75db359ec56cc257cbb93fa4"
@@ -21976,6 +21943,16 @@ raw-body@2.4.0:
     iconv-lite "0.4.24"
     unpipe "1.0.0"
 
+raw-body@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.4.1.tgz#30ac82f98bb5ae8c152e67149dac8d55153b168c"
+  integrity sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA==
+  dependencies:
+    bytes "3.1.0"
+    http-errors "1.7.3"
+    iconv-lite "0.4.24"
+    unpipe "1.0.0"
+
 raw-loader@^4.0.1:
   version "4.0.2"
   resolved "https://registry.npmjs.org/raw-loader/-/raw-loader-4.0.2.tgz#1aac6b7d1ad1501e66efdac1522c73e59a584eb6"

From e4cdec27e936359aad1a499a62ebac9405b44193 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Fri, 19 Mar 2021 17:10:15 +0100
Subject: [PATCH 05/11] Fix comments on GoogleGcsUrlReader

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 ...der.test.ts => GoogleGcsUrlReader.test.ts} | 36 ++---------
 ...{GcsUrlReader.ts => GoogleGcsUrlReader.ts} | 60 ++++++++----------
 .../backend-common/src/reading/UrlReaders.ts  |  4 +-
 packages/integration/config.d.ts              | 14 +++++
 .../integration/src/googleGcs/config.test.ts  | 62 ++++++++++++++++++
 packages/integration/src/googleGcs/config.ts  | 63 +++++++++++++++++++
 packages/integration/src/googleGcs/index.ts   | 22 +++++++
 packages/integration/src/index.ts             |  1 +
 8 files changed, 196 insertions(+), 66 deletions(-)
 rename packages/backend-common/src/reading/{GcsUrlReader.test.ts => GoogleGcsUrlReader.test.ts} (77%)
 rename packages/backend-common/src/reading/{GcsUrlReader.ts => GoogleGcsUrlReader.ts} (60%)
 create mode 100644 packages/integration/src/googleGcs/config.test.ts
 create mode 100644 packages/integration/src/googleGcs/config.ts
 create mode 100644 packages/integration/src/googleGcs/index.ts

diff --git a/packages/backend-common/src/reading/GcsUrlReader.test.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts
similarity index 77%
rename from packages/backend-common/src/reading/GcsUrlReader.test.ts
rename to packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts
index 16102d2812..c8262767e1 100644
--- a/packages/backend-common/src/reading/GcsUrlReader.test.ts
+++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts
@@ -17,12 +17,12 @@
 import { ConfigReader, JsonObject } from '@backstage/config';
 import { getVoidLogger } from '../logging';
 import { ReadTreeResponseFactory } from './tree';
-import { GcsUrlReader } from './GcsUrlReader';
+import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 import { UrlReaderPredicateTuple } from './types';
 
 describe('GcsUrlReader', () => {
   const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
-    return GcsUrlReader.factory({
+    return GoogleGcsUrlReader.factory({
       config: new ConfigReader(config),
       logger: getVoidLogger(),
       treeResponseFactory: ReadTreeResponseFactory.create({
@@ -57,43 +57,19 @@ describe('GcsUrlReader', () => {
     expect(entries).toHaveLength(2);
   });
 
-  it('does not create a reader if the privateKey is missing', () => {
+  it('creates a reader with default credentials provider', () => {
     const entries = createReader({
       integrations: {
-        googleGcs: [
-          {
-            clientEmail: 'someone@example.com',
-          },
-        ],
+        googleGcs: [{}],
       },
     });
-    expect(entries).toHaveLength(0);
-  });
-
-  it('does not create a reader if the clientEmail is missing', () => {
-    const entries = createReader({
-      integrations: {
-        googleGcs: [
-          {
-            privateKey:
-              '-----BEGIN PRIVATE KEY----- fakekey -----END PRIVATE KEY-----',
-          },
-        ],
-      },
-    });
-    expect(entries).toHaveLength(0);
+    expect(entries).toHaveLength(1);
   });
 
   describe('predicates', () => {
     const readers = createReader({
       integrations: {
-        googleGcs: [
-          {
-            privateKey:
-              '-----BEGIN PRIVATE KEY----- fakekey -----END PRIVATE KEY-----',
-            clientEmail: 'someone@example.com',
-          },
-        ],
+        googleGcs: [{}],
       },
     });
     const predicate = readers[0].predicate;
diff --git a/packages/backend-common/src/reading/GcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
similarity index 60%
rename from packages/backend-common/src/reading/GcsUrlReader.ts
rename to packages/backend-common/src/reading/GoogleGcsUrlReader.ts
index 08891d7b49..a9100051a6 100644
--- a/packages/backend-common/src/reading/GcsUrlReader.ts
+++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
@@ -22,13 +22,17 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
+import {
+  GOOGLE_GCS_HOST,
+  readGoogleGcsIntegrationConfigs,
+} from '@backstage/integration';
 
 const parseURL = (
   url: string,
 ): { host: string; bucket: string; key: string } => {
   const { host, pathname } = new URL(url);
 
-  if (host !== 'storage.cloud.google.com') {
+  if (host !== GOOGLE_GCS_HOST) {
     throw new Error(`not a valid GCS URL: ${url}`);
   }
 
@@ -40,45 +44,33 @@ const parseURL = (
   };
 };
 
-export class GcsUrlReader implements UrlReader {
+export class GoogleGcsUrlReader implements UrlReader {
   static factory: ReaderFactory = ({ config, logger }) => {
     if (!config.has('integrations.googleGcs')) {
       return [];
     }
-    const configs = config.getOptionalConfigArray('integrations.googleGcs');
-    if (!configs) {
-      return [];
-    }
-    return configs
-      .filter(integration => {
-        if (!integration.has('clientEmail') || !integration.has('privateKey')) {
-          logger.warn(
-            "Skipping gcs integration, Missing required config value at 'integration.gcs.clientEmail' or 'integration.gcs.privateKey'",
-          );
-          return false;
-        }
-        return true;
-      })
-      .map(integration => {
-        const privKey = integration
-          .getOptionalString('privateKey')
-          ?.split('\\n')
-          .join('\n');
-
-        const storage = new Storage({
+    const configs = readGoogleGcsIntegrationConfigs(
+      config.getOptionalConfigArray('integrations.googleGcs') ?? [],
+    );
+    return configs.map(integration => {
+      let storage: Storage;
+      if (!integration.clientEmail || !integration.privateKey) {
+        logger.warn(
+          'googleGcs credentials not found in config. Using default credentials provider.',
+        );
+        storage = new Storage();
+      } else {
+        storage = new Storage({
           credentials: {
-            client_email: integration.getOptionalString('clientEmail'),
-            private_key: privKey,
+            client_email: integration.clientEmail || undefined,
+            private_key: integration.privateKey || undefined,
           },
         });
-        const reader = new GcsUrlReader(storage);
-        const host =
-          integration.getOptionalString('host') || 'storage.cloud.google.com';
-
-        logger.info('Configuring integration, gcs');
-        const predicate = (url: URL) => url.host === host;
-        return { reader, predicate };
-      });
+      }
+      const reader = new GoogleGcsUrlReader(storage);
+      const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
+      return { reader, predicate };
+    });
   };
 
   constructor(private readonly storage: Storage) {}
@@ -104,6 +96,6 @@ export class GcsUrlReader implements UrlReader {
   }
 
   toString() {
-    return `gcs{host=storage.cloud.google.com,authed=true}}`;
+    return `gcs{host=${GOOGLE_GCS_HOST},authed=true}}`;
   }
 }
diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts
index 785325faa7..06c6fdcdcf 100644
--- a/packages/backend-common/src/reading/UrlReaders.ts
+++ b/packages/backend-common/src/reading/UrlReaders.ts
@@ -24,7 +24,7 @@ import { GithubUrlReader } from './GithubUrlReader';
 import { GitlabUrlReader } from './GitlabUrlReader';
 import { ReadTreeResponseFactory } from './tree';
 import { FetchUrlReader } from './FetchUrlReader';
-import { GcsUrlReader } from './GcsUrlReader';
+import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 
 type CreateOptions = {
   /** Root config object */
@@ -71,7 +71,7 @@ export class UrlReaders {
         BitbucketUrlReader.factory,
         GithubUrlReader.factory,
         GitlabUrlReader.factory,
-        GcsUrlReader.factory,
+        GoogleGcsUrlReader.factory,
         FetchUrlReader.factory,
       ]),
     });
diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts
index f0d72b7ceb..2947ba3889 100644
--- a/packages/integration/config.d.ts
+++ b/packages/integration/config.d.ts
@@ -149,5 +149,19 @@ export interface Config {
        */
       baseUrl?: string;
     }>;
+
+    /** Integration configuration for Google Cloud Storage */
+    googleGcs?: Array<{
+      /**
+       * Service account email used to authenticate requests.
+       * @visibility secret
+       */
+      clientEmail?: string;
+      /**
+       * Service account private key used to authenticate requests.
+       * @visibility secret
+       */
+      privateKey?: string;
+    }>;
   };
 }
diff --git a/packages/integration/src/googleGcs/config.test.ts b/packages/integration/src/googleGcs/config.test.ts
new file mode 100644
index 0000000000..d612343b5e
--- /dev/null
+++ b/packages/integration/src/googleGcs/config.test.ts
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config, ConfigReader } from '@backstage/config';
+import {
+  GoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfigs,
+} from './config';
+
+describe('readGoogleGcsIntegrationConfig', () => {
+  function buildConfig(data: Partial<GoogleGcsIntegrationConfig>): Config {
+    return new ConfigReader(data);
+  }
+
+  it('reads all values', () => {
+    const output = readGoogleGcsIntegrationConfig(
+      buildConfig({
+        privateKey: 'fake-key',
+        clientEmail: 'someone@example.com',
+      }),
+    );
+    expect(output).toEqual({
+      privateKey: 'fake-key',
+      token: 'someone@example.com',
+    });
+  });
+});
+
+describe('readGoogleGcsIntegrationConfigs', () => {
+  function buildConfig(data: Partial<GoogleGcsIntegrationConfig>[]): Config[] {
+    return data.map(item => new ConfigReader(item));
+  }
+
+  it('reads all values', () => {
+    const output = readGoogleGcsIntegrationConfigs(
+      buildConfig([
+        {
+          privateKey: 'fake-key',
+          clientEmail: 'someone@example.com',
+        },
+      ]),
+    );
+    expect(output).toContainEqual({
+      privateKey: 'fake-key',
+      clientEmail: 'someone@example.com',
+    });
+  });
+});
diff --git a/packages/integration/src/googleGcs/config.ts b/packages/integration/src/googleGcs/config.ts
new file mode 100644
index 0000000000..5f70932638
--- /dev/null
+++ b/packages/integration/src/googleGcs/config.ts
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+/**
+ * The configuration parameters for a single Google Cloud Storage provider.
+ */
+export type GoogleGcsIntegrationConfig = {
+  /**
+   * Service account email used to authenticate requests.
+   */
+  clientEmail?: string;
+  /**
+   * Service account private key used to authenticate requests.
+   */
+  privateKey?: string;
+};
+
+/**
+ * Reads a single Google GCS integration config.
+ *
+ * @param config The config object of a single integration
+ */
+export function readGoogleGcsIntegrationConfig(
+  config: Config,
+): GoogleGcsIntegrationConfig {
+  if (!config.has('clientEmail') || !config.has('privateKey')) {
+    return {};
+  }
+
+  const privateKey = config
+    .getOptionalString('privateKey')
+    ?.split('\\n')
+    .join('\n');
+
+  const clientEmail = config.getOptionalString('clientEmail');
+  return { clientEmail: clientEmail, privateKey: privateKey };
+}
+
+/**
+ * Reads a set of Google Cloud Storage integration configs.
+ *
+ * @param configs All of the integration config objects
+ */
+export function readGoogleGcsIntegrationConfigs(
+  configs: Config[],
+): GoogleGcsIntegrationConfig[] {
+  return configs.map(readGoogleGcsIntegrationConfig);
+}
diff --git a/packages/integration/src/googleGcs/index.ts b/packages/integration/src/googleGcs/index.ts
new file mode 100644
index 0000000000..f120256d35
--- /dev/null
+++ b/packages/integration/src/googleGcs/index.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {
+  readGoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfigs,
+} from './config';
+export type { GoogleGcsIntegrationConfig } from './config';
+export const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts
index e89cf6ef92..d15eea0832 100644
--- a/packages/integration/src/index.ts
+++ b/packages/integration/src/index.ts
@@ -18,6 +18,7 @@ export * from './azure';
 export * from './bitbucket';
 export * from './github';
 export * from './gitlab';
+export * from './googleGcs';
 export { defaultScmResolveUrl } from './helpers';
 export { ScmIntegrations } from './ScmIntegrations';
 export type {

From 5bd790984e91fb200aa9d1c23ab5100bb381e1af Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Fri, 19 Mar 2021 17:29:43 +0100
Subject: [PATCH 06/11] Add integration documentation

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 docs/integrations/google-cloud-storage/gcs.md | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 docs/integrations/google-cloud-storage/gcs.md

diff --git a/docs/integrations/google-cloud-storage/gcs.md b/docs/integrations/google-cloud-storage/gcs.md
new file mode 100644
index 0000000000..f9cba7a04c
--- /dev/null
+++ b/docs/integrations/google-cloud-storage/gcs.md
@@ -0,0 +1,57 @@
+---
+id: googlegcs
+title: Google Cloud Storage
+sidebar_label: Google Cloud Storage Data
+# prettier-ignore
+description: Setting up an integration with Google Cloud Storage
+---
+
+The Backstage catalog can import entities from a yaml file stored in a GCS
+(Google Cloud Storage) bucket. To enable the ingestion of said entities the
+`GoogleGcs` integration must be enabled first.
+
+## Configuration
+
+To configure the integration add the appropriate credentials to the Backstage
+backend. There are two main ways to do this: by explicitly setting a
+`clientEmail` and a `privateKey` or by letting the Google Storage SDK discover
+the credentials automatically.
+
+### Explicit credentials
+
+Explicit credentials can be set in the following format:
+
+```yaml
+integrations:
+  googleGcs:
+    - clientEmail:
+        $env: GCS_CLIENT_EMAIL
+      privateKey:
+        $env: GCS_PRIVATE_KEY
+```
+
+Then make sure the environment variables `GCS_CLIENT_EMAIL` and
+`GCS_PRIVATE_KEY` are set when you run Backstage.
+
+### Automatic discovery of Google credentials
+
+Since this integration uses the Google Storage SDK, you can also choose to not
+provide any explicit credentials and let the SDK discover them automatically.
+
+One of these discovery methods is to provide an environment variable called
+`GOOGLE_APPLICATION_CREDENTIALS` and set it to the file path of your JSON
+service account key.
+
+For more details and methods to provide credentials to the Google Storage SDK
+you can check [this documentation page][google gcs docs].
+
+## Usage
+
+To use this integration to import entities from a GCS bucket go to the Google
+console and browse the file you would like to import. Then copy the
+`Authenticated URL` and paste it into the text box in the `register component`
+form. This url should look like
+`https://storage.cloud.google.com/<bucket>/<path>/catalog-info.yaml`.
+
+[google gcs docs]:
+  https://cloud.google.com/docs/authentication/production#auth-cloud-implicit-nodejs

From d5cb51331a42ae8483e2922b2dcad89712231746 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Fri, 19 Mar 2021 17:50:25 +0100
Subject: [PATCH 07/11] Only allow one googleGcs config object

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 app-config.yaml                               |  8 ++--
 docs/integrations/google-cloud-storage/gcs.md |  8 ++--
 .../src/reading/GoogleGcsUrlReader.test.ts    | 21 ++++------
 .../src/reading/GoogleGcsUrlReader.ts         | 42 +++++++++----------
 packages/integration/config.d.ts              |  4 +-
 .../integration/src/googleGcs/config.test.ts  | 23 ++--------
 packages/integration/src/googleGcs/config.ts  | 15 ++-----
 packages/integration/src/googleGcs/index.ts   |  5 +--
 8 files changed, 45 insertions(+), 81 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 49ac0b0d00..3dc071c85c 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -123,10 +123,10 @@ kafka:
 
 integrations:
   googleGcs:
-    - clientEmail:
-        $env: GCS_CLIENT_EMAIL
-      privateKey:
-        $env: GCS_PRIVATE_KEY
+    clientEmail:
+      $env: GCS_CLIENT_EMAIL
+    privateKey:
+      $env: GCS_PRIVATE_KEY
 
   github:
     - host: github.com
diff --git a/docs/integrations/google-cloud-storage/gcs.md b/docs/integrations/google-cloud-storage/gcs.md
index f9cba7a04c..1d46c57b10 100644
--- a/docs/integrations/google-cloud-storage/gcs.md
+++ b/docs/integrations/google-cloud-storage/gcs.md
@@ -24,10 +24,10 @@ Explicit credentials can be set in the following format:
 ```yaml
 integrations:
   googleGcs:
-    - clientEmail:
-        $env: GCS_CLIENT_EMAIL
-      privateKey:
-        $env: GCS_PRIVATE_KEY
+    clientEmail:
+      $env: GCS_CLIENT_EMAIL
+    privateKey:
+      $env: GCS_PRIVATE_KEY
 ```
 
 Then make sure the environment variables `GCS_CLIENT_EMAIL` and
diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts
index c8262767e1..e201361a91 100644
--- a/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts
+++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts
@@ -41,26 +41,19 @@ describe('GcsUrlReader', () => {
   it('creates a reader with credentials correctly configured', () => {
     const entries = createReader({
       integrations: {
-        googleGcs: [
-          {
-            privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---',
-            clientEmail: 'someone@example.com',
-          },
-          {
-            host: 'proxy.storage.cloud.google.com',
-            privateKey: '--- BEGIN KEY ---- fakekey2 --- END KEY ---',
-            clientEmail: 'someone2@example.com',
-          },
-        ],
+        googleGcs: {
+          privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---',
+          clientEmail: 'someone@example.com',
+        },
       },
     });
-    expect(entries).toHaveLength(2);
+    expect(entries).toHaveLength(1);
   });
 
   it('creates a reader with default credentials provider', () => {
     const entries = createReader({
       integrations: {
-        googleGcs: [{}],
+        googleGcs: {},
       },
     });
     expect(entries).toHaveLength(1);
@@ -69,7 +62,7 @@ describe('GcsUrlReader', () => {
   describe('predicates', () => {
     const readers = createReader({
       integrations: {
-        googleGcs: [{}],
+        googleGcs: {},
       },
     });
     const predicate = readers[0].predicate;
diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
index a9100051a6..80b42dd5da 100644
--- a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
+++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
@@ -24,7 +24,7 @@ import {
 import getRawBody from 'raw-body';
 import {
   GOOGLE_GCS_HOST,
-  readGoogleGcsIntegrationConfigs,
+  readGoogleGcsIntegrationConfig,
 } from '@backstage/integration';
 
 const parseURL = (
@@ -49,28 +49,26 @@ export class GoogleGcsUrlReader implements UrlReader {
     if (!config.has('integrations.googleGcs')) {
       return [];
     }
-    const configs = readGoogleGcsIntegrationConfigs(
-      config.getOptionalConfigArray('integrations.googleGcs') ?? [],
+    const gcsConfig = readGoogleGcsIntegrationConfig(
+      config.getConfig('integrations.googleGcs'),
     );
-    return configs.map(integration => {
-      let storage: Storage;
-      if (!integration.clientEmail || !integration.privateKey) {
-        logger.warn(
-          'googleGcs credentials not found in config. Using default credentials provider.',
-        );
-        storage = new Storage();
-      } else {
-        storage = new Storage({
-          credentials: {
-            client_email: integration.clientEmail || undefined,
-            private_key: integration.privateKey || undefined,
-          },
-        });
-      }
-      const reader = new GoogleGcsUrlReader(storage);
-      const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
-      return { reader, predicate };
-    });
+    let storage: Storage;
+    if (!gcsConfig.clientEmail || !gcsConfig.privateKey) {
+      logger.warn(
+        'googleGcs credentials not found in config. Using default credentials provider.',
+      );
+      storage = new Storage();
+    } else {
+      storage = new Storage({
+        credentials: {
+          client_email: gcsConfig.clientEmail || undefined,
+          private_key: gcsConfig.privateKey || undefined,
+        },
+      });
+    }
+    const reader = new GoogleGcsUrlReader(storage);
+    const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
+    return [{ reader, predicate }];
   };
 
   constructor(private readonly storage: Storage) {}
diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts
index 2947ba3889..9f9d588f0b 100644
--- a/packages/integration/config.d.ts
+++ b/packages/integration/config.d.ts
@@ -151,7 +151,7 @@ export interface Config {
     }>;
 
     /** Integration configuration for Google Cloud Storage */
-    googleGcs?: Array<{
+    googleGcs?: {
       /**
        * Service account email used to authenticate requests.
        * @visibility secret
@@ -162,6 +162,6 @@ export interface Config {
        * @visibility secret
        */
       privateKey?: string;
-    }>;
+    };
   };
 }
diff --git a/packages/integration/src/googleGcs/config.test.ts b/packages/integration/src/googleGcs/config.test.ts
index d612343b5e..f688df4071 100644
--- a/packages/integration/src/googleGcs/config.test.ts
+++ b/packages/integration/src/googleGcs/config.test.ts
@@ -18,7 +18,6 @@ import { Config, ConfigReader } from '@backstage/config';
 import {
   GoogleGcsIntegrationConfig,
   readGoogleGcsIntegrationConfig,
-  readGoogleGcsIntegrationConfigs,
 } from './config';
 
 describe('readGoogleGcsIntegrationConfig', () => {
@@ -38,25 +37,9 @@ describe('readGoogleGcsIntegrationConfig', () => {
       token: 'someone@example.com',
     });
   });
-});
 
-describe('readGoogleGcsIntegrationConfigs', () => {
-  function buildConfig(data: Partial<GoogleGcsIntegrationConfig>[]): Config[] {
-    return data.map(item => new ConfigReader(item));
-  }
-
-  it('reads all values', () => {
-    const output = readGoogleGcsIntegrationConfigs(
-      buildConfig([
-        {
-          privateKey: 'fake-key',
-          clientEmail: 'someone@example.com',
-        },
-      ]),
-    );
-    expect(output).toContainEqual({
-      privateKey: 'fake-key',
-      clientEmail: 'someone@example.com',
-    });
+  it('does not fail when config is not set', () => {
+    const output = readGoogleGcsIntegrationConfig(buildConfig({}));
+    expect(output).toEqual({});
   });
 });
diff --git a/packages/integration/src/googleGcs/config.ts b/packages/integration/src/googleGcs/config.ts
index 5f70932638..90a679bb4c 100644
--- a/packages/integration/src/googleGcs/config.ts
+++ b/packages/integration/src/googleGcs/config.ts
@@ -38,6 +38,10 @@ export type GoogleGcsIntegrationConfig = {
 export function readGoogleGcsIntegrationConfig(
   config: Config,
 ): GoogleGcsIntegrationConfig {
+  if (!config) {
+    return {};
+  }
+
   if (!config.has('clientEmail') || !config.has('privateKey')) {
     return {};
   }
@@ -50,14 +54,3 @@ export function readGoogleGcsIntegrationConfig(
   const clientEmail = config.getOptionalString('clientEmail');
   return { clientEmail: clientEmail, privateKey: privateKey };
 }
-
-/**
- * Reads a set of Google Cloud Storage integration configs.
- *
- * @param configs All of the integration config objects
- */
-export function readGoogleGcsIntegrationConfigs(
-  configs: Config[],
-): GoogleGcsIntegrationConfig[] {
-  return configs.map(readGoogleGcsIntegrationConfig);
-}
diff --git a/packages/integration/src/googleGcs/index.ts b/packages/integration/src/googleGcs/index.ts
index f120256d35..ac85d706b5 100644
--- a/packages/integration/src/googleGcs/index.ts
+++ b/packages/integration/src/googleGcs/index.ts
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-export {
-  readGoogleGcsIntegrationConfig,
-  readGoogleGcsIntegrationConfigs,
-} from './config';
+export { readGoogleGcsIntegrationConfig } from './config';
 export type { GoogleGcsIntegrationConfig } from './config';
 export const GOOGLE_GCS_HOST = 'storage.cloud.google.com';

From 18f4301d3baea0e46dd952a9e77217ce8d035f22 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Fri, 19 Mar 2021 18:13:52 +0100
Subject: [PATCH 08/11] Fix googleGcs integration test

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 packages/integration/src/googleGcs/config.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/integration/src/googleGcs/config.test.ts b/packages/integration/src/googleGcs/config.test.ts
index f688df4071..a924e953b9 100644
--- a/packages/integration/src/googleGcs/config.test.ts
+++ b/packages/integration/src/googleGcs/config.test.ts
@@ -34,7 +34,7 @@ describe('readGoogleGcsIntegrationConfig', () => {
     );
     expect(output).toEqual({
       privateKey: 'fake-key',
-      token: 'someone@example.com',
+      clientEmail: 'someone@example.com',
     });
   });
 

From 6a9b9d0f7c1eca836bb52687921308a6cf5bce61 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Mon, 29 Mar 2021 10:14:52 +0200
Subject: [PATCH 09/11] Fix some comments

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 app-config.yaml                                        | 10 ++++------
 .../backend-common/src/reading/GoogleGcsUrlReader.ts   |  9 ++++-----
 packages/integration/config.d.ts                       |  2 +-
 packages/integration/src/googleGcs/index.ts            |  1 -
 4 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 3dc071c85c..48d58dc837 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -122,12 +122,6 @@ kafka:
         - localhost:9092
 
 integrations:
-  googleGcs:
-    clientEmail:
-      $env: GCS_CLIENT_EMAIL
-    privateKey:
-      $env: GCS_PRIVATE_KEY
-
   github:
     - host: github.com
       token:
@@ -156,6 +150,10 @@ integrations:
     - host: dev.azure.com
       token:
         $env: AZURE_TOKEN
+#  googleGcs:
+#    clientEmail:
+#      $env: GCS_CLIENT_EMAIL
+#    privateKey: 'example@example.com'
 
 catalog:
   rules:
diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
index 80b42dd5da..a05a41e54a 100644
--- a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
+++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
@@ -22,10 +22,9 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
-import {
-  GOOGLE_GCS_HOST,
-  readGoogleGcsIntegrationConfig,
-} from '@backstage/integration';
+import { readGoogleGcsIntegrationConfig } from '@backstage/integration';
+
+const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
 
 const parseURL = (
   url: string,
@@ -81,7 +80,7 @@ export class GoogleGcsUrlReader implements UrlReader {
         this.storage.bucket(bucket).file(key).createReadStream(),
       );
     } catch (error) {
-      throw new Error(`unable to read gcs file from ${url}`);
+      throw new Error(`unable to read gcs file from ${url}, ${error}`);
     }
   }
 
diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts
index 9f9d588f0b..ddc84ae011 100644
--- a/packages/integration/config.d.ts
+++ b/packages/integration/config.d.ts
@@ -154,7 +154,7 @@ export interface Config {
     googleGcs?: {
       /**
        * Service account email used to authenticate requests.
-       * @visibility secret
+       * @visibility backend
        */
       clientEmail?: string;
       /**
diff --git a/packages/integration/src/googleGcs/index.ts b/packages/integration/src/googleGcs/index.ts
index ac85d706b5..d9355fd299 100644
--- a/packages/integration/src/googleGcs/index.ts
+++ b/packages/integration/src/googleGcs/index.ts
@@ -16,4 +16,3 @@
 
 export { readGoogleGcsIntegrationConfig } from './config';
 export type { GoogleGcsIntegrationConfig } from './config';
-export const GOOGLE_GCS_HOST = 'storage.cloud.google.com';

From c410cae0f2e141749653dbab5a2f5d45ba245343 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Wed, 7 Apr 2021 12:06:07 +0200
Subject: [PATCH 10/11] Fix feedback

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 app-config.yaml                                 |  6 +++---
 .../src/reading/GoogleGcsUrlReader.ts           | 17 ++++++++++++-----
 packages/integration/src/googleGcs/config.ts    |  9 +++------
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 48d58dc837..309fa28234 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -151,9 +151,9 @@ integrations:
       token:
         $env: AZURE_TOKEN
 #  googleGcs:
-#    clientEmail:
-#      $env: GCS_CLIENT_EMAIL
-#    privateKey: 'example@example.com'
+#    clientEmail: 'example@example.com'
+#    privateKey:
+#      $env: GCS_PRIVATE_KEY
 
 catalog:
   rules:
diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
index a05a41e54a..4ea24fa22f 100644
--- a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
+++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts
@@ -22,7 +22,10 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
-import { readGoogleGcsIntegrationConfig } from '@backstage/integration';
+import {
+  GoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfig,
+} from '@backstage/integration';
 
 const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
 
@@ -53,7 +56,7 @@ export class GoogleGcsUrlReader implements UrlReader {
     );
     let storage: Storage;
     if (!gcsConfig.clientEmail || !gcsConfig.privateKey) {
-      logger.warn(
+      logger.info(
         'googleGcs credentials not found in config. Using default credentials provider.',
       );
       storage = new Storage();
@@ -65,12 +68,15 @@ export class GoogleGcsUrlReader implements UrlReader {
         },
       });
     }
-    const reader = new GoogleGcsUrlReader(storage);
+    const reader = new GoogleGcsUrlReader(gcsConfig, storage);
     const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
     return [{ reader, predicate }];
   };
 
-  constructor(private readonly storage: Storage) {}
+  constructor(
+    private readonly integration: GoogleGcsIntegrationConfig,
+    private readonly storage: Storage,
+  ) {}
 
   async read(url: string): Promise<Buffer> {
     try {
@@ -93,6 +99,7 @@ export class GoogleGcsUrlReader implements UrlReader {
   }
 
   toString() {
-    return `gcs{host=${GOOGLE_GCS_HOST},authed=true}}`;
+    const key = this.integration.privateKey;
+    return `googleGcs{host=${GOOGLE_GCS_HOST},authed=${Boolean(key)}}`;
   }
 }
diff --git a/packages/integration/src/googleGcs/config.ts b/packages/integration/src/googleGcs/config.ts
index 90a679bb4c..7924138b5a 100644
--- a/packages/integration/src/googleGcs/config.ts
+++ b/packages/integration/src/googleGcs/config.ts
@@ -42,15 +42,12 @@ export function readGoogleGcsIntegrationConfig(
     return {};
   }
 
-  if (!config.has('clientEmail') || !config.has('privateKey')) {
+  if (!config.has('clientEmail') && !config.has('privateKey')) {
     return {};
   }
 
-  const privateKey = config
-    .getOptionalString('privateKey')
-    ?.split('\\n')
-    .join('\n');
+  const privateKey = config.getString('privateKey').split('\\n').join('\n');
 
-  const clientEmail = config.getOptionalString('clientEmail');
+  const clientEmail = config.getString('clientEmail');
   return { clientEmail: clientEmail, privateKey: privateKey };
 }

From fa188885177a2fb650182fbacd375d628567fd27 Mon Sep 17 00:00:00 2001
From: Martina Iglesias Fernandez <martina@roadie.io>
Date: Wed, 7 Apr 2021 18:00:24 +0200
Subject: [PATCH 11/11] Fix sidebar docs for GCS integration

Signed-off-by: Martina Iglesias Fernandez <martina@roadie.io>
---
 .../google-cloud-storage/{gcs.md => locations.md}           | 6 +++---
 microsite/sidebars.json                                     | 5 +++++
 2 files changed, 8 insertions(+), 3 deletions(-)
 rename docs/integrations/google-cloud-storage/{gcs.md => locations.md} (95%)

diff --git a/docs/integrations/google-cloud-storage/gcs.md b/docs/integrations/google-cloud-storage/locations.md
similarity index 95%
rename from docs/integrations/google-cloud-storage/gcs.md
rename to docs/integrations/google-cloud-storage/locations.md
index 1d46c57b10..e3f6b72847 100644
--- a/docs/integrations/google-cloud-storage/gcs.md
+++ b/docs/integrations/google-cloud-storage/locations.md
@@ -1,7 +1,7 @@
 ---
-id: googlegcs
-title: Google Cloud Storage
-sidebar_label: Google Cloud Storage Data
+id: locations
+sidebar_label: Locations
+title: Google Cloud Storage Locations
 # prettier-ignore
 description: Setting up an integration with Google Cloud Storage
 ---
diff --git a/microsite/sidebars.json b/microsite/sidebars.json
index b719f9329c..5f3015582d 100644
--- a/microsite/sidebars.json
+++ b/microsite/sidebars.json
@@ -139,6 +139,11 @@
         "type": "subcategory",
         "label": "LDAP",
         "ids": ["integrations/ldap/org"]
+      },
+      {
+        "type": "subcategory",
+        "label": "Google GCS",
+        "ids": ["integrations/google-cloud-storage/locations"]
       }
     ],
     "Plugins": [