Merge pull request #4954 from RoadieHQ/gcs-url-reader

Gcs url reader

.changeset/sour-clocks-pay.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Add UrlReader for Google Cloud Storage

app-config.yaml

@@ -150,6 +150,10 @@ integrations:
     - host: dev.azure.com
       token:
         $env: AZURE_TOKEN
+#  googleGcs:
+#    clientEmail: 'example@example.com'
+#    privateKey:
+#      $env: GCS_PRIVATE_KEY
 
 catalog:
   rules:

docs/integrations/google-cloud-storage/locations.md

@@ -0,0 +1,57 @@
+---
+id: locations
+sidebar_label: Locations
+title: Google Cloud Storage Locations
+# prettier-ignore
+description: Setting up an integration with Google Cloud Storage
+---
+
+The Backstage catalog can import entities from a yaml file stored in a GCS
+(Google Cloud Storage) bucket. To enable the ingestion of said entities the
+`GoogleGcs` integration must be enabled first.
+
+## Configuration
+
+To configure the integration add the appropriate credentials to the Backstage
+backend. There are two main ways to do this: by explicitly setting a
+`clientEmail` and a `privateKey` or by letting the Google Storage SDK discover
+the credentials automatically.
+
+### Explicit credentials
+
+Explicit credentials can be set in the following format:
+
+```yaml
+integrations:
+  googleGcs:
+    clientEmail:
+      $env: GCS_CLIENT_EMAIL
+    privateKey:
+      $env: GCS_PRIVATE_KEY
+```
+
+Then make sure the environment variables `GCS_CLIENT_EMAIL` and
+`GCS_PRIVATE_KEY` are set when you run Backstage.
+
+### Automatic discovery of Google credentials
+
+Since this integration uses the Google Storage SDK, you can also choose to not
+provide any explicit credentials and let the SDK discover them automatically.
+
+One of these discovery methods is to provide an environment variable called
+`GOOGLE_APPLICATION_CREDENTIALS` and set it to the file path of your JSON
+service account key.
+
+For more details and methods to provide credentials to the Google Storage SDK
+you can check [this documentation page][google gcs docs].
+
+## Usage
+
+To use this integration to import entities from a GCS bucket go to the Google
+console and browse the file you would like to import. Then copy the
+`Authenticated URL` and paste it into the text box in the `register component`
+form. This url should look like
+`https://storage.cloud.google.com/<bucket>/<path>/catalog-info.yaml`.
+
+[google gcs docs]:
+  https://cloud.google.com/docs/authentication/production#auth-cloud-implicit-nodejs

microsite/sidebars.json

@@ -142,6 +142,11 @@
         "type": "subcategory",
         "label": "LDAP",
         "ids": ["integrations/ldap/org"]
+      },
+      {
+        "type": "subcategory",
+        "label": "Google GCS",
+        "ids": ["integrations/google-cloud-storage/locations"]
       }
     ],
     "Plugins": [

packages/backend-common/package.json

@@ -34,6 +34,7 @@
     "@backstage/config-loader": "^0.5.1",
     "@backstage/errors": "^0.1.1",
     "@backstage/integration": "^0.5.1",
+    "@google-cloud/storage": "^5.8.0",
     "@octokit/rest": "^18.0.12",
     "@types/cors": "^2.8.6",
     "@types/dockerode": "^3.2.1",
@@ -56,6 +57,7 @@
     "minimatch": "^3.0.4",
     "minimist": "^1.2.5",
     "morgan": "^1.10.0",
+    "raw-body": "^2.4.1",
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
     "tar": "^6.0.5",

packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader, JsonObject } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { ReadTreeResponseFactory } from './tree';
+import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
+import { UrlReaderPredicateTuple } from './types';
+
+describe('GcsUrlReader', () => {
+  const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
+    return GoogleGcsUrlReader.factory({
+      config: new ConfigReader(config),
+      logger: getVoidLogger(),
+      treeResponseFactory: ReadTreeResponseFactory.create({
+        config: new ConfigReader({}),
+      }),
+    });
+  };
+
+  it('does not create a reader without the googleGcs field', () => {
+    const entries = createReader({
+      integrations: {},
+    });
+    expect(entries).toHaveLength(0);
+  });
+
+  it('creates a reader with credentials correctly configured', () => {
+    const entries = createReader({
+      integrations: {
+        googleGcs: {
+          privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---',
+          clientEmail: 'someone@example.com',
+        },
+      },
+    });
+    expect(entries).toHaveLength(1);
+  });
+
+  it('creates a reader with default credentials provider', () => {
+    const entries = createReader({
+      integrations: {
+        googleGcs: {},
+      },
+    });
+    expect(entries).toHaveLength(1);
+  });
+
+  describe('predicates', () => {
+    const readers = createReader({
+      integrations: {
+        googleGcs: {},
+      },
+    });
+    const predicate = readers[0].predicate;
+
+    it('returns true for the correct google cloud storage host', () => {
+      expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(true);
+    });
+    it('returns true for a url with the full path and the correct host', () => {
+      expect(
+        predicate(
+          new URL(
+            'https://storage.cloud.google.com/team1/service1/catalog-info.yaml',
+          ),
+        ),
+      ).toBe(true);
+    });
+    it('returns false for the wrong hostname under cloud.google.com', () => {
+      expect(predicate(new URL('https://storage2.cloud.google.com'))).toBe(
+        false,
+      );
+    });
+    it('returns false for a partially correct host', () => {
+      expect(predicate(new URL('https://cloud.google.com'))).toBe(false);
+    });
+    it('returns false for a completely different host', () => {
+      expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
+    });
+  });
+});

packages/backend-common/src/reading/GoogleGcsUrlReader.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Storage } from '@google-cloud/storage';
+import {
+  ReaderFactory,
+  ReadTreeResponse,
+  SearchResponse,
+  UrlReader,
+} from './types';
+import getRawBody from 'raw-body';
+import {
+  GoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfig,
+} from '@backstage/integration';
+
+const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
+
+const parseURL = (
+  url: string,
+): { host: string; bucket: string; key: string } => {
+  const { host, pathname } = new URL(url);
+
+  if (host !== GOOGLE_GCS_HOST) {
+    throw new Error(`not a valid GCS URL: ${url}`);
+  }
+
+  const [, bucket, ...key] = pathname.split('/');
+  return {
+    host: host,
+    bucket,
+    key: key.join('/'),
+  };
+};
+
+export class GoogleGcsUrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config, logger }) => {
+    if (!config.has('integrations.googleGcs')) {
+      return [];
+    }
+    const gcsConfig = readGoogleGcsIntegrationConfig(
+      config.getConfig('integrations.googleGcs'),
+    );
+    let storage: Storage;
+    if (!gcsConfig.clientEmail || !gcsConfig.privateKey) {
+      logger.info(
+        'googleGcs credentials not found in config. Using default credentials provider.',
+      );
+      storage = new Storage();
+    } else {
+      storage = new Storage({
+        credentials: {
+          client_email: gcsConfig.clientEmail || undefined,
+          private_key: gcsConfig.privateKey || undefined,
+        },
+      });
+    }
+    const reader = new GoogleGcsUrlReader(gcsConfig, storage);
+    const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
+    return [{ reader, predicate }];
+  };
+
+  constructor(
+    private readonly integration: GoogleGcsIntegrationConfig,
+    private readonly storage: Storage,
+  ) {}
+
+  async read(url: string): Promise<Buffer> {
+    try {
+      const { bucket, key } = parseURL(url);
+
+      return await getRawBody(
+        this.storage.bucket(bucket).file(key).createReadStream(),
+      );
+    } catch (error) {
+      throw new Error(`unable to read gcs file from ${url}, ${error}`);
+    }
+  }
+
+  async readTree(): Promise<ReadTreeResponse> {
+    throw new Error('GcsUrlReader does not implement readTree');
+  }
+
+  async search(): Promise<SearchResponse> {
+    throw new Error('GcsUrlReader does not implement search');
+  }
+
+  toString() {
+    const key = this.integration.privateKey;
+    return `googleGcs{host=${GOOGLE_GCS_HOST},authed=${Boolean(key)}}`;
+  }
+}

packages/backend-common/src/reading/UrlReaders.ts

@@ -24,6 +24,7 @@ import { GithubUrlReader } from './GithubUrlReader';
 import { GitlabUrlReader } from './GitlabUrlReader';
 import { ReadTreeResponseFactory } from './tree';
 import { FetchUrlReader } from './FetchUrlReader';
+import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 
 type CreateOptions = {
   /** Root config object */
@@ -70,6 +71,7 @@ export class UrlReaders {
         BitbucketUrlReader.factory,
         GithubUrlReader.factory,
         GitlabUrlReader.factory,
+        GoogleGcsUrlReader.factory,
         FetchUrlReader.factory,
       ]),
     });

packages/integration/config.d.ts

@@ -149,5 +149,19 @@ export interface Config {
        */
       baseUrl?: string;
     }>;
+
+    /** Integration configuration for Google Cloud Storage */
+    googleGcs?: {
+      /**
+       * Service account email used to authenticate requests.
+       * @visibility backend
+       */
+      clientEmail?: string;
+      /**
+       * Service account private key used to authenticate requests.
+       * @visibility secret
+       */
+      privateKey?: string;
+    };
   };
 }

packages/integration/src/googleGcs/config.test.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config, ConfigReader } from '@backstage/config';
+import {
+  GoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfig,
+} from './config';
+
+describe('readGoogleGcsIntegrationConfig', () => {
+  function buildConfig(data: Partial<GoogleGcsIntegrationConfig>): Config {
+    return new ConfigReader(data);
+  }
+
+  it('reads all values', () => {
+    const output = readGoogleGcsIntegrationConfig(
+      buildConfig({
+        privateKey: 'fake-key',
+        clientEmail: 'someone@example.com',
+      }),
+    );
+    expect(output).toEqual({
+      privateKey: 'fake-key',
+      clientEmail: 'someone@example.com',
+    });
+  });
+
+  it('does not fail when config is not set', () => {
+    const output = readGoogleGcsIntegrationConfig(buildConfig({}));
+    expect(output).toEqual({});
+  });
+});

packages/integration/src/googleGcs/config.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+/**
+ * The configuration parameters for a single Google Cloud Storage provider.
+ */
+export type GoogleGcsIntegrationConfig = {
+  /**
+   * Service account email used to authenticate requests.
+   */
+  clientEmail?: string;
+  /**
+   * Service account private key used to authenticate requests.
+   */
+  privateKey?: string;
+};
+
+/**
+ * Reads a single Google GCS integration config.
+ *
+ * @param config The config object of a single integration
+ */
+export function readGoogleGcsIntegrationConfig(
+  config: Config,
+): GoogleGcsIntegrationConfig {
+  if (!config) {
+    return {};
+  }
+
+  if (!config.has('clientEmail') && !config.has('privateKey')) {
+    return {};
+  }
+
+  const privateKey = config.getString('privateKey').split('\\n').join('\n');
+
+  const clientEmail = config.getString('clientEmail');
+  return { clientEmail: clientEmail, privateKey: privateKey };
+}

packages/integration/src/googleGcs/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { readGoogleGcsIntegrationConfig } from './config';
+export type { GoogleGcsIntegrationConfig } from './config';

packages/integration/src/index.ts

@@ -18,6 +18,7 @@ export * from './azure';
 export * from './bitbucket';
 export * from './github';
 export * from './gitlab';
+export * from './googleGcs';
 export { defaultScmResolveUrl } from './helpers';
 export { ScmIntegrations } from './ScmIntegrations';
 export type {

yarn.lock

@@ -2570,6 +2570,33 @@
     stream-events "^1.0.1"
     xdg-basedir "^4.0.0"
 
+"@google-cloud/storage@^5.8.0":
+  version "5.8.0"
+  resolved "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz#1f580e276f1d453790b382156421d1bcc4bd3f4b"
+  integrity sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g==
+  dependencies:
+    "@google-cloud/common" "^3.6.0"
+    "@google-cloud/paginator" "^3.0.0"
+    "@google-cloud/promisify" "^2.0.0"
+    arrify "^2.0.0"
+    async-retry "^1.3.1"
+    compressible "^2.0.12"
+    date-and-time "^0.14.2"
+    duplexify "^4.0.0"
+    extend "^3.0.2"
+    gaxios "^4.0.0"
+    gcs-resumable-upload "^3.1.3"
+    get-stream "^6.0.0"
+    hash-stream-validation "^0.2.2"
+    mime "^2.2.0"
+    mime-types "^2.0.8"
+    onetime "^5.1.0"
+    p-limit "^3.0.1"
+    pumpify "^2.0.0"
+    snakeize "^0.1.0"
+    stream-events "^1.0.1"
+    xdg-basedir "^4.0.0"
+
 "@graphiql/toolkit@^0.1.0":
   version "0.1.1"
   resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.1.1.tgz#a7da3ba460ceae27bcdc8f03831ca4f88f90f3d7"
@@ -15099,6 +15126,17 @@ http-errors@1.7.2:
     statuses ">= 1.5.0 < 2"
     toidentifier "1.0.0"
 
+http-errors@1.7.3, http-errors@~1.7.2:
+  version "1.7.3"
+  resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06"
+  integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==
+  dependencies:
+    depd "~1.1.2"
+    inherits "2.0.4"
+    setprototypeof "1.1.1"
+    statuses ">= 1.5.0 < 2"
+    toidentifier "1.0.0"
+
 http-errors@^1.7.3:
   version "1.8.0"
   resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.8.0.tgz#75d1bbe497e1044f51e4ee9e704a62f28d336507"
@@ -15120,17 +15158,6 @@ http-errors@~1.6.2:
     setprototypeof "1.1.0"
     statuses ">= 1.4.0 < 2"
 
-http-errors@~1.7.2:
-  version "1.7.3"
-  resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06"
-  integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==
-  dependencies:
-    depd "~1.1.2"
-    inherits "2.0.4"
-    setprototypeof "1.1.1"
-    statuses ">= 1.5.0 < 2"
-    toidentifier "1.0.0"
-
 "http-parser-js@>=0.4.0 <0.4.11":
   version "0.4.10"
   resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.10.tgz#92c9c1374c35085f75db359ec56cc257cbb93fa4"
@@ -21928,6 +21955,16 @@ raw-body@2.4.0:
     iconv-lite "0.4.24"
     unpipe "1.0.0"
 
+raw-body@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.4.1.tgz#30ac82f98bb5ae8c152e67149dac8d55153b168c"
+  integrity sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA==
+  dependencies:
+    bytes "3.1.0"
+    http-errors "1.7.3"
+    iconv-lite "0.4.24"
+    unpipe "1.0.0"
+
 raw-loader@^4.0.1:
   version "4.0.2"
   resolved "https://registry.npmjs.org/raw-loader/-/raw-loader-4.0.2.tgz#1aac6b7d1ad1501e66efdac1522c73e59a584eb6"