diff --git a/.changeset/sour-clocks-pay.md b/.changeset/sour-clocks-pay.md new file mode 100644 index 0000000000..11196aeebe --- /dev/null +++ b/.changeset/sour-clocks-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Add UrlReader for Google Cloud Storage diff --git a/app-config.yaml b/app-config.yaml index 465c7b0049..309fa28234 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -150,6 +150,10 @@ integrations: - host: dev.azure.com token: $env: AZURE_TOKEN +# googleGcs: +# clientEmail: 'example@example.com' +# privateKey: +# $env: GCS_PRIVATE_KEY catalog: rules: diff --git a/docs/integrations/google-cloud-storage/locations.md b/docs/integrations/google-cloud-storage/locations.md new file mode 100644 index 0000000000..e3f6b72847 --- /dev/null +++ b/docs/integrations/google-cloud-storage/locations.md @@ -0,0 +1,57 @@ +--- +id: locations +sidebar_label: Locations +title: Google Cloud Storage Locations +# prettier-ignore +description: Setting up an integration with Google Cloud Storage +--- + +The Backstage catalog can import entities from a yaml file stored in a GCS +(Google Cloud Storage) bucket. To enable the ingestion of said entities the +`GoogleGcs` integration must be enabled first. + +## Configuration + +To configure the integration add the appropriate credentials to the Backstage +backend. There are two main ways to do this: by explicitly setting a +`clientEmail` and a `privateKey` or by letting the Google Storage SDK discover +the credentials automatically. + +### Explicit credentials + +Explicit credentials can be set in the following format: + +```yaml +integrations: + googleGcs: + clientEmail: + $env: GCS_CLIENT_EMAIL + privateKey: + $env: GCS_PRIVATE_KEY +``` + +Then make sure the environment variables `GCS_CLIENT_EMAIL` and +`GCS_PRIVATE_KEY` are set when you run Backstage. + +### Automatic discovery of Google credentials + +Since this integration uses the Google Storage SDK, you can also choose to not +provide any explicit credentials and let the SDK discover them automatically. + +One of these discovery methods is to provide an environment variable called +`GOOGLE_APPLICATION_CREDENTIALS` and set it to the file path of your JSON +service account key. + +For more details and methods to provide credentials to the Google Storage SDK +you can check [this documentation page][google gcs docs]. + +## Usage + +To use this integration to import entities from a GCS bucket go to the Google +console and browse the file you would like to import. Then copy the +`Authenticated URL` and paste it into the text box in the `register component` +form. This url should look like +`https://storage.cloud.google.com///catalog-info.yaml`. + +[google gcs docs]: + https://cloud.google.com/docs/authentication/production#auth-cloud-implicit-nodejs diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 3eb887d357..64ff108a78 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -142,6 +142,11 @@ "type": "subcategory", "label": "LDAP", "ids": ["integrations/ldap/org"] + }, + { + "type": "subcategory", + "label": "Google GCS", + "ids": ["integrations/google-cloud-storage/locations"] } ], "Plugins": [ diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index b860336d31..ba846e0f5f 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -34,6 +34,7 @@ "@backstage/config-loader": "^0.5.1", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", + "@google-cloud/storage": "^5.8.0", "@octokit/rest": "^18.0.12", "@types/cors": "^2.8.6", "@types/dockerode": "^3.2.1", @@ -56,6 +57,7 @@ "minimatch": "^3.0.4", "minimist": "^1.2.5", "morgan": "^1.10.0", + "raw-body": "^2.4.1", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", "tar": "^6.0.5", diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts new file mode 100644 index 0000000000..e201361a91 --- /dev/null +++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader, JsonObject } from '@backstage/config'; +import { getVoidLogger } from '../logging'; +import { ReadTreeResponseFactory } from './tree'; +import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; +import { UrlReaderPredicateTuple } from './types'; + +describe('GcsUrlReader', () => { + const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { + return GoogleGcsUrlReader.factory({ + config: new ConfigReader(config), + logger: getVoidLogger(), + treeResponseFactory: ReadTreeResponseFactory.create({ + config: new ConfigReader({}), + }), + }); + }; + + it('does not create a reader without the googleGcs field', () => { + const entries = createReader({ + integrations: {}, + }); + expect(entries).toHaveLength(0); + }); + + it('creates a reader with credentials correctly configured', () => { + const entries = createReader({ + integrations: { + googleGcs: { + privateKey: '--- BEGIN KEY ---- fakekey --- END KEY ---', + clientEmail: 'someone@example.com', + }, + }, + }); + expect(entries).toHaveLength(1); + }); + + it('creates a reader with default credentials provider', () => { + const entries = createReader({ + integrations: { + googleGcs: {}, + }, + }); + expect(entries).toHaveLength(1); + }); + + describe('predicates', () => { + const readers = createReader({ + integrations: { + googleGcs: {}, + }, + }); + const predicate = readers[0].predicate; + + it('returns true for the correct google cloud storage host', () => { + expect(predicate(new URL('https://storage.cloud.google.com'))).toBe(true); + }); + it('returns true for a url with the full path and the correct host', () => { + expect( + predicate( + new URL( + 'https://storage.cloud.google.com/team1/service1/catalog-info.yaml', + ), + ), + ).toBe(true); + }); + it('returns false for the wrong hostname under cloud.google.com', () => { + expect(predicate(new URL('https://storage2.cloud.google.com'))).toBe( + false, + ); + }); + it('returns false for a partially correct host', () => { + expect(predicate(new URL('https://cloud.google.com'))).toBe(false); + }); + it('returns false for a completely different host', () => { + expect(predicate(new URL('https://a.example.com/test'))).toBe(false); + }); + }); +}); diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts new file mode 100644 index 0000000000..4ea24fa22f --- /dev/null +++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Storage } from '@google-cloud/storage'; +import { + ReaderFactory, + ReadTreeResponse, + SearchResponse, + UrlReader, +} from './types'; +import getRawBody from 'raw-body'; +import { + GoogleGcsIntegrationConfig, + readGoogleGcsIntegrationConfig, +} from '@backstage/integration'; + +const GOOGLE_GCS_HOST = 'storage.cloud.google.com'; + +const parseURL = ( + url: string, +): { host: string; bucket: string; key: string } => { + const { host, pathname } = new URL(url); + + if (host !== GOOGLE_GCS_HOST) { + throw new Error(`not a valid GCS URL: ${url}`); + } + + const [, bucket, ...key] = pathname.split('/'); + return { + host: host, + bucket, + key: key.join('/'), + }; +}; + +export class GoogleGcsUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config, logger }) => { + if (!config.has('integrations.googleGcs')) { + return []; + } + const gcsConfig = readGoogleGcsIntegrationConfig( + config.getConfig('integrations.googleGcs'), + ); + let storage: Storage; + if (!gcsConfig.clientEmail || !gcsConfig.privateKey) { + logger.info( + 'googleGcs credentials not found in config. Using default credentials provider.', + ); + storage = new Storage(); + } else { + storage = new Storage({ + credentials: { + client_email: gcsConfig.clientEmail || undefined, + private_key: gcsConfig.privateKey || undefined, + }, + }); + } + const reader = new GoogleGcsUrlReader(gcsConfig, storage); + const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST; + return [{ reader, predicate }]; + }; + + constructor( + private readonly integration: GoogleGcsIntegrationConfig, + private readonly storage: Storage, + ) {} + + async read(url: string): Promise { + try { + const { bucket, key } = parseURL(url); + + return await getRawBody( + this.storage.bucket(bucket).file(key).createReadStream(), + ); + } catch (error) { + throw new Error(`unable to read gcs file from ${url}, ${error}`); + } + } + + async readTree(): Promise { + throw new Error('GcsUrlReader does not implement readTree'); + } + + async search(): Promise { + throw new Error('GcsUrlReader does not implement search'); + } + + toString() { + const key = this.integration.privateKey; + return `googleGcs{host=${GOOGLE_GCS_HOST},authed=${Boolean(key)}}`; + } +} diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2f233463bb..06c6fdcdcf 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -24,6 +24,7 @@ import { GithubUrlReader } from './GithubUrlReader'; import { GitlabUrlReader } from './GitlabUrlReader'; import { ReadTreeResponseFactory } from './tree'; import { FetchUrlReader } from './FetchUrlReader'; +import { GoogleGcsUrlReader } from './GoogleGcsUrlReader'; type CreateOptions = { /** Root config object */ @@ -70,6 +71,7 @@ export class UrlReaders { BitbucketUrlReader.factory, GithubUrlReader.factory, GitlabUrlReader.factory, + GoogleGcsUrlReader.factory, FetchUrlReader.factory, ]), }); diff --git a/packages/integration/config.d.ts b/packages/integration/config.d.ts index f0d72b7ceb..ddc84ae011 100644 --- a/packages/integration/config.d.ts +++ b/packages/integration/config.d.ts @@ -149,5 +149,19 @@ export interface Config { */ baseUrl?: string; }>; + + /** Integration configuration for Google Cloud Storage */ + googleGcs?: { + /** + * Service account email used to authenticate requests. + * @visibility backend + */ + clientEmail?: string; + /** + * Service account private key used to authenticate requests. + * @visibility secret + */ + privateKey?: string; + }; }; } diff --git a/packages/integration/src/googleGcs/config.test.ts b/packages/integration/src/googleGcs/config.test.ts new file mode 100644 index 0000000000..a924e953b9 --- /dev/null +++ b/packages/integration/src/googleGcs/config.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, ConfigReader } from '@backstage/config'; +import { + GoogleGcsIntegrationConfig, + readGoogleGcsIntegrationConfig, +} from './config'; + +describe('readGoogleGcsIntegrationConfig', () => { + function buildConfig(data: Partial): Config { + return new ConfigReader(data); + } + + it('reads all values', () => { + const output = readGoogleGcsIntegrationConfig( + buildConfig({ + privateKey: 'fake-key', + clientEmail: 'someone@example.com', + }), + ); + expect(output).toEqual({ + privateKey: 'fake-key', + clientEmail: 'someone@example.com', + }); + }); + + it('does not fail when config is not set', () => { + const output = readGoogleGcsIntegrationConfig(buildConfig({})); + expect(output).toEqual({}); + }); +}); diff --git a/packages/integration/src/googleGcs/config.ts b/packages/integration/src/googleGcs/config.ts new file mode 100644 index 0000000000..7924138b5a --- /dev/null +++ b/packages/integration/src/googleGcs/config.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +/** + * The configuration parameters for a single Google Cloud Storage provider. + */ +export type GoogleGcsIntegrationConfig = { + /** + * Service account email used to authenticate requests. + */ + clientEmail?: string; + /** + * Service account private key used to authenticate requests. + */ + privateKey?: string; +}; + +/** + * Reads a single Google GCS integration config. + * + * @param config The config object of a single integration + */ +export function readGoogleGcsIntegrationConfig( + config: Config, +): GoogleGcsIntegrationConfig { + if (!config) { + return {}; + } + + if (!config.has('clientEmail') && !config.has('privateKey')) { + return {}; + } + + const privateKey = config.getString('privateKey').split('\\n').join('\n'); + + const clientEmail = config.getString('clientEmail'); + return { clientEmail: clientEmail, privateKey: privateKey }; +} diff --git a/packages/integration/src/googleGcs/index.ts b/packages/integration/src/googleGcs/index.ts new file mode 100644 index 0000000000..d9355fd299 --- /dev/null +++ b/packages/integration/src/googleGcs/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { readGoogleGcsIntegrationConfig } from './config'; +export type { GoogleGcsIntegrationConfig } from './config'; diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index e89cf6ef92..d15eea0832 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -18,6 +18,7 @@ export * from './azure'; export * from './bitbucket'; export * from './github'; export * from './gitlab'; +export * from './googleGcs'; export { defaultScmResolveUrl } from './helpers'; export { ScmIntegrations } from './ScmIntegrations'; export type { diff --git a/yarn.lock b/yarn.lock index 4b1dea9b04..e335f31c18 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2570,6 +2570,33 @@ stream-events "^1.0.1" xdg-basedir "^4.0.0" +"@google-cloud/storage@^5.8.0": + version "5.8.0" + resolved "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz#1f580e276f1d453790b382156421d1bcc4bd3f4b" + integrity sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g== + dependencies: + "@google-cloud/common" "^3.6.0" + "@google-cloud/paginator" "^3.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.0" + async-retry "^1.3.1" + compressible "^2.0.12" + date-and-time "^0.14.2" + duplexify "^4.0.0" + extend "^3.0.2" + gaxios "^4.0.0" + gcs-resumable-upload "^3.1.3" + get-stream "^6.0.0" + hash-stream-validation "^0.2.2" + mime "^2.2.0" + mime-types "^2.0.8" + onetime "^5.1.0" + p-limit "^3.0.1" + pumpify "^2.0.0" + snakeize "^0.1.0" + stream-events "^1.0.1" + xdg-basedir "^4.0.0" + "@graphiql/toolkit@^0.1.0": version "0.1.1" resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.1.1.tgz#a7da3ba460ceae27bcdc8f03831ca4f88f90f3d7" @@ -15099,6 +15126,17 @@ http-errors@1.7.2: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" +http-errors@1.7.3, http-errors@~1.7.2: + version "1.7.3" + resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" + integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + http-errors@^1.7.3: version "1.8.0" resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.8.0.tgz#75d1bbe497e1044f51e4ee9e704a62f28d336507" @@ -15120,17 +15158,6 @@ http-errors@~1.6.2: setprototypeof "1.1.0" statuses ">= 1.4.0 < 2" -http-errors@~1.7.2: - version "1.7.3" - resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" - integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== - dependencies: - depd "~1.1.2" - inherits "2.0.4" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" - "http-parser-js@>=0.4.0 <0.4.11": version "0.4.10" resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.10.tgz#92c9c1374c35085f75db359ec56cc257cbb93fa4" @@ -21928,6 +21955,16 @@ raw-body@2.4.0: iconv-lite "0.4.24" unpipe "1.0.0" +raw-body@^2.4.1: + version "2.4.1" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.4.1.tgz#30ac82f98bb5ae8c152e67149dac8d55153b168c" + integrity sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA== + dependencies: + bytes "3.1.0" + http-errors "1.7.3" + iconv-lite "0.4.24" + unpipe "1.0.0" + raw-loader@^4.0.1: version "4.0.2" resolved "https://registry.npmjs.org/raw-loader/-/raw-loader-4.0.2.tgz#1aac6b7d1ad1501e66efdac1522c73e59a584eb6"