diff --git a/app-config.yaml b/app-config.yaml index 465c7b0049..4d12e3e474 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -122,6 +122,13 @@ kafka: - localhost:9092 integrations: + gcs: + - host: 'storage.cloud.google.com' + clientEmail: + $env: GCS_CLIENT_EMAIL + privateKey: + $env: GCS_PRIVATE_KEY + github: - host: github.com token: diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index b860336d31..a720f9e3ab 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -34,6 +34,7 @@ "@backstage/config-loader": "^0.5.1", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", + "@google-cloud/storage": "^5.8.0", "@octokit/rest": "^18.0.12", "@types/cors": "^2.8.6", "@types/dockerode": "^3.2.1", diff --git a/packages/backend-common/src/reading/GcsUrlReader.ts b/packages/backend-common/src/reading/GcsUrlReader.ts new file mode 100644 index 0000000000..23c8f45fe7 --- /dev/null +++ b/packages/backend-common/src/reading/GcsUrlReader.ts @@ -0,0 +1,112 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Storage } from '@google-cloud/storage'; +import { + ReaderFactory, + ReadTreeResponse, + SearchResponse, + UrlReader, +} from './types'; +import getRawBody from 'raw-body'; +import { Logger } from 'winston'; + +const parseURL = ( + url: string, +): { host: string; bucket: string; key: string } => { + const { host, pathname } = new URL(url); + + if (host !== 'storage.cloud.google.com') { + throw new Error(`not a valid GCS URL: ${url}`); + } + + const [, bucket, ...key] = pathname.split('/'); + return { + host: host, + bucket, + key: key.join('/'), + }; +}; + +export class GcsUrlReader implements UrlReader { + static factory: ReaderFactory = ({ config, logger }) => { + if (!config.has('integrations.gcs')) { + return []; + } + return config + .getConfigArray('integrations.gcs') + .filter(integration => { + if (!integration.has('clientEmail') || !integration.has('privateKey')) { + logger.warn( + "Skipping gcs integration, Missing required config value at 'integration.gcs.clientEmail' or 'integration.gcs.privateKey'", + ); + return false; + } + return true; + }) + .map(integration => { + const privKey = integration + .getOptionalString('privateKey') + ?.split('\\n') + .join('\n'); + + const storage = new Storage({ + credentials: { + client_email: integration.getOptionalString('clientEmail'), + private_key: privKey, + }, + }); + const reader = new GcsUrlReader(storage, logger); + const host = + integration.getOptionalString('host') || 'storage.cloud.google.com'; + + logger.info('Configuring integration, gcs'); + const predicate = (url: URL) => url.host === host; + return { reader, predicate }; + }); + }; + + constructor( + private readonly storage: Storage, + private readonly logger: Logger, + ) {} + + async read(url: string): Promise { + try { + const { bucket, key } = parseURL(url); + + this.logger.info('Reading GCS Location'); + return await getRawBody( + this.storage.bucket(bucket).file(key).createReadStream(), + ); + } catch (error) { + this.logger.warn(error.stack); + throw new Error(`unable to read gcs file from ${url}`); + } + } + + async readTree(): Promise { + throw new Error('GcsUrlReader does not implement readTree'); + } + + async search(): Promise { + throw new Error('GcsUrlReader does not implement readTree'); + } + + toString() { + return `gcs{host=storage.cloud.google.com,authed=true}}`; + } +} diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index 2f233463bb..785325faa7 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -24,6 +24,7 @@ import { GithubUrlReader } from './GithubUrlReader'; import { GitlabUrlReader } from './GitlabUrlReader'; import { ReadTreeResponseFactory } from './tree'; import { FetchUrlReader } from './FetchUrlReader'; +import { GcsUrlReader } from './GcsUrlReader'; type CreateOptions = { /** Root config object */ @@ -70,6 +71,7 @@ export class UrlReaders { BitbucketUrlReader.factory, GithubUrlReader.factory, GitlabUrlReader.factory, + GcsUrlReader.factory, FetchUrlReader.factory, ]), }); diff --git a/yarn.lock b/yarn.lock index b0b8c04313..d9dcd3f5b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2471,6 +2471,21 @@ retry-request "^4.1.1" teeny-request "^7.0.0" +"@google-cloud/common@^3.6.0": + version "3.6.0" + resolved "https://registry.npmjs.org/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b" + integrity sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q== + dependencies: + "@google-cloud/projectify" "^2.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.1" + duplexify "^4.1.1" + ent "^2.2.0" + extend "^3.0.2" + google-auth-library "^7.0.2" + retry-request "^4.1.1" + teeny-request "^7.0.0" + "@google-cloud/container@^2.2.0": version "2.2.0" resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.2.0.tgz#e97ae1cee9040b6af09cc8199ed0aa2d4ae6238e" @@ -2570,6 +2585,33 @@ stream-events "^1.0.1" xdg-basedir "^4.0.0" +"@google-cloud/storage@^5.8.0": + version "5.8.0" + resolved "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.8.0.tgz#1f580e276f1d453790b382156421d1bcc4bd3f4b" + integrity sha512-WOShvBPOfkDXUzXMO+3j8Bzus+PFI9r1Ey9dLG2Zf458/PVuFTtaRWntd9ZiDG8g90zl2LmnA1JkDCreGUKr5g== + dependencies: + "@google-cloud/common" "^3.6.0" + "@google-cloud/paginator" "^3.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.0" + async-retry "^1.3.1" + compressible "^2.0.12" + date-and-time "^0.14.2" + duplexify "^4.0.0" + extend "^3.0.2" + gaxios "^4.0.0" + gcs-resumable-upload "^3.1.3" + get-stream "^6.0.0" + hash-stream-validation "^0.2.2" + mime "^2.2.0" + mime-types "^2.0.8" + onetime "^5.1.0" + p-limit "^3.0.1" + pumpify "^2.0.0" + snakeize "^0.1.0" + stream-events "^1.0.1" + xdg-basedir "^4.0.0" + "@graphiql/toolkit@^0.1.0": version "0.1.1" resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.1.1.tgz#a7da3ba460ceae27bcdc8f03831ca4f88f90f3d7" @@ -11393,6 +11435,11 @@ date-and-time@^0.14.2: resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e" integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA== +date-and-time@^0.14.2: + version "0.14.2" + resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.2.tgz#a4266c3dead460f6c231fe9674e585908dac354e" + integrity sha512-EFTCh9zRSEpGPmJaexg7HTuzZHh6cnJj1ui7IGCFNXzd2QdpsNh05Db5TF3xzJm30YN+A8/6xHSuRcQqoc3kFA== + date-and-time@^0.6.3: version "0.6.3" resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.6.3.tgz#2daee52df67c28bd93bce862756ac86b68cf4237" @@ -13979,6 +14026,19 @@ gcs-resumable-upload@^3.1.3: pumpify "^2.0.0" stream-events "^1.0.4" +gcs-resumable-upload@^3.1.3: + version "3.1.3" + resolved "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.3.tgz#1e38e1339600b85812e6430a5ab455453c64cce3" + integrity sha512-LjVrv6YVH0XqBr/iBW0JgRA1ndxhK6zfEFFJR4im51QVTj/4sInOXimY2evDZuSZ75D3bHxTaQAdXRukMc1y+w== + dependencies: + abort-controller "^3.0.0" + configstore "^5.0.0" + extend "^3.0.2" + gaxios "^4.0.0" + google-auth-library "^7.0.0" + pumpify "^2.0.0" + stream-events "^1.0.4" + generic-names@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872"