diff --git a/.changeset/heavy-owls-float.md b/.changeset/heavy-owls-float.md new file mode 100644 index 0000000000..274876e723 --- /dev/null +++ b/.changeset/heavy-owls-float.md @@ -0,0 +1,12 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-techdocs': patch +--- + +Google Cloud authentication in TechDocs has been improved. + +1. `techdocs.publisher.googleGcs.credentials` is now optional. If it is missing, `GOOGLE_APPLICATION_CREDENTIALS` + environment variable (and some other methods) will be used to authenticate. + Read more here https://cloud.google.com/docs/authentication/production + +2. `techdocs.publisher.googleGcs.projectId` is no longer used. You can remove it from your `app-config.yaml`. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 7a0a4dbbbf..8c1a024f33 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -54,16 +54,15 @@ techdocs: # Required when techdocs.publisher.type is set to 'googleGcs'. Skip otherwise. googleGcs: - # An API key is required to write to a storage bucket. + # (Required) Cloud Storage Bucket Name + bucketName: 'techdocs-storage' + + # (Optional) An API key is required to write to a storage bucket. + # If missing, GOOGLE_APPLICATION_CREDENTIALS environment variable will be used. + # https://cloud.google.com/docs/authentication/production credentials: $file: '/path/to/google_application_credentials.json' - # Your GCP Project ID where the Cloud Storage Bucket is hosted. - projectId: 'gcp-project-id' - - # Cloud Storage Bucket Name - bucketName: 'techdocs-storage' - # Required when techdocs.publisher.type is set to 'awsS3'. Skip otherwise. awsS3: diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index c05346ca4a..474d9d64e6 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -30,20 +30,35 @@ techdocs: type: 'googleGcs' ``` -**2. GCP (Google Cloud Platform) Project** +**2. GCS Bucket** -Create or choose a dedicated GCP project. Set -`techdocs.publisher.googleGcs.projectId` to the project ID. +Create a dedicated Google Cloud Storage bucket for TechDocs sites. +techdocs-backend will publish documentation to this bucket. TechDocs will fetch +files from here to serve documentation in Backstage. Note that the bucket names +are globally unique. + +Set the config `techdocs.publisher.googleGcs.bucketName` in your +`app-config.yaml` to the name of the bucket you just created. ```yaml techdocs: publisher: type: 'googleGcs' - googleGcs: - projectId: 'gcp-project-id' + googleGcs: + bucketName: 'name-of-techdocs-storage-bucket' ``` -**3. Service account API key** +**3a. (Recommended) Authentication using environment variable** + +The GCS Node.js client will automatically use the environment variable +`GOOGLE_APPLICATION_CREDENTIALS` to authenticate with Google Cloud. It might +already be set in Compute Engine, Google Kubernetes Engine, etc. Read +https://cloud.google.com/docs/authentication/production for more details. + +**3b. Authentication using app-config.yaml** + +If you do not prefer (3a) and optionally like to use a service account, you can +follow these steps. Create a new Service Account and a key associated with it. In roles of the service account, use "Storage Admin". @@ -65,34 +80,29 @@ techdocs: publisher: type: 'googleGcs' googleGcs: - projectId: 'gcp-project-id' + bucketName: 'name-of-techdocs-storage-bucket' credentials: $file: '/path/to/google_application_credentials.json' ``` -**4. GCS Bucket** - -Create a dedicated bucket for TechDocs sites. techdocs-backend will publish -documentation to this bucket. TechDocs will fetch files from here to serve -documentation in Backstage. - -Set the name of the bucket to `techdocs.publisher.googleGcs.bucketName`. +Note: If you are finding it difficult to make the file +`google_application_credentials.json` available on a server, you could use the +file's content and set as an environment variable. And then use ```yaml techdocs: publisher: type: 'googleGcs' googleGcs: - projectId: 'gcp-project-id' - credentials: - $file: '/path/to/google_application_credentials.json' bucketName: 'name-of-techdocs-storage-bucket' + credentials: + $env: GOOGLE_APPLICATION_CREDENTIALS ``` -**5. That's it!** +**4. That's it!** Your Backstage app is now ready to use Google Cloud Storage for TechDocs, to -store the static generated documentation files. +store and read the static generated documentation files. ## Configuring AWS S3 Bucket with TechDocs @@ -113,9 +123,8 @@ techdocs: **2. AWS Policies** AWS Policies lets you **control access** to Amazon Web Services (AWS) products -and resources. -Here we will use a user policy **and** a bucket policy to show you the different -possibilities you have but you can use only one. +and resources. Here we will use a user policy **and** a bucket policy to show +you the different possibilities you have but you can use only one. AWS S3 @@ -135,9 +144,9 @@ and the **user** policy. **2.1.1 Create an Admin user** (if you don't have one yet) Create an **administrator user** account `ADMIN_USER` and grant it administrator -privileges by attaching a user policy giving the account **full access**. -Note down the Admin User credentials and IAM User Sign-In URL as you will need -to use this information in the next step. +privileges by attaching a user policy giving the account **full access**. Note +down the Admin User credentials and IAM User Sign-In URL as you will need to use +this information in the next step. **2.1.2 Create an AWS S3 Bucket** @@ -171,9 +180,9 @@ In the IAM console, do the following: **2.2 Attach policies** -Remember that you can use Bucket policy **or** User policy. -Just make sure that you grant all the permissions to the TechDocs user: -`3:PutObject`, `s3:GetObject`, `s3:ListBucket` and `s3:GetBucketLocation`. +Remember that you can use Bucket policy **or** User policy. Just make sure that +you grant all the permissions to the TechDocs user: `s3:PutObject`, +`s3:GetObject`, `s3:ListBucket` and `s3:GetBucketLocation`. **2.2.1 Create the bucket policy** @@ -209,9 +218,9 @@ section: - The first statement grants **TechDocs User** the bucket operation permissions `s3:GetBucketLocation` and `s3:ListBucket` which are permissions required by the console. -- The second statement grants the `s3:GetObject` permission. - (**NOTE :** if you do not use the user policy defined below you must also add - the `s3:PutObject` permission to allow the TechDocs user to add objects.) +- The second statement grants the `s3:GetObject` permission. (**NOTE :** if you + do not use the user policy defined below you must also add the `s3:PutObject` + permission to allow the TechDocs user to add objects.) **2.2.2 Create the user policy** diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index e95cee11d0..b84018c089 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -14,7 +14,6 @@ * limitations under the License. */ type storageOptions = { - projectId?: string; keyFilename?: string; }; @@ -39,11 +38,9 @@ class Bucket { } export class Storage { - private readonly projectId; private readonly keyFilename; constructor(options: storageOptions) { - this.projectId = options.projectId; this.keyFilename = options.keyFilename; } diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 43375b72d0..f8fb00647c 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -37,7 +37,7 @@ jest.spyOn(logger, 'info').mockReturnValue(logger); let publisher: PublisherBase; -beforeEach(() => { +beforeEach(async () => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -45,14 +45,13 @@ beforeEach(() => { type: 'googleGcs', googleGcs: { credentials: '{}', - projectId: 'gcp-project-id', bucketName: 'bucketName', }, }, }, }); - publisher = GoogleGCSPublish.fromConfig(mockConfig, logger); + publisher = await GoogleGCSPublish.fromConfig(mockConfig, logger); }); describe('GoogleGCSPublish', () => { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 12a92aeb53..e6b2c26943 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -27,57 +27,56 @@ import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; import { PublisherBase, PublishRequest } from './types'; export class GoogleGCSPublish implements PublisherBase { - static fromConfig(config: Config, logger: Logger): PublisherBase { - let credentials = ''; - let projectId = ''; + static async fromConfig( + config: Config, + logger: Logger, + ): Promise { let bucketName = ''; try { - credentials = config.getString( - 'techdocs.publisher.googleGcs.credentials', - ); - projectId = config.getString('techdocs.publisher.googleGcs.projectId'); bucketName = config.getString('techdocs.publisher.googleGcs.bucketName'); } catch (error) { throw new Error( "Since techdocs.publisher.type is set to 'googleGcs' in your app config, " + - 'credentials, projectId and bucketName are required in techdocs.publisher.googleGcs ' + - 'required to authenticate with Google Cloud Storage.', + 'techdocs.publisher.googleGcs.bucketName is required.', ); } + // Credentials is an optional config. If missing, default GCS environment variables will be used. + // Read more here https://cloud.google.com/docs/authentication/production + const credentials = config.getOptionalString( + 'techdocs.publisher.googleGcs.credentials', + ); let credentialsJson = {}; - try { - credentialsJson = JSON.parse(credentials); - } catch (err) { - throw new Error( - 'Error in parsing techdocs.publisher.googleGcs.credentials config to JSON.', - ); + if (credentials) { + try { + credentialsJson = JSON.parse(credentials); + } catch (err) { + throw new Error( + 'Error in parsing techdocs.publisher.googleGcs.credentials config to JSON.', + ); + } } const storageClient = new Storage({ - credentials: credentialsJson, - projectId: projectId, + ...(credentials && { + credentials: credentialsJson, + }), }); // Check if the defined bucket exists. Being able to connect means the configuration is good // and the storage client will work. - storageClient - .bucket(bucketName) - .getMetadata() - .then(() => { - logger.info( - `Successfully connected to the GCS bucket ${bucketName} in the GCP project ${projectId}.`, - ); - }) - .catch(reason => { - logger.error( - `Could not retrieve metadata about the GCS bucket ${bucketName} in the GCP project ${projectId}. ` + - 'Make sure the GCP project and the bucket exists and the access key located at the path ' + - "techdocs.publisher.googleGcs.credentials defined in app config has the role 'Storage Object Creator'. " + - 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', - ); - throw new Error(`from GCS client library: ${reason.message}`); - }); + try { + await storageClient.bucket(bucketName).getMetadata(); + logger.info(`Successfully connected to the GCS bucket ${bucketName}.`); + } catch (err) { + logger.error( + `Could not retrieve metadata about the GCS bucket ${bucketName}. ` + + 'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' + + 'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + throw new Error(err.message); + } return new GoogleGCSPublish(storageClient, bucketName, logger); } diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index d7cc257b89..89f2c0ffdd 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -69,7 +69,6 @@ describe('Publisher', () => { type: 'googleGcs', googleGcs: { credentials: '{}', - projectId: 'gcp-project-id', bucketName: 'bucketName', }, }, diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index caaf8a4cec..82232c2fd1 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -43,7 +43,7 @@ export class Publisher { switch (publisherType) { case 'googleGcs': logger.info('Creating Google Storage Bucket publisher for TechDocs'); - return GoogleGCSPublish.fromConfig(config, logger); + return await GoogleGCSPublish.fromConfig(config, logger); case 'awsS3': logger.info('Creating AWS S3 Bucket publisher for TechDocs'); return AwsS3Publish.fromConfig(config, logger); diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index ad2daeb9a3..fc05d79339 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -123,23 +123,19 @@ export interface Config { */ googleGcs?: { /** - * API key used to write to a storage bucket. - * attr: 'credentials' - accepts a string value - * @visibility secret - */ - credentials: string; - /** - * GCP Project ID where the Cloud Storage Bucket is hosted. - * attr: 'projectId' - accepts a string value - * @visibility secret - */ - projectId: string; - /** - * Cloud Storage Bucket Name + * (Required) Cloud Storage Bucket Name * attr: 'bucketName' - accepts a string value * @visibility secret */ bucketName: string; + /** + * (Optional) API key used to write to a storage bucket. + * If not set, environment variables will be used to authenticate. + * Read more: https://cloud.google.com/docs/authentication/production + * attr: 'credentials' - accepts a string value + * @visibility secret + */ + credentials?: string; }; }; };