From 16b73697a3663adc5bb6a573bf3d7eff4cf179dc Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 12 Jan 2021 11:41:00 +0100 Subject: [PATCH 1/4] TechDocs/AWS: Enable authentication using env variables and ~/.aws/config shared file If authentication secrets are provided in app-config.yaml, it will be used. If not, environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION will be used. If not present, ~/.aws/config will be used to read the configs. https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html --- docs/features/techdocs/configuration.md | 17 ++++-- .../src/stages/publish/awsS3.ts | 54 ++++++++++++------- plugins/techdocs/config.d.ts | 15 ++++-- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 8c1a024f33..acd8f700e6 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -66,15 +66,22 @@ techdocs: # Required when techdocs.publisher.type is set to 'awsS3'. Skip otherwise. awsS3: - # An API key is required to write to a storage bucket. + # (Required) AWS S3 Bucket Name + bucketName: 'techdocs-storage' + + # (Optional) An API key is required to write to a storage bucket. + # If not set, environment variables or aws config file will be used to authenticate. + # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html credentials: accessKeyId: $env: TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL secretAccessKey: $env: TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL - region: - $env: AWSS3_REGION - # AWS S3 Bucket Name - bucketName: 'techdocs-storage' + # (Optional) AWS Region of the bucket. + # If not set, AWS_REGION environment variable or aws config file will be used. + # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html + region: + $env: AWS_REGION ``` diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index b7982a9ccd..4f273f0600 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -39,30 +39,47 @@ const streamToString = (stream: Readable): Promise => { export class AwsS3Publish implements PublisherBase { static fromConfig(config: Config, logger: Logger): PublisherBase { - let region = null; - let accessKeyId = null; - let secretAccessKey = null; let bucketName = ''; try { - accessKeyId = config.getString( - 'techdocs.publisher.awsS3.credentials.accessKeyId', - ); - secretAccessKey = config.getString( - 'techdocs.publisher.awsS3.credentials.secretAccessKey', - ); - region = config.getOptionalString('techdocs.publisher.awsS3.region'); bucketName = config.getString('techdocs.publisher.awsS3.bucketName'); } catch (error) { throw new Error( "Since techdocs.publisher.type is set to 'awsS3' in your app config, " + - 'credentials and bucketName are required in techdocs.publisher.awsS3 ' + - 'required to authenticate with AWS S3.', + 'techdocs.publisher.awsS3.bucketName is required.', ); } + // Credentials is an optional config. If missing, default AWS environment variables + // or AWS config file in ~/.aws/config will be used to authenticate + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html + const credentials = config.getOptionalConfig( + 'techdocs.publisher.awsS3.credentials', + ); + let accessKeyId = undefined; + let secretAccessKey = undefined; + if (credentials) { + accessKeyId = credentials.getOptionalString('accessKeyId'); + secretAccessKey = credentials.getOptionalString('secretAccessKey'); + } + + // AWS Region is an optional config. If missing, default AWS env variable AWS_REGION + // or AWS config file in ~/.aws/config will be used. But the AWS SDK v3 client needs + // to have the AWS Region information for it to work. + const region = config.getOptionalString('techdocs.publisher.awsS3.region'); + const storageClient = new S3({ - credentials: { accessKeyId, secretAccessKey }, - ...(region && { region }), + ...(credentials && + accessKeyId && + secretAccessKey && { + credentials: { + accessKeyId, + secretAccessKey, + }, + }), + ...(region && { + region, + }), }); // Check if the defined bucket exists. Being able to connect means the configuration is good @@ -75,11 +92,12 @@ export class AwsS3Publish implements PublisherBase { if (err) { logger.error( `Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` + - 'Make sure the AWS project and the bucket exists and the access key located at the path ' + - "techdocs.publisher.awsS3.credentials defined in app config has the role 'Storage Object Creator'. " + - 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + 'Make sure the bucket exists. Also make sure that authentication is setup either by ' + + 'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' + + 'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); - throw new Error(`from AWS client library: ${err.message}`); + logger.error(`from AWS client library: ${err.message}`); + throw new Error(); } else { logger.info( `Successfully connected to the AWS S3 bucket ${bucketName}.`, diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index fc05d79339..d484f2245f 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -78,10 +78,13 @@ export interface Config { */ awsS3?: { /** - * Credentials used to access a storage bucket + * (Optional) Credentials used to access a storage bucket. + * If not set, environment variables or aws config file will be used to authenticate. + * https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html + * https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html * @visibility secret */ - credentials: { + credentials?: { /** * User access key id * attr: 'accessKeyId' - accepts a string value @@ -98,11 +101,13 @@ export interface Config { /** * Cloud Storage Bucket Name * attr: 'bucketName' - accepts a string value - * @visibility secret + * @visibility backend */ bucketName: string; /** - * AWS Region + * (Optional) AWS Region. + * If not set, AWS_REGION environment variable or aws config file will be used. + * https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html * attr: 'region' - accepts a string value * @visibility secret */ @@ -125,7 +130,7 @@ export interface Config { /** * (Required) Cloud Storage Bucket Name * attr: 'bucketName' - accepts a string value - * @visibility secret + * @visibility backend */ bucketName: string; /** From f454b9f8659b46a666f7c5f8edc5c26cad2def59 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 12 Jan 2021 12:22:30 +0100 Subject: [PATCH 2/4] TechDocs/AWS: Update tutorial with auth best practices --- .github/styles/vocab.txt | 1 + docs/features/techdocs/using-cloud-storage.md | 175 +++++------------- .../src/stages/publish/awsS3.ts | 5 +- 3 files changed, 52 insertions(+), 129 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 29df243e23..cdce74f700 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -90,6 +90,7 @@ heroku Heroku horizontalpodautoscalers Hostname +html http https Iain diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 474d9d64e6..21206dede5 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -30,7 +30,7 @@ techdocs: type: 'googleGcs' ``` -**2. GCS Bucket** +**2. Create a GCS Bucket** Create a dedicated Google Cloud Storage bucket for TechDocs sites. techdocs-backend will publish documentation to this bucket. TechDocs will fetch @@ -106,10 +106,6 @@ store and read the static generated documentation files. ## Configuring AWS S3 Bucket with TechDocs -Follow the -[official AWS S3 documentation](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html) -for the latest instructions on the following steps involving AWS S3. - **1. Set `techdocs.publisher.type` config in your `app-config.yaml`** Set `techdocs.publisher.type` to `'awsS3'`. @@ -120,42 +116,17 @@ techdocs: type: 'awsS3' ``` -**2. AWS Policies** +**2. Create an S3 Bucket** -AWS Policies lets you **control access** to Amazon Web Services (AWS) products -and resources. Here we will use a user policy **and** a bucket policy to show -you the different possibilities you have but you can use only one. +Create a dedicated AWS S3 bucket for the storage of TechDocs sites. +[Refer to the official documentation](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html). -AWS S3 +TechDocs will publish documentation to this bucket and will fetch files from +here to serve documentation in Backstage. Note that the bucket names are +globally unique. -This is an example of how you can manage your policies: - -a. Admin user creates a **bucket policy** granting a set of permissions to our -TechDocs user. - -b. Admin user attaches a **user policy** to the TechDocs user granting -additional permissions. - -c. TechDocs User then tries permissions granted via both the **bucket** policy -and the **user** policy. - -**2.1 Creation** - -**2.1.1 Create an Admin user** (if you don't have one yet) - -Create an **administrator user** account `ADMIN_USER` and grant it administrator -privileges by attaching a user policy giving the account **full access**. Note -down the Admin User credentials and IAM User Sign-In URL as you will need to use -this information in the next step. - -**2.1.2 Create an AWS S3 Bucket** - -Using the credentials of your Admin User `ADMIN_USER`, and the special IAM user -sign-in URL, create a dedicated **bucket** for TechDocs sites. techdocs-backend -will publish documentation to this bucket. TechDocs will fetch files from here -to serve documentation in Backstage. - -Set the name of the bucket to `techdocs.publisher.awsS3.bucketName`. +Set the config `techdocs.publisher.awsS3.bucketName` in your `app-config.yaml` +to the name of the bucket you just created. ```yaml techdocs: @@ -165,112 +136,62 @@ techdocs: bucketName: 'name-of-techdocs-storage-bucket' ``` -**2.1.3 Create the `TechDocs` user** +**3a. (Recommended) Setup authentication the AWS way, using environment +variables** -This user will be used to interact with your bucket, it will only have -permissions to **get - put** objects. +You should follow the +[AWS security best practices guide for authentication](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html). -In the IAM console, do the following: +If the environment variables -- Create a new user, `TechDocs` -- Note down the TechDocs User credentials -- Note down the Amazon Resource Name (ARN) for the TechDocs user. In the IAM - console, select the TechDocs user, and you can find the user ARN in the - Summary tab. +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` +- `AWS_REGION` -**2.2 Attach policies** +are set and can be used to access the bucket you created in step 2, they will be +used by the AWS SDK v3 Node.js client for authentication. +[Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html) -Remember that you can use Bucket policy **or** User policy. Just make sure that -you grant all the permissions to the TechDocs user: `s3:PutObject`, -`s3:GetObject`, `s3:ListBucket` and `s3:GetBucketLocation`. +If the environment variables are missing, the AWS SDK tries to read the +`~/.aws/credentials` file for credentials. +[Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html) -**2.2.1 Create the bucket policy** +Note that the region of the bucket has to be set for the AWS SDK to work. +[See this](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html). -You now have to attach the following policy to your bucket in the Permission -section: +**3b. Authentication using app-config.yaml** -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "statement1", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::YOUR_ACCOUNT_ID:user/TechDocs" - }, - "Action": ["s3:GetBucketLocation", "s3:ListBucket"], - "Resource": ["arn:aws:s3:::name-of-techdocs-storage-bucket"] - }, - { - "Sid": "statement2", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::YOUR_ACCOUNT_ID:user/TechDocs" - }, - "Action": ["s3:GetObject"], - "Resource": ["arn:aws:s3:::name-of-techdocs-storage-bucket/*"] - } - ] -} -``` - -- The first statement grants **TechDocs User** the bucket operation permissions - `s3:GetBucketLocation` and `s3:ListBucket` which are permissions required by - the console. -- The second statement grants the `s3:GetObject` permission. (**NOTE :** if you - do not use the user policy defined below you must also add the `s3:PutObject` - permission to allow the TechDocs user to add objects.) - -**2.2.2 Create the user policy** - -Create an inline policy for the TechDocs user by using the following policy: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "PermissionForObjectOperations", - "Effect": "Allow", - "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::name-of-techdocs-storage-bucket/*"] - } - ] -} -``` - -See more details in the section -[Working with Inline Policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_manage.html). - -Now you need to fill in the environment variables with the `TechDocs` User -credentials. You can also specify a region if you want to accesses the resources -in a specific region. Otherwise no region will be selected by default. - -```properties -TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL="TECHDOCS_ACCESS_KEY_ID" -TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL="TECHDOCS_SECRET_ACCESS_KEY" -AWSS3_REGION="" // Optional -``` - -Make it available in your Backstage server and/or your local development server -and set it in the app config techdocs.publisher.awsS3. +AWS credentials and region can be provided to the AWS SDK via `app-config.yaml`. +If the configs below are present, they will be used over existing `AWS_*` +environment variables and the `~/.aws/credentials` config file. ```yaml techdocs: publisher: type: 'awsS3' awsS3: + bucketName: 'name-of-techdocs-storage-bucket' + region: + $env: AWS_REGION credentials: accessKeyId: - $env: TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL + $env: AWS_ACCESS_KEY_ID secretAccessKey: - $env: TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL - region: - $env: AWSS3_REGION + $env: AWS_SECRET_ACCESS_KEY ``` -**3. That's it!** +Refer to the +[official AWS documentation for obtaining the credentials](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/getting-your-credentials.html). -Your Backstage app is now ready to use AWS S3 for TechDocs, to store the static -generated documentation files. +Note: If you are using Amazon EC2 instance to deploy Backstage, you do not need +to obtain the access keys separately. They can be made available in the +environment automatically by defining appropriate IAM role with access to the +bucket. Read more +[here](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html#use-roles). + +**4. That's it!** + +Your Backstage app is now ready to use AWS S3 for TechDocs, to store and read +the static generated documentation files. When you start the backend of the app, +you should be able to see +`techdocs info Successfully connected to the AWS S3 bucket` in the logs. diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 4f273f0600..7a21ae6475 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -50,7 +50,7 @@ export class AwsS3Publish implements PublisherBase { } // Credentials is an optional config. If missing, default AWS environment variables - // or AWS config file in ~/.aws/config will be used to authenticate + // or AWS shared credentials file at ~/.aws/credentials will be used to authenticate // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html const credentials = config.getOptionalConfig( @@ -64,8 +64,9 @@ export class AwsS3Publish implements PublisherBase { } // AWS Region is an optional config. If missing, default AWS env variable AWS_REGION - // or AWS config file in ~/.aws/config will be used. But the AWS SDK v3 client needs + // or AWS shared credentials file at ~/.aws/credentials will be used. Any way, AWS SDK v3 client needs // to have the AWS Region information for it to work. + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html const region = config.getOptionalString('techdocs.publisher.awsS3.region'); const storageClient = new S3({ From b3b9445df15e84591a1e1d59d47dcd5a72ad40ee Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 12 Jan 2021 12:35:33 +0100 Subject: [PATCH 3/4] TechDocs/AWS: Add changeset about new config management for auth --- .changeset/stale-cougars-wink.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/stale-cougars-wink.md diff --git a/.changeset/stale-cougars-wink.md b/.changeset/stale-cougars-wink.md new file mode 100644 index 0000000000..306d92ef8f --- /dev/null +++ b/.changeset/stale-cougars-wink.md @@ -0,0 +1,10 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-techdocs': patch +--- + +AWS S3 authentication in TechDocs has been improved. + +1. `techdocs.publisher.awsS3.bucketName` is now the only required config. `techdocs.publisher.awsS3.credentials` and `techdocs.publisher.awsS3.region` are optional. + +2. If `techdocs.publisher.awsS3.credentials` and `techdocs.publisher.awsS3.region` are missing, the AWS environment variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION` will be used. There are more better ways of setting up AWS authentication. Read the guide at https://backstage.io/docs/features/techdocs/using-cloud-storage From fda570687e1f1c170aa79e006051cef8082e2aaa Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 12 Jan 2021 14:43:52 +0100 Subject: [PATCH 4/4] Update plugins/techdocs/config.d.ts Co-authored-by: Emma Indal --- plugins/techdocs/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index d484f2245f..f9831f27cb 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -99,7 +99,7 @@ export interface Config { secretAccessKey: string; }; /** - * Cloud Storage Bucket Name + * (Required) Cloud Storage Bucket Name * attr: 'bucketName' - accepts a string value * @visibility backend */