From 2499f6cdefc52f7c781fc5c1ecbe7b8c3ec31be2 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 11:00:24 -0800 Subject: [PATCH 01/13] feat: add support for assuming role in plugins that use AWS Signed-off-by: Jonah Back --- .changeset/sixty-chicken-ring.md | 7 ++ .../src/stages/publish/awsS3.ts | 33 ++++++-- plugins/catalog-backend/config.d.ts | 13 ++++ plugins/catalog-backend/package-lock.json | 76 +++++++++++++++++++ ...sOrganizationCloudAccountProcessor.test.ts | 6 +- .../AwsOrganizationCloudAccountProcessor.ts | 41 +++++++++- .../processors/awsOrganization/config.test.ts | 37 +++++++++ .../processors/awsOrganization/config.ts | 44 +++++++++++ plugins/techdocs/config.d.ts | 6 ++ 9 files changed, 253 insertions(+), 10 deletions(-) create mode 100644 .changeset/sixty-chicken-ring.md create mode 100644 plugins/catalog-backend/package-lock.json create mode 100644 plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts diff --git a/.changeset/sixty-chicken-ring.md b/.changeset/sixty-chicken-ring.md new file mode 100644 index 0000000000..0fff26d6f9 --- /dev/null +++ b/.changeset/sixty-chicken-ring.md @@ -0,0 +1,7 @@ +--- +'@backstage/techdocs-common': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-techdocs': patch +--- + +Add support for assuming role in AWS integrations diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 8eaf03eb2f..b29d0dfb58 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -15,7 +15,7 @@ */ import path from 'path'; import express from 'express'; -import aws from 'aws-sdk'; +import aws, { Credentials } from 'aws-sdk'; import { ManagedUpload } from 'aws-sdk/clients/s3'; import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; @@ -26,6 +26,7 @@ import fs from 'fs-extra'; import { Readable } from 'stream'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; +import { CredentialsOptions } from 'aws-sdk/lib/credentials'; const streamToBuffer = (stream: Readable): Promise => { return new Promise((resolve, reject) => { @@ -61,9 +62,30 @@ export class AwsS3Publish implements PublisherBase { ); let accessKeyId = undefined; let secretAccessKey = undefined; + let awsCredentials: + | Credentials + | CredentialsOptions + | undefined = undefined; if (credentials) { - accessKeyId = credentials.getOptionalString('accessKeyId'); - secretAccessKey = credentials.getOptionalString('secretAccessKey'); + const roleArn = credentials.getOptionalString('roleArn'); + if (roleArn && aws.config.credentials instanceof Credentials) { + awsCredentials = new aws.ChainableTemporaryCredentials({ + masterCredentials: aws.config.credentials as Credentials, + params: { + RoleSessionName: 'backstage-aws-organization-processor', + RoleArn: roleArn, + }, + }); + } else { + accessKeyId = credentials.getOptionalString('accessKeyId'); + secretAccessKey = credentials.getOptionalString('secretAccessKey'); + if (accessKeyId && secretAccessKey) { + awsCredentials = { + accessKeyId, + secretAccessKey, + }; + } + } } // AWS Region is an optional config. If missing, default AWS env variable AWS_REGION @@ -76,10 +98,7 @@ export class AwsS3Publish implements PublisherBase { ...(credentials && accessKeyId && secretAccessKey && { - credentials: { - accessKeyId, - secretAccessKey, - }, + credentials: awsCredentials, }), ...(region && { region, diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 7877f34dcf..b85846af2a 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -334,6 +334,19 @@ export interface Config { }>; }; + /** + * AwsOrganizationCloudAccountProcessor configuration + */ + awsOrganization?: { + providers: Array<{ + /** + * The role to be assumed by this processor + * + */ + roleArn?: string; + }>; + }; + /** * MicrosoftGraphOrgReaderProcessor configuration */ diff --git a/plugins/catalog-backend/package-lock.json b/plugins/catalog-backend/package-lock.json new file mode 100644 index 0000000000..a0474f89fc --- /dev/null +++ b/plugins/catalog-backend/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "@backstage/plugin-catalog-backend", + "version": "0.5.1", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, + "create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true + }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "ts-node": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", + "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", + "dev": true, + "requires": { + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + }, + "dependencies": { + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true + } + } + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts index ba163fa0c3..596e62748f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -15,10 +15,14 @@ */ import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; +import * as winston from 'winston'; describe('AwsOrganizationCloudAccountProcessor', () => { describe('readLocation', () => { - const processor = new AwsOrganizationCloudAccountProcessor(); + const processor = new AwsOrganizationCloudAccountProcessor({ + providers: [], + logger: winston.createLogger(), + }); const location = { type: 'aws-cloud-accounts', target: '' }; const emit = jest.fn(); const listAccounts = jest.fn(); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts index 0e988e7307..83fd89801f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -14,11 +14,17 @@ * limitations under the License. */ import { LocationSpec, ResourceEntityV1alpha1 } from '@backstage/catalog-model'; -import AWS, { Organizations } from 'aws-sdk'; +import AWS, { Credentials, Organizations } from 'aws-sdk'; import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { Config } from '../../../../../packages/config/src'; +import { Logger } from 'winston'; +import { + AwsOrganizationProviderConfig, + readAwsOrganizationConfig, +} from './awsOrganization/config'; const AWS_ORGANIZATION_REGION = 'us-east-1'; const LOCATION_TYPE = 'aws-cloud-accounts'; @@ -33,9 +39,40 @@ const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id'; * If custom authentication is needed, it can be achieved by configuring the global AWS.credentials object. */ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { + logger: Logger; organizations: Organizations; - constructor() { + providers: AwsOrganizationProviderConfig[]; + + static fromConfig(config: Config, options: { logger: Logger }) { + const c = config.getOptionalConfig('catalog.processors.awsOrganization'); + return new AwsOrganizationCloudAccountProcessor({ + ...options, + providers: c ? readAwsOrganizationConfig(c) : [], + }); + } + + constructor(options: { + providers: AwsOrganizationProviderConfig[]; + logger: Logger; + }) { + this.providers = options.providers; + this.logger = options.logger; + let credentials = undefined; + if ( + this.providers.length > 0 && + this.providers[0].roleArn !== undefined && + AWS.config.credentials instanceof Credentials + ) { + credentials = new AWS.ChainableTemporaryCredentials({ + masterCredentials: AWS.config.credentials as Credentials, + params: { + RoleSessionName: 'backstage-aws-organization-processor', + RoleArn: this.providers[0].roleArn, + }, + }); + } this.organizations = new AWS.Organizations({ + credentials, region: AWS_ORGANIZATION_REGION, }); // Only available in us-east-1 } diff --git a/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts new file mode 100644 index 0000000000..571d19c829 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readAwsOrganizationConfig } from './config'; + +describe('readAwsOrganizationConfig', () => { + it('applies all of the defaults', () => { + const config = { + providers: [ + { + roleArn: 'aws::arn::foo', + }, + ], + }; + const actual = readAwsOrganizationConfig(new ConfigReader(config)); + const expected = [ + { + roleArn: 'aws::arn::foo', + }, + ]; + expect(actual).toEqual(expected); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts new file mode 100644 index 0000000000..120079d679 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +/** + * The configuration parameters for a single AWS Organization Processor + */ +export type AwsOrganizationProviderConfig = { + /** + * The role to assume for the processor. + */ + roleArn?: string; +}; + +export function readAwsOrganizationConfig( + config: Config, +): AwsOrganizationProviderConfig[] { + const providers: AwsOrganizationProviderConfig[] = []; + const providerConfigs = config.getOptionalConfigArray('providers') ?? []; + + for (const providerConfig of providerConfigs) { + const roleArn = providerConfig.getOptionalString('roleArn'); + + providers.push({ + roleArn, + }); + } + + return providers; +} diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index 1104d75237..d0c2ee2e6f 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -86,6 +86,12 @@ export interface Config { * @visibility secret */ secretAccessKey: string; + /** + * ARN of role to be assumed + * attr: 'roleArn' - accepts a string value + * @visibility secret + */ + roleArn: string; }; /** * (Required) Cloud Storage Bucket Name From 6237522a1f1f31f67d0be70182e7b6354267c627 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 11:45:23 -0800 Subject: [PATCH 02/13] remove accidental package-lock.json --- plugins/catalog-backend/package-lock.json | 76 ----------------------- 1 file changed, 76 deletions(-) delete mode 100644 plugins/catalog-backend/package-lock.json diff --git a/plugins/catalog-backend/package-lock.json b/plugins/catalog-backend/package-lock.json deleted file mode 100644 index a0474f89fc..0000000000 --- a/plugins/catalog-backend/package-lock.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name": "@backstage/plugin-catalog-backend", - "version": "0.5.1", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true - }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true - }, - "create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true - }, - "diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true - }, - "make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true - }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - }, - "source-map-support": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", - "dev": true, - "requires": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "ts-node": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", - "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", - "dev": true, - "requires": { - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "source-map-support": "^0.5.17", - "yn": "3.1.1" - }, - "dependencies": { - "yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true - } - } - } - } -} From 86c391d260bd2aa888d8c19d31c8df90707e9625 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 12:27:37 -0800 Subject: [PATCH 03/13] Make roleArn optional Co-authored-by: Himanshu Mishra --- plugins/techdocs/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index d0c2ee2e6f..da30578c99 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -91,7 +91,7 @@ export interface Config { * attr: 'roleArn' - accepts a string value * @visibility secret */ - roleArn: string; + roleArn?: string; }; /** * (Required) Cloud Storage Bucket Name From 58fdcb6a3033307c445540e49bbaa5ebf018b491 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 12:48:23 -0800 Subject: [PATCH 04/13] add docs around assuming role for techdocs storage --- docs/features/techdocs/using-cloud-storage.md | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index de2d505c87..084279b09b 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -149,7 +149,7 @@ If the environment variables - `AWS_REGION` are set and can be used to access the bucket you created in step 2, they will be -used by the AWS SDK v3 Node.js client for authentication. +used by the AWS SDK v2 Node.js client for authentication. [Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html) If the environment variables are missing, the AWS SDK tries to read the @@ -181,7 +181,7 @@ techdocs: ``` Refer to the -[official AWS documentation for obtaining the credentials](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/getting-your-credentials.html). +[official AWS documentation for obtaining the credentials](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/getting-your-credentials.html). Note: If you are using Amazon EC2 instance to deploy Backstage, you do not need to obtain the access keys separately. They can be made available in the @@ -189,6 +189,27 @@ environment automatically by defining appropriate IAM role with access to the bucket. Read more [here](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html#use-roles). +**3c. Authentication using an assumed role** Users with multiple AWS accounts +may want to use a role for S3 storage that is in a different AWS account. Using +the `roleArn` parameter as seen below, you can instruct the TechDocs publisher +to assume a role before accessing S3. + +```yaml +techdocs: + publisher: + type: 'awsS3' + awsS3: + bucketName: 'name-of-techdocs-storage-bucket' + region: + $env: AWS_REGION + credentials: + roleArn: arn:aws:iam::123456789012:role/my-backstage-role +``` + +Note: Assuming a role requires that primary credentials are already configured +at `AWS.config.credentials`. Read more about assuming roles +[here](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) + **4. That's it!** Your Backstage app is now ready to use AWS S3 for TechDocs, to store and read From 5cd564dc0d562e8b597af69bcb24676721469c84 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 12:49:48 -0800 Subject: [PATCH 05/13] cleanup conditionals on s3 client creation --- packages/techdocs-common/src/stages/publish/awsS3.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index b29d0dfb58..0a78af7135 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -95,11 +95,7 @@ export class AwsS3Publish implements PublisherBase { const region = config.getOptionalString('techdocs.publisher.awsS3.region'); const storageClient = new aws.S3({ - ...(credentials && - accessKeyId && - secretAccessKey && { - credentials: awsCredentials, - }), + credentials, ...(region && { region, }), From 0d8b3253a48914f24a00675d3556285ac6257938 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 14:23:37 -0800 Subject: [PATCH 06/13] Update docs/features/techdocs/using-cloud-storage.md Co-authored-by: Himanshu Mishra --- docs/features/techdocs/using-cloud-storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 084279b09b..a51a586813 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -150,7 +150,7 @@ If the environment variables are set and can be used to access the bucket you created in step 2, they will be used by the AWS SDK v2 Node.js client for authentication. -[Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html) +[Refer to the official documentation.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html) If the environment variables are missing, the AWS SDK tries to read the `~/.aws/credentials` file for credentials. From 9c80eca8d5e15582fad70a94fd6bd90d3338e054 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 14:24:49 -0800 Subject: [PATCH 07/13] Update docs/features/techdocs/using-cloud-storage.md Co-authored-by: Himanshu Mishra --- docs/features/techdocs/using-cloud-storage.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index a51a586813..3c648e5798 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -207,8 +207,7 @@ techdocs: ``` Note: Assuming a role requires that primary credentials are already configured -at `AWS.config.credentials`. Read more about assuming roles -[here](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) +at `AWS.config.credentials`. Read more about [assuming roles in AWS](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html). **4. That's it!** From 0e19926cf49421028a9efe8e5238e121ad6c0223 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 14:29:38 -0800 Subject: [PATCH 08/13] fix ts errors --- .../src/stages/publish/awsS3.ts | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 0a78af7135..28c545c420 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -57,19 +57,16 @@ export class AwsS3Publish implements PublisherBase { // or AWS shared credentials file at ~/.aws/credentials will be used to authenticate // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html - const credentials = config.getOptionalConfig( + const credentialsConfig = config.getOptionalConfig( 'techdocs.publisher.awsS3.credentials', ); let accessKeyId = undefined; let secretAccessKey = undefined; - let awsCredentials: - | Credentials - | CredentialsOptions - | undefined = undefined; - if (credentials) { - const roleArn = credentials.getOptionalString('roleArn'); + let credentials: Credentials | CredentialsOptions | undefined = undefined; + if (credentialsConfig) { + const roleArn = credentialsConfig.getOptionalString('roleArn'); if (roleArn && aws.config.credentials instanceof Credentials) { - awsCredentials = new aws.ChainableTemporaryCredentials({ + credentials = new aws.ChainableTemporaryCredentials({ masterCredentials: aws.config.credentials as Credentials, params: { RoleSessionName: 'backstage-aws-organization-processor', @@ -77,10 +74,12 @@ export class AwsS3Publish implements PublisherBase { }, }); } else { - accessKeyId = credentials.getOptionalString('accessKeyId'); - secretAccessKey = credentials.getOptionalString('secretAccessKey'); + accessKeyId = credentialsConfig.getOptionalString('accessKeyId'); + secretAccessKey = credentialsConfig.getOptionalString( + 'secretAccessKey', + ); if (accessKeyId && secretAccessKey) { - awsCredentials = { + credentials = { accessKeyId, secretAccessKey, }; From 05e169464c62446a44de6e81789d6f4e52678214 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 11 Feb 2021 15:32:52 -0800 Subject: [PATCH 09/13] run prettier fix --- docs/features/techdocs/using-cloud-storage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 3c648e5798..72db05ec1c 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -207,7 +207,8 @@ techdocs: ``` Note: Assuming a role requires that primary credentials are already configured -at `AWS.config.credentials`. Read more about [assuming roles in AWS](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html). +at `AWS.config.credentials`. Read more about +[assuming roles in AWS](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html). **4. That's it!** From 01bbdeef381ef2884bcf082acb022878041dbcd9 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Wed, 17 Feb 2021 12:35:47 -0800 Subject: [PATCH 10/13] address review comments, fix merge conflict --- packages/techdocs-common/src/stages/publish/awsS3.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 52d5647e71..a5857bd604 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -15,7 +15,7 @@ */ import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import aws from 'aws-sdk'; +import aws, { Credentials } from 'aws-sdk'; import { ManagedUpload } from 'aws-sdk/clients/s3'; import express from 'express'; import fs from 'fs-extra'; @@ -69,7 +69,7 @@ export class AwsS3Publish implements PublisherBase { credentials = new aws.ChainableTemporaryCredentials({ masterCredentials: aws.config.credentials as Credentials, params: { - RoleSessionName: 'backstage-aws-organization-processor', + RoleSessionName: 'backstage-aws-techdocs-s3-publisher', RoleArn: roleArn, }, }); From a5ff1a1e0fd318a9a32dc6f6828ad3c14006ecac Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 18 Feb 2021 15:34:54 -0800 Subject: [PATCH 11/13] Address review comments --- plugins/catalog-backend/config.d.ts | 4 ++-- .../AwsOrganizationCloudAccountProcessor.ts | 15 +++++++-------- .../processors/awsOrganization/config.ts | 18 ++++++------------ plugins/techdocs/config.d.ts | 3 +-- 4 files changed, 16 insertions(+), 24 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index b85846af2a..32ff44457c 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -338,13 +338,13 @@ export interface Config { * AwsOrganizationCloudAccountProcessor configuration */ awsOrganization?: { - providers: Array<{ + provider: { /** * The role to be assumed by this processor * */ roleArn?: string; - }>; + }; }; /** diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts index 83fd89801f..7964541840 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -19,7 +19,7 @@ import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { Config } from '../../../../../packages/config/src'; +import { Config } from '@backstage/config'; import { Logger } from 'winston'; import { AwsOrganizationProviderConfig, @@ -41,33 +41,32 @@ const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id'; export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { logger: Logger; organizations: Organizations; - providers: AwsOrganizationProviderConfig[]; + provider: AwsOrganizationProviderConfig; static fromConfig(config: Config, options: { logger: Logger }) { const c = config.getOptionalConfig('catalog.processors.awsOrganization'); return new AwsOrganizationCloudAccountProcessor({ ...options, - providers: c ? readAwsOrganizationConfig(c) : [], + provider: c ? readAwsOrganizationConfig(c) : {}, }); } constructor(options: { - providers: AwsOrganizationProviderConfig[]; + provider: AwsOrganizationProviderConfig; logger: Logger; }) { - this.providers = options.providers; + this.provider = options.provider; this.logger = options.logger; let credentials = undefined; if ( - this.providers.length > 0 && - this.providers[0].roleArn !== undefined && + this.provider.roleArn !== undefined && AWS.config.credentials instanceof Credentials ) { credentials = new AWS.ChainableTemporaryCredentials({ masterCredentials: AWS.config.credentials as Credentials, params: { RoleSessionName: 'backstage-aws-organization-processor', - RoleArn: this.providers[0].roleArn, + RoleArn: this.provider.roleArn, }, }); } diff --git a/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts index 120079d679..90206f504c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts +++ b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.ts @@ -28,17 +28,11 @@ export type AwsOrganizationProviderConfig = { export function readAwsOrganizationConfig( config: Config, -): AwsOrganizationProviderConfig[] { - const providers: AwsOrganizationProviderConfig[] = []; - const providerConfigs = config.getOptionalConfigArray('providers') ?? []; +): AwsOrganizationProviderConfig { + const providerConfig = config.getOptionalConfig('provider'); - for (const providerConfig of providerConfigs) { - const roleArn = providerConfig.getOptionalString('roleArn'); - - providers.push({ - roleArn, - }); - } - - return providers; + const roleArn = providerConfig?.getOptionalString('roleArn'); + return { + roleArn, + }; } diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index da30578c99..5285cc9071 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -88,8 +88,7 @@ export interface Config { secretAccessKey: string; /** * ARN of role to be assumed - * attr: 'roleArn' - accepts a string value - * @visibility secret + * @visibility backend */ roleArn?: string; }; From fd3214557cb04b0c64a6cb0071b5035781068923 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 18 Feb 2021 16:13:33 -0800 Subject: [PATCH 12/13] Fix ts error --- .../processors/AwsOrganizationCloudAccountProcessor.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts index 596e62748f..292564dc4f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -20,7 +20,7 @@ import * as winston from 'winston'; describe('AwsOrganizationCloudAccountProcessor', () => { describe('readLocation', () => { const processor = new AwsOrganizationCloudAccountProcessor({ - providers: [], + provider: {}, logger: winston.createLogger(), }); const location = { type: 'aws-cloud-accounts', target: '' }; From e0f141d36e44287c2e49c99f104e28af2e058688 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Thu, 18 Feb 2021 19:37:58 -0800 Subject: [PATCH 13/13] fix test after changing config schema --- .../processors/awsOrganization/config.test.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts index 571d19c829..09d95b479d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/awsOrganization/config.test.ts @@ -20,18 +20,14 @@ import { readAwsOrganizationConfig } from './config'; describe('readAwsOrganizationConfig', () => { it('applies all of the defaults', () => { const config = { - providers: [ - { - roleArn: 'aws::arn::foo', - }, - ], - }; - const actual = readAwsOrganizationConfig(new ConfigReader(config)); - const expected = [ - { + provider: { roleArn: 'aws::arn::foo', }, - ]; + }; + const actual = readAwsOrganizationConfig(new ConfigReader(config)); + const expected = { + roleArn: 'aws::arn::foo', + }; expect(actual).toEqual(expected); }); });