paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Use integration-aws-node for credentials in S3 Techdocs

Browse Source 
Signed-off-by: Clare Liguori <liguori@amazon.com>
This commit is contained in:
Clare Liguori
2022-11-28 13:23:26 -08:00
parent 13278732f6
commit e40790d0c2
9 changed files with 198 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/silly-knives-warn.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-techdocs-node': patch
---
Add support for specifying an S3 bucket's account ID and retrieving the credentials from the `aws` app config section. This is now the preferred way to configure AWS credentials for Techdocs.
docs/features/techdocs/configuration.md
+14 -2
View File
@@ -106,8 +106,20 @@ techdocs:
# If not set, the default location will be the root of the storage bucket
bucketRootPath: '/'
# (Optional) An API key is required to write to a storage bucket.
# If not set, environment variables or aws config file will be used to authenticate.
# (Optional) The AWS account ID where the storage bucket is located.
# Credentials for the account ID must be configured in the 'aws' app config section.
# See the integration-aws-node package for details on how to configure credentials in
# the 'aws' app config section.
# https://www.npmjs.com/package/@backstage/integration-aws-node
# If account ID is not set and no credentials are set, environment variables or aws config file will be used to authenticate.
# https://www.npmjs.com/package/@aws-sdk/credential-provider-node
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html
accountId: ${TECHDOCS_AWSS3_ACCOUNT_ID}
# (Optional) AWS credentials to use to write to the storage bucket.
# This configuration section is now deprecated.
# Configuring the account ID is now preferred, with credentials in the 'aws' app config section.
# If credentials are not set and no account ID is set, environment variables or aws config file will be used to authenticate.
# https://www.npmjs.com/package/@aws-sdk/credential-provider-node
# https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-credentials-node.html
credentials:
docs/features/techdocs/using-cloud-storage.md
+6 -3
View File
@@ -244,10 +244,13 @@ techdocs:
type: 'awsS3'
awsS3:
bucketName: 'name-of-techdocs-storage-bucket'
accountId: '123456789012'
region: ${AWS_REGION}
credentials:
accessKeyId: ${AWS_ACCESS_KEY_ID}
secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
aws:
accounts:
- accountId: '123456789012'
accessKeyId: ${AWS_ACCESS_KEY_ID}
secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
```
Refer to the
plugins/techdocs-backend/config.d.ts
Vendored
+15 -1
View File
@@ -81,9 +81,23 @@ export interface Config {
* Required when 'type' is set to awsS3
*/
awsS3?: {
/**
* (Optional) The AWS account ID where the storage bucket is located.
* Credentials for the account ID will be sourced from the 'aws' app config section.
* See the
* [integration-aws-node package](https://github.com/backstage/backstage/blob/master/packages/integration-aws-node/README.md)
* for details on how to configure the credentials in the app config.
* If account ID is not set and no credentials are set, environment variables or aws config file will be used to authenticate.
* @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
* @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
* @visibility secret
*/
accountId?: string;
/**
* (Optional) Credentials used to access a storage bucket.
* If not set, environment variables or aws config file will be used to authenticate.
* This section is now deprecated. Configuring the account ID is now preferred, with credentials in the 'aws'
* app config section.
* If not set and no account ID is set, environment variables or aws config file will be used to authenticate.
* @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
* @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
* @visibility secret
plugins/techdocs-node/package.json
+1
View File
@@ -50,6 +50,7 @@
"@backstage/config": "workspace:^",
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/integration-aws-node": "workspace:^",
"@backstage/plugin-search-common": "workspace:^",
"@google-cloud/storage": "^6.0.0",
"@trendyol-js/openstack-swift-sdk": "^0.0.5",
plugins/techdocs-node/src/stages/publish/awsS3.test.ts
+115 -32
View File
@@ -27,6 +27,11 @@ import {
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import {
AwsCredentials,
AwsCredentialsProviderOptions,
DefaultAwsCredentialsProvider,
} from '@backstage/integration-aws-node';
import { mockClient, AwsClientStub } from 'aws-sdk-client-mock';
import express from 'express';
import request from 'supertest';
@@ -40,6 +45,21 @@ import { Readable } from 'stream';
const env = process.env;
let s3Mock: AwsClientStub<S3Client>;
function getMockCredentials(): Promise<AwsCredentials> {
return Promise.resolve({
provider: async () => {
return Promise.resolve({
accessKeyId: 'MY_ACCESS_KEY_ID',
secretAccessKey: 'MY_SECRET_ACCESS_KEY',
});
},
});
}
const credsProviderMock = jest.spyOn(
DefaultAwsCredentialsProvider.prototype,
'getCredentials',
);
const getEntityRootDir = (entity: Entity) => {
const {
kind,
@@ -70,7 +90,7 @@ const logger = getVoidLogger();
const loggerInfoSpy = jest.spyOn(logger, 'info');
const loggerErrorSpy = jest.spyOn(logger, 'error');
const createPublisherFromConfig = ({
const createPublisherFromConfig = async ({
bucketName = 'bucketName',
bucketRootPath = '/',
legacyUseCaseSensitiveTripletPaths = false,
@@ -86,10 +106,7 @@ const createPublisherFromConfig = ({
publisher: {
type: 'awsS3',
awsS3: {
credentials: {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
accountId: '111111111111',
bucketName,
bucketRootPath,
sse,
@@ -97,9 +114,18 @@ const createPublisherFromConfig = ({
},
legacyUseCaseSensitiveTripletPaths,
},
aws: {
accounts: [
{
accountId: '111111111111',
accessKeyId: 'my-access-key',
secretAccessKey: 'my-secret-access-key',
},
],
},
});
return AwsS3Publish.fromConfig(mockConfig, logger);
return await AwsS3Publish.fromConfig(mockConfig, logger);
};
describe('AwsS3Publish', () => {
@@ -151,6 +177,11 @@ describe('AwsS3Publish', () => {
process.env = { ...env };
process.env.AWS_REGION = 'us-west-2';
jest.resetAllMocks();
credsProviderMock.mockImplementation((_?: AwsCredentialsProviderOptions) =>
getMockCredentials(),
);
mockFs({
[directory]: files,
});
@@ -215,16 +246,64 @@ describe('AwsS3Publish', () => {
process.env = env;
});
describe('buildCredentials', () => {
it('should retrieve credentials for a specific account ID', async () => {
await createPublisherFromConfig();
expect(credsProviderMock).toHaveBeenCalledWith({
accountId: '111111111111',
});
expect(credsProviderMock).toHaveBeenCalledTimes(1);
});
it('should retrieve default credentials when no config is present', async () => {
const mockConfig = new ConfigReader({
techdocs: {
publisher: {
type: 'awsS3',
awsS3: {
bucketName: 'bucketName',
},
},
},
});
await AwsS3Publish.fromConfig(mockConfig, logger);
expect(credsProviderMock).toHaveBeenCalledWith();
expect(credsProviderMock).toHaveBeenCalledTimes(1);
});
it('should fall back to deprecated method of retrieving credentials', async () => {
const mockConfig = new ConfigReader({
techdocs: {
publisher: {
type: 'awsS3',
awsS3: {
credentials: {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
bucketName: 'bucketName',
bucketRootPath: '/',
},
},
},
});
await AwsS3Publish.fromConfig(mockConfig, logger);
expect(credsProviderMock).toHaveBeenCalledTimes(0);
});
});
describe('getReadiness', () => {
it('should validate correct config', async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
expect(await publisher.getReadiness()).toEqual({
isAvailable: true,
});
});
it('should reject incorrect config', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketName: 'errorBucket',
});
expect(await publisher.getReadiness()).toEqual({
@@ -235,7 +314,7 @@ describe('AwsS3Publish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'default/component/backstage/404.html',
@@ -246,7 +325,7 @@ describe('AwsS3Publish', () => {
});
it('should publish a directory as well when legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toMatchObject({
@@ -259,7 +338,7 @@ describe('AwsS3Publish', () => {
});
it('should publish a directory when root path is specified', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
});
expect(await publisher.publish({ entity, directory })).toMatchObject({
@@ -272,7 +351,7 @@ describe('AwsS3Publish', () => {
});
it('should publish a directory when root path is specified and legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
@@ -286,7 +365,7 @@ describe('AwsS3Publish', () => {
});
it('should publish a directory when sse is specified', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
sse: 'aws:kms',
});
expect(await publisher.publish({ entity, directory })).toMatchObject({
@@ -307,7 +386,7 @@ describe('AwsS3Publish', () => {
'generatedDirectory',
);
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
const fails = publisher.publish({
entity,
@@ -327,7 +406,9 @@ describe('AwsS3Publish', () => {
it('should delete stale files after upload', async () => {
const bucketName = 'delete_stale_files_success';
const publisher = createPublisherFromConfig({ bucketName: bucketName });
const publisher = await createPublisherFromConfig({
bucketName: bucketName,
});
await publisher.publish({ entity, directory });
expect(loggerInfoSpy).toHaveBeenLastCalledWith(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`,
@@ -336,7 +417,9 @@ describe('AwsS3Publish', () => {
it('should log error when the stale files deletion fails', async () => {
const bucketName = 'delete_stale_files_error';
const publisher = createPublisherFromConfig({ bucketName: bucketName });
const publisher = await createPublisherFromConfig({
bucketName: bucketName,
});
await publisher.publish({ entity, directory });
expect(loggerErrorSpy).toHaveBeenLastCalledWith(
'Unable to delete file(s) from AWS S3. Error: Message',
@@ -346,13 +429,13 @@ describe('AwsS3Publish', () => {
describe('hasDocsBeenGenerated', () => {
it('should return true if docs has been generated', async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
await publisher.publish({ entity, directory });
expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
});
it('should return true if docs has been generated even if the legacy case is enabled', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
await publisher.publish({ entity, directory });
@@ -360,7 +443,7 @@ describe('AwsS3Publish', () => {
});
it('should return true if docs has been generated if root path is specified', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
});
await publisher.publish({ entity, directory });
@@ -368,7 +451,7 @@ describe('AwsS3Publish', () => {
});
it('should return true if docs has been generated if root path is specified and legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
@@ -377,7 +460,7 @@ describe('AwsS3Publish', () => {
});
it('should return false if docs has not been generated', async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
expect(
await publisher.hasDocsBeenGenerated({
kind: 'entity',
@@ -392,7 +475,7 @@ describe('AwsS3Publish', () => {
describe('fetchTechDocsMetadata', () => {
it('should return tech docs metadata', async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
await publisher.publish({ entity, directory });
expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
techdocsMetadata,
@@ -400,7 +483,7 @@ describe('AwsS3Publish', () => {
});
it('should return tech docs metadata even if the legacy case is enabled', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
await publisher.publish({ entity, directory });
@@ -410,7 +493,7 @@ describe('AwsS3Publish', () => {
});
it('should return tech docs metadata even if root path is specified', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
});
await publisher.publish({ entity, directory });
@@ -420,7 +503,7 @@ describe('AwsS3Publish', () => {
});
it('should return tech docs metadata if root path is specified and legacy casing is used', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
@@ -442,7 +525,7 @@ describe('AwsS3Publish', () => {
techdocsMetadataContent.replace(/"/g, "'"),
);
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
await publisher.publish({ entity, directory });
expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual(
@@ -453,7 +536,7 @@ describe('AwsS3Publish', () => {
});
it('should return an error if the techdocs_metadata.json file is not present', async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
const invalidEntityName = {
namespace: 'invalid',
@@ -477,7 +560,7 @@ describe('AwsS3Publish', () => {
};
});
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
const invalidEntityName = {
namespace: 'invalid',
@@ -501,7 +584,7 @@ describe('AwsS3Publish', () => {
let app: express.Express;
beforeEach(async () => {
const publisher = createPublisherFromConfig();
const publisher = await createPublisherFromConfig();
await publisher.publish({ entity, directory });
app = express().use(publisher.docsRouter());
});
@@ -521,7 +604,7 @@ describe('AwsS3Publish', () => {
});
it('should pass expected object path to bucket even if the legacy case is enabled', async () => {
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
await publisher.publish({ entity, directory });
@@ -541,7 +624,7 @@ describe('AwsS3Publish', () => {
it('should pass expected object path to bucket if root path is specified', async () => {
const rootPath = 'backstage-data/techdocs';
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: rootPath,
});
await publisher.publish({ entity, directory });
@@ -561,7 +644,7 @@ describe('AwsS3Publish', () => {
it('should pass expected object path to bucket if root path is specified and legacy case is enabled', async () => {
const rootPath = 'backstage-data/techdocs';
const publisher = createPublisherFromConfig({
const publisher = await createPublisherFromConfig({
bucketRootPath: rootPath,
legacyUseCaseSensitiveTripletPaths: true,
});
plugins/techdocs-node/src/stages/publish/awsS3.ts
+39 -21
View File
@@ -16,6 +16,10 @@
import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { assertError, ForwardedError } from '@backstage/errors';
import {
AwsCredentialsProvider,
DefaultAwsCredentialsProvider,
} from '@backstage/integration-aws-node';
import {
GetObjectCommand,
CopyObjectCommand,
@@ -27,12 +31,9 @@ import {
ListObjectsV2Command,
S3Client,
} from '@aws-sdk/client-s3';
import {
fromNodeProviderChain,
fromTemporaryCredentials,
} from '@aws-sdk/credential-providers';
import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
import { Upload } from '@aws-sdk/lib-storage';
import { CredentialProvider } from '@aws-sdk/types';
import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
import express from 'express';
import fs from 'fs-extra';
import JSON5 from 'json5';
@@ -97,7 +98,10 @@ export class AwsS3Publish implements PublisherBase {
this.sse = options.sse;
}
static fromConfig(config: Config, logger: Logger): PublisherBase {
static async fromConfig(
config: Config,
logger: Logger,
): Promise<PublisherBase> {
let bucketName = '';
try {
bucketName = config.getString('techdocs.publisher.awsS3.bucketName');
@@ -121,17 +125,21 @@ export class AwsS3Publish implements PublisherBase {
// or AWS shared credentials file at ~/.aws/credentials will be used.
const region = config.getOptionalString('techdocs.publisher.awsS3.region');
// Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used.
// 1. AWS environment variables
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
// 2. AWS shared credentials file at ~/.aws/credentials
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html
// 3. IAM Roles for EC2
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-iam.html
// Credentials can optionally be configured by specifying the AWS account ID, which will retrieve credentials
// for the account from the 'aws' section of the app config.
// Credentials can also optionally be directly configured in the techdocs awsS3 config, but this method is
// deprecated.
// If no credentials are configured, the AWS SDK V3's default credential chain will be used.
const accountId = config.getOptionalString(
'techdocs.publisher.awsS3.accountId',
);
const credentialsConfig = config.getOptionalConfig(
'techdocs.publisher.awsS3.credentials',
);
const credentials = AwsS3Publish.buildCredentials(
const credsProvider = DefaultAwsCredentialsProvider.fromConfig(config);
const credentials = await AwsS3Publish.buildCredentials(
credsProvider,
accountId,
credentialsConfig,
region,
);
@@ -174,7 +182,7 @@ export class AwsS3Publish implements PublisherBase {
private static buildStaticCredentials(
accessKeyId: string,
secretAccessKey: string,
): CredentialProvider {
): AwsCredentialIdentityProvider {
return async () => {
return Promise.resolve({
accessKeyId,
@@ -183,20 +191,30 @@ export class AwsS3Publish implements PublisherBase {
};
}
private static buildCredentials(
private static async buildCredentials(
credsProvider: AwsCredentialsProvider,
accountId?: string,
config?: Config,
region?: string,
): CredentialProvider {
if (!config) {
return fromNodeProviderChain();
): Promise<AwsCredentialIdentityProvider> {
// Pull credentials for the specified account ID from the 'aws' config section
if (accountId) {
return (await credsProvider.getCredentials({ accountId })).provider;
}
// Fall back to the default credential chain if neither account ID
// nor explicit credentials are provided
if (!config) {
return (await credsProvider.getCredentials()).provider;
}
// Pull credentials from the techdocs config section (deprecated)
const accessKeyId = config.getOptionalString('accessKeyId');
const secretAccessKey = config.getOptionalString('secretAccessKey');
const explicitCredentials: CredentialProvider =
const explicitCredentials: AwsCredentialIdentityProvider =
accessKeyId && secretAccessKey
? AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey)
: fromNodeProviderChain();
: (await credsProvider.getCredentials()).provider;
const roleArn = config.getOptionalString('roleArn');
if (roleArn) {
plugins/techdocs-node/src/stages/publish/publish.ts
+1 -1
View File
@@ -47,7 +47,7 @@ export class Publisher {
return GoogleGCSPublish.fromConfig(config, logger);
case 'awsS3':
logger.info('Creating AWS S3 Bucket publisher for TechDocs');
return AwsS3Publish.fromConfig(config, logger);
return await AwsS3Publish.fromConfig(config, logger);
case 'azureBlobStorage':
logger.info(
'Creating Azure Blob Storage Container publisher for TechDocs',
yarn.lock
+2 -1
View File
@@ -4135,7 +4135,7 @@ __metadata:
languageName: unknown
linkType: soft
"@backstage/integration-aws-node@workspace:packages/integration-aws-node":
"@backstage/integration-aws-node@workspace:^, @backstage/integration-aws-node@workspace:packages/integration-aws-node":
version: 0.0.0-use.local
resolution: "@backstage/integration-aws-node@workspace:packages/integration-aws-node"
dependencies:
@@ -8142,6 +8142,7 @@ __metadata:
"@backstage/config": "workspace:^"
"@backstage/errors": "workspace:^"
"@backstage/integration": "workspace:^"
"@backstage/integration-aws-node": "workspace:^"
"@backstage/plugin-search-common": "workspace:^"
"@google-cloud/storage": ^6.0.0
"@trendyol-js/openstack-swift-sdk": ^0.0.5