paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 18 Packages Projects Releases Wiki Activity

Merge pull request #16519 from JosiahCraw/feature/add-proxy-support-for-techdocs-awsS3

Browse Source 
feat: add support for HTTPS proxy for AWS S3 requests in Techdocs
This commit is contained in:
Morgan Bentell
2023-02-27 09:38:39 +01:00
committed by GitHub
parent a12106a262 8b36144ab0
commit 7e2fc35df2
9 changed files with 56 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/wicked-lions-repeat.md
+6
View File
@@ -0,0 +1,6 @@
---
'@techdocs/cli': minor
'@backstage/plugin-techdocs-node': minor
---
Added support for an HTTPS proxy for techdocs AWS S3 requests
docs/features/techdocs/cli.md
+22 -23
View File
@@ -176,29 +176,28 @@ Usage: techdocs-cli publish [options]
Publish generated TechDocs site to an external storage AWS S3, Google GCS, etc.
Options:
--publisher-type <TYPE> (Required always) awsS3 | googleGcs | azureBlobStorage
- same as techdocs.publisher.type in Backstage
app-config.yaml
--storage-name <BUCKET/CONTAINER NAME> (Required always) In case of AWS/GCS, use the bucket
name. In case of Azure, use container name. Same as
techdocs.publisher.[TYPE].bucketName
--entity <NAMESPACE/KIND/NAME> (Required always) Entity uid separated by / in
namespace/kind/name order (case-sensitive). Example:
default/Component/myEntity
--legacyUseCaseSensitiveTripletPaths Publishes objects with cased entity triplet prefix when set (e.g. namespace/Kind/name).
Only use if your TechDocs backend is configured the same way
--azureAccountName <AZURE ACCOUNT NAME> (Required for Azure) specify when --publisher-type
azureBlobStorage
--azureAccountKey <AZURE ACCOUNT KEY> Azure Storage Account key to use for authentication.
If not specified, you must set AZURE_TENANT_ID,
AZURE_CLIENT_ID & AZURE_CLIENT_SECRET as environment
variables.
--awsRoleArn <AWS ROLE ARN> Optional AWS ARN of role to be assumed.
--awsEndpoint <AWS ENDPOINT> Optional AWS endpoint to send requests to.
--awsS3ForcePathStyle Optional AWS S3 option to force path style.
--directory <PATH> Path of the directory containing generated files to
publish (default: "./site/")
-h, --help display help for command
--publisher-type <TYPE> (Required always) awsS3 | googleGcs | azureBlobStorage | openStackSwift - same as techdocs.publisher.type in Backstage app-config.yaml
--storage-name <BUCKET/CONTAINER NAME> (Required always) In case of AWS/GCS, use the bucket name. In case of Azure, use container name. Same as
techdocs.publisher.[TYPE].bucketName
--entity <NAMESPACE/KIND/NAME> (Required always) Entity uid separated by / in namespace/kind/name order (case-sensitive). Example: default/Component/myEntity
--legacyUseCaseSensitiveTripletPaths Publishes objects with cased entity triplet prefix when set (e.g. namespace/Kind/name). Only use if your TechDocs backend is configured
the same way. (default: false)
--azureAccountName <AZURE ACCOUNT NAME> (Required for Azure) specify when --publisher-type azureBlobStorage
--azureAccountKey <AZURE ACCOUNT KEY> Azure Storage Account key to use for authentication. If not specified, you must set AZURE_TENANT_ID, AZURE_CLIENT_ID &
AZURE_CLIENT_SECRET as environment variables.
--awsRoleArn <AWS ROLE ARN> Optional AWS ARN of role to be assumed.
--awsEndpoint <AWS ENDPOINT> Optional AWS endpoint to send requests to.
--awsProxy <HTTPS Proxy> Optional Proxy to use for AWS requests.
--awsS3sse <AWS SSE> Optional AWS S3 Server Side Encryption.
--awsS3ForcePathStyle Optional AWS S3 option to force path style.
--awsBucketRootPath <AWS BUCKET ROOT PATH> Optional sub-directory to store files in Amazon S3
--osCredentialId <OPENSTACK SWIFT APPLICATION CREDENTIAL ID> (Required for OpenStack) specify when --publisher-type openStackSwift
--osSecret <OPENSTACK SWIFT APPLICATION CREDENTIAL SECRET> (Required for OpenStack) specify when --publisher-type openStackSwift
--osAuthUrl <OPENSTACK SWIFT AUTHURL> (Required for OpenStack) specify when --publisher-type openStackSwift
--osSwiftUrl <OPENSTACK SWIFT SWIFTURL> (Required for OpenStack) specify when --publisher-type openStackSwift
--gcsBucketRootPath <GCS BUCKET ROOT PATH> Optional sub-directory to store files in Google cloud storage
--directory <PATH> Path of the directory containing generated files to publish (default: "./site/")
-h, --help display help for command
```
### Migrate content for case-insensitive access
docs/features/techdocs/configuration.md
+5
View File
@@ -136,6 +136,11 @@ techdocs:
# https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html#endpoint
endpoint: ${AWS_ENDPOINT}
# (Optional) HTTPS proxy to use for S3 Requests
# Defaults to using no proxy
# This allows docs to be published and read from behind a proxy
httpsProxy: ${HTTPS_PROXY}
# (Optional) Whether to use path style URLs when communicating with S3.
# Defaults to false.
# This allows providers like LocalStack, Minio and Wasabi (and possibly others) to be used to host tech docs.
packages/techdocs-cli/cli-report.md
+1
View File
@@ -77,6 +77,7 @@ Options:
--azureAccountKey <AZURE ACCOUNT KEY>
--awsRoleArn <AWS ROLE ARN>
--awsEndpoint <AWS ENDPOINT>
--awsProxy <HTTPS Proxy>
--awsS3sse <AWS SSE>
--awsS3ForcePathStyle
--awsBucketRootPath <AWS BUCKET ROOT PATH>
packages/techdocs-cli/src/commands/index.ts
+4
View File
@@ -173,6 +173,10 @@ export function registerCommands(program: Command) {
'--awsEndpoint <AWS ENDPOINT>',
'Optional AWS endpoint to send requests to.',
)
.option(
'--awsProxy <HTTPS Proxy>',
'Optional Proxy to use for AWS requests.',
)
.option('--awsS3sse <AWS SSE>', 'Optional AWS S3 Server Side Encryption.')
.option(
'--awsS3ForcePathStyle',
packages/techdocs-cli/src/lib/PublisherConfig.ts
+1
View File
@@ -94,6 +94,7 @@ export class PublisherConfig {
...(opts.awsEndpoint && { endpoint: opts.awsEndpoint }),
...(opts.awsS3ForcePathStyle && { s3ForcePathStyle: true }),
...(opts.awsS3sse && { sse: opts.awsS3sse }),
...(opts.awsProxy && { httpsProxy: opts.awsProxy }),
},
};
}
plugins/techdocs-node/package.json
+2
View File
@@ -42,6 +42,7 @@
"@aws-sdk/client-s3": "^3.208.0",
"@aws-sdk/credential-providers": "^3.208.0",
"@aws-sdk/lib-storage": "^3.208.0",
"@aws-sdk/node-http-handler": "^3.208.0",
"@aws-sdk/types": "^3.208.0",
"@azure/identity": "^2.1.0",
"@azure/storage-blob": "^12.5.0",
@@ -58,6 +59,7 @@
"express": "^4.17.1",
"fs-extra": "10.1.0",
"git-url-parse": "^13.0.0",
"hpagent": "^1.2.0",
"js-yaml": "^4.0.0",
"json5": "^2.1.3",
"mime-types": "^2.1.27",
plugins/techdocs-node/src/stages/publish/awsS3.ts
+12
View File
@@ -32,8 +32,10 @@ import {
S3Client,
} from '@aws-sdk/client-s3';
import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
import { NodeHttpHandler } from '@aws-sdk/node-http-handler';
import { Upload } from '@aws-sdk/lib-storage';
import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
import { HttpsProxyAgent } from 'hpagent';
import express from 'express';
import fs from 'fs-extra';
import JSON5 from 'json5';
@@ -150,6 +152,11 @@ export class AwsS3Publish implements PublisherBase {
'techdocs.publisher.awsS3.endpoint',
);
// AWS HTTPS proxy is an optional config. If missing, no proxy is used
const httpsProxy = config.getOptionalString(
'techdocs.publisher.awsS3.httpsProxy',
);
// AWS forcePathStyle is an optional config. If missing, it defaults to false. Needs to be enabled for cases
// where endpoint url points to locally hosted S3 compatible storage like Localstack
const forcePathStyle = config.getOptionalBoolean(
@@ -162,6 +169,11 @@ export class AwsS3Publish implements PublisherBase {
...(region && { region }),
...(endpoint && { endpoint }),
...(forcePathStyle && { forcePathStyle }),
...(httpsProxy && {
requestHandler: new NodeHttpHandler({
httpsAgent: new HttpsProxyAgent({ proxy: httpsProxy }),
}),
}),
});
const legacyPathCasing =
yarn.lock
+3 -1
View File
@@ -1211,7 +1211,7 @@ __metadata:
languageName: node
linkType: hard
"@aws-sdk/node-http-handler@npm:3.272.0":
"@aws-sdk/node-http-handler@npm:3.272.0, @aws-sdk/node-http-handler@npm:^3.208.0":
version: 3.272.0
resolution: "@aws-sdk/node-http-handler@npm:3.272.0"
dependencies:
@@ -8517,6 +8517,7 @@ __metadata:
"@aws-sdk/client-s3": ^3.208.0
"@aws-sdk/credential-providers": ^3.208.0
"@aws-sdk/lib-storage": ^3.208.0
"@aws-sdk/node-http-handler": ^3.208.0
"@aws-sdk/types": ^3.208.0
"@azure/identity": ^2.1.0
"@azure/storage-blob": ^12.5.0
@@ -8541,6 +8542,7 @@ __metadata:
express: ^4.17.1
fs-extra: 10.1.0
git-url-parse: ^13.0.0
hpagent: ^1.2.0
js-yaml: ^4.0.0
json5: ^2.1.3
mime-types: ^2.1.27