From 61299519fb853a65ab7af66506cb2ee86e564098 Mon Sep 17 00:00:00 2001
From: Joel Low <joel@joelsplace.sg>
Date: Fri, 26 Feb 2021 11:50:35 +0800
Subject: [PATCH] Replace S3 read-store-upload loop with file streams

This no longer buffers file contents in memory and limits the maximum
number of file descriptors open concurrently.

Signed-off-by: Joel Low <joel@joelsplace.sg>
---
 .changeset/young-pens-argue.md                |  5 +++++
 .../src/stages/publish/awsS3.test.ts          |  3 +++
 .../src/stages/publish/awsS3.ts               | 22 +++++++++----------
 3 files changed, 19 insertions(+), 11 deletions(-)
 create mode 100644 .changeset/young-pens-argue.md

diff --git a/.changeset/young-pens-argue.md b/.changeset/young-pens-argue.md
new file mode 100644
index 0000000000..909bf741fd
--- /dev/null
+++ b/.changeset/young-pens-argue.md
@@ -0,0 +1,5 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Remove read-store-upload loop when uploading S3 objects for TechDocs
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
index 3d7bd337d2..218d3dd88f 100644
--- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts
+++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
@@ -99,6 +99,9 @@ describe('AwsS3Publish', () => {
           assets: {
             'main.css': '',
           },
+          attachments: {
+            'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]),
+          },
         },
       });
     });
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts
index fc45a2d5cd..90b70ba747 100644
--- a/packages/techdocs-common/src/stages/publish/awsS3.ts
+++ b/packages/techdocs-common/src/stages/publish/awsS3.ts
@@ -167,18 +167,18 @@ export class AwsS3Publish implements PublisherBase {
         const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
         const destination = `${entityRootDir}/${relativeFilePathPosix}`; // S3 Bucket file relative path
 
-        const fileContent = await fs.readFile(filePath, 'utf8');
-
-        const params = {
-          Bucket: this.bucketName,
-          Key: destination,
-          Body: fileContent,
-        };
-
         // Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
-        const uploadFile = limiter(() =>
-          this.storageClient.upload(params).promise(),
-        );
+        const uploadFile = limiter(() => {
+          const fileStream = fs.createReadStream(filePath);
+
+          const params = {
+            Bucket: this.bucketName,
+            Key: destination,
+            Body: fileStream,
+          };
+
+          return this.storageClient.upload(params).promise();
+        });
         uploadPromises.push(uploadFile);
       }
       await Promise.all(uploadPromises);