Merge pull request #3953 from ayshiff/feature/techdocs-aws-sdkv3

TechDocs: AWS S3 SDK version bump

packages/techdocs-common/src/stages/publish/awsS3.test.ts

@@ -102,12 +102,6 @@ describe('AwsS3Publish', () => {
     });
 
     it('should fail to publish a directory', async () => {
-      const wrongPathToGeneratedDirectory = path.join(
-        'wrong',
-        'path',
-        'to',
-        'generatedDirectory',
-      );
       const entity = createMockEntity();
       const entityRootDir = getEntityRootDir(entity);
 
@@ -124,13 +118,11 @@ describe('AwsS3Publish', () => {
       await publisher
         .publish({
           entity,
-          directory: wrongPathToGeneratedDirectory,
+          directory: '/wrong/path/to/generatedDirectory',
         })
         .catch(error =>
-          expect(error).toEqual(
-            new Error(
-              `Unable to upload file(s) to AWS S3. Error Failed to read template directory: ENOENT, no such file or directory '${wrongPathToGeneratedDirectory}'`,
-            ),
+          expect(error.message).toContain(
+            'Unable to upload file(s) to AWS S3. Error Failed to read template directory',
           ),
         );
       mockFs.restore();
@@ -180,14 +172,17 @@ describe('AwsS3Publish', () => {
     it('should return an error if the techdocs_metadata.json file is not present', async () => {
       const entityNameMock = createMockEntityName();
       const entity = createMockEntity();
-      const entityRootDir = getEntityRootDir(entity);
+      const {
+        metadata: { name, namespace },
+        kind,
+      } = entity;
 
       await publisher
         .fetchTechDocsMetadata(entityNameMock)
         .catch(error =>
           expect(error).toEqual(
             new Error(
-              `TechDocs metadata fetch failed, The file ${entityRootDir}/techdocs_metadata.json doest not exist !`,
+              `TechDocs metadata fetch failed, The file ${namespace}/${kind}/${name}/techdocs_metadata.json doest not exist.`,
             ),
           ),
         );

packages/techdocs-common/src/stages/publish/awsS3.ts

@@ -15,14 +15,27 @@
  */
 import path from 'path';
 import express from 'express';
-import aws from 'aws-sdk';
+import { PutObjectCommandOutput, S3 } from '@aws-sdk/client-s3';
 import { Logger } from 'winston';
 import { Entity, EntityName } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers';
 import { PublisherBase, PublishRequest } from './types';
-import { ManagedUpload } from 'aws-sdk/clients/s3';
 import fs from 'fs-extra';
+import { Readable } from 'stream';
+
+const streamToString = (stream: Readable): Promise<string> => {
+  return new Promise((resolve, reject) => {
+    try {
+      const chunks: any[] = [];
+      stream.on('data', chunk => chunks.push(chunk));
+      stream.on('error', reject);
+      stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+    } catch (e) {
+      throw new Error(`Unable to parse the response data, ${e.message}`);
+    }
+  });
+};
 
 export class AwsS3Publish implements PublisherBase {
   static fromConfig(config: Config, logger: Logger): PublisherBase {
@@ -47,7 +60,7 @@ export class AwsS3Publish implements PublisherBase {
       );
     }
 
-    const storageClient = new aws.S3({
+    const storageClient = new S3({
       credentials: { accessKeyId, secretAccessKey },
       ...(region && { region }),
     });
@@ -79,7 +92,7 @@ export class AwsS3Publish implements PublisherBase {
   }
 
   constructor(
-    private readonly storageClient: aws.S3,
+    private readonly storageClient: S3,
     private readonly bucketName: string,
     private readonly logger: Logger,
   ) {
@@ -98,7 +111,7 @@ export class AwsS3Publish implements PublisherBase {
       // So collecting path of only the files is good enough.
       const allFilesToUpload = await getFileTreeRecursively(directory);
 
-      const uploadPromises: Array<Promise<ManagedUpload.SendData>> = [];
+      const uploadPromises: Array<Promise<PutObjectCommandOutput>> = [];
 
       for (const filePath of allFilesToUpload) {
         // Remove the absolute path prefix of the source directory
@@ -116,7 +129,7 @@ export class AwsS3Publish implements PublisherBase {
           Body: fileContent,
         };
 
-        uploadPromises.push(this.storageClient.upload(params).promise());
+        uploadPromises.push(this.storageClient.putObject(params));
       }
       await Promise.all(uploadPromises);
       this.logger.info(
@@ -135,25 +148,27 @@ export class AwsS3Publish implements PublisherBase {
       return await new Promise<string>((resolve, reject) => {
         const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
 
-        const fileStreamChunks: Array<any> = [];
         this.storageClient
           .getObject({
             Bucket: this.bucketName,
             Key: `${entityRootDir}/techdocs_metadata.json`,
           })
-          .createReadStream()
-          .on('error', err => {
+          .then(async file => {
+            const techdocsMetadataJson = await streamToString(
+              file.Body as Readable,
+            );
+
+            if (!techdocsMetadataJson) {
+              throw new Error(
+                `Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
+              );
+            }
+
+            resolve(techdocsMetadataJson);
+          })
+          .catch(err => {
             this.logger.error(err.message);
             reject(new Error(err.message));
-          })
-          .on('data', chunk => {
-            fileStreamChunks.push(chunk);
-          })
-          .on('end', () => {
-            const techdocsMetadataJson = Buffer.concat(
-              fileStreamChunks,
-            ).toString();
-            resolve(techdocsMetadataJson);
           });
       });
     } catch (e) {
@@ -174,19 +189,14 @@ export class AwsS3Publish implements PublisherBase {
       const fileExtension = path.extname(filePath);
       const responseHeaders = getHeadersForFileExtension(fileExtension);
 
-      const fileStreamChunks: Array<any> = [];
       this.storageClient
         .getObject({ Bucket: this.bucketName, Key: filePath })
-        .createReadStream()
-        .on('error', err => {
-          this.logger.warn(err.message);
-          res.status(404).send(err.message);
-        })
-        .on('data', chunk => {
-          fileStreamChunks.push(chunk);
-        })
-        .on('end', () => {
-          const fileContent = Buffer.concat(fileStreamChunks);
+        .then(async object => {
+          const fileContent = await streamToString(object.Body as Readable);
+          if (!fileContent) {
+            throw new Error(`Unable to parse the file ${filePath}.`);
+          }
+
           // Inject response headers
           for (const [headerKey, headerValue] of Object.entries(
             responseHeaders,
@@ -195,6 +205,10 @@ export class AwsS3Publish implements PublisherBase {
           }
 
           res.send(fileContent);
+        })
+        .catch(err => {
+          this.logger.warn(err.message);
+          res.status(404).send(err.message);
         });
     };
   }
@@ -206,12 +220,10 @@ export class AwsS3Publish implements PublisherBase {
   async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
     try {
       const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
-      await this.storageClient
-        .headObject({
-          Bucket: this.bucketName,
-          Key: `${entityRootDir}/index.html`,
-        })
-        .promise();
+      await this.storageClient.headObject({
+        Bucket: this.bucketName,
+        Key: `${entityRootDir}/index.html`,
+      });
       return Promise.resolve(true);
     } catch (e) {
       return Promise.resolve(false);