TechDocs: upgrade to AWS SDK v3

Signed-off-by: Clare Liguori <liguori@amazon.com>
2022-11-15 10:27:51 -08:00
parent 3228a8a951
commit 37931c33ce
9 changed files with 1427 additions and 612 deletions
@@ -39,6 +39,10 @@
    "url": "https://github.com/backstage/backstage/issues"
  },
  "dependencies": {
+    "@aws-sdk/client-s3": "^3.208.0",
+    "@aws-sdk/credential-providers": "^3.208.0",
+    "@aws-sdk/lib-storage": "^3.208.0",
+    "@aws-sdk/types": "^3.208.0",
    "@azure/identity": "^2.0.1",
    "@azure/storage-blob": "^12.5.0",
    "@backstage/backend-common": "workspace:^",
@@ -50,7 +54,6 @@
    "@google-cloud/storage": "^6.0.0",
    "@trendyol-js/openstack-swift-sdk": "^0.0.5",
    "@types/express": "^4.17.6",
-    "aws-sdk": "^2.840.0",
    "express": "^4.17.1",
    "fs-extra": "10.1.0",
    "git-url-parse": "^13.0.0",
@@ -70,6 +73,7 @@
    "@types/mock-fs": "^4.13.0",
    "@types/recursive-readdir": "^2.2.0",
    "@types/supertest": "^2.0.8",
+    "aws-sdk-client-mock": "^2.0.0",
    "supertest": "^6.1.3"
  }
 }
@@ -14,119 +14,31 @@
 * limitations under the License.
 */

+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  HeadBucketCommand,
+  HeadObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+  UploadPartCommand
+} from '@aws-sdk/client-s3';
 import { getVoidLogger } from '@backstage/backend-common';
 import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
+import { mockClient, AwsClientStub } from 'aws-sdk-client-mock';
 import express from 'express';
 import request from 'supertest';
 import mockFs from 'mock-fs';
 import path from 'path';
-import fs, { ReadStream } from 'fs-extra';
-import { EventEmitter } from 'events';
+import fs from 'fs-extra';
 import { AwsS3Publish } from './awsS3';
 import { storageRootDir } from '../../testUtils/StorageFilesMock';
+import { Readable } from 'stream';

-jest.mock('aws-sdk', () => {
-  const { StorageFilesMock } = require('../../testUtils/StorageFilesMock');
-  const storage = new StorageFilesMock();
-
-  return {
-    __esModule: true,
-    Credentials: jest.requireActual('aws-sdk').Credentials,
-    default: {
-      S3: class {
-        constructor() {
-          storage.emptyFiles();
-        }
-
-        headObject({ Key }: { Key: string }) {
-          return {
-            promise: async () => {
-              if (!storage.fileExists(Key)) {
-                throw new Error('File does not exist');
-              }
-            },
-          };
-        }
-
-        getObject({ Key }: { Key: string }) {
-          return {
-            promise: async () => storage.fileExists(Key),
-            createReadStream: () => {
-              const emitter = new EventEmitter();
-              process.nextTick(() => {
-                if (storage.fileExists(Key)) {
-                  emitter.emit('data', Buffer.from(storage.readFile(Key)));
-                  emitter.emit('end');
-                } else {
-                  emitter.emit(
-                    'error',
-                    new Error(`The file ${Key} does not exist!`),
-                  );
-                }
-              });
-              return emitter;
-            },
-          };
-        }
-
-        headBucket({ Bucket }: { Bucket: string }) {
-          return {
-            promise: async () => {
-              if (Bucket === 'errorBucket') {
-                throw new Error('Bucket does not exist');
-              }
-              return {};
-            },
-          };
-        }
-
-        upload({ Key, Body }: { Key: string; Body: ReadStream }) {
-          return {
-            promise: () =>
-              new Promise(async resolve => {
-                const chunks = new Array<Buffer>();
-                Body.on('data', chunk => {
-                  chunks.push(chunk as Buffer);
-                });
-                Body.once('end', () => {
-                  storage.writeFile(Key, Buffer.concat(chunks));
-                  resolve(null);
-                });
-              }),
-          };
-        }
-
-        listObjectsV2({ Bucket }: { Bucket: string }) {
-          return {
-            promise: () => {
-              if (
-                Bucket === 'delete_stale_files_success' ||
-                Bucket === 'delete_stale_files_error'
-              ) {
-                return Promise.resolve({
-                  Contents: [{ Key: 'stale_file.png' }],
-                });
-              }
-              return Promise.resolve({});
-            },
-          };
-        }
-
-        deleteObject({ Bucket }: { Bucket: string }) {
-          return {
-            promise: () => {
-              if (Bucket === 'delete_stale_files_error') {
-                throw new Error('Message');
-              }
-              return Promise.resolve();
-            },
-          };
-        }
-      },
-    },
-  };
-});
+const env = process.env;
+let s3Mock: AwsClientStub<S3Client>;

 const getEntityRootDir = (entity: Entity) => {
  const {
@@ -137,6 +49,23 @@ const getEntityRootDir = (entity: Entity) => {
  return path.join(storageRootDir, namespace || DEFAULT_NAMESPACE, kind, name);
 };

+class ErrorReadable extends Readable {
+  errorMessage: string;
+
+  constructor(errorMessage: string) {
+    super();
+    this.errorMessage = errorMessage;
+  }
+
+  _read() {
+    this.destroy(new Error(this.errorMessage));
+  }
+
+  _destroy(error: Error | null, callback: (error?: Error | null) => void) {
+    callback(error);
+  }
+}
+
 const logger = getVoidLogger();
 const loggerInfoSpy = jest.spyOn(logger, 'info');
 const loggerErrorSpy = jest.spyOn(logger, 'error');
@@ -219,13 +148,71 @@ describe('AwsS3Publish', () => {
  };

  beforeEach(() => {
+    process.env = { ...env };
+    process.env.AWS_REGION = 'us-west-2';
+
    mockFs({
      [directory]: files,
    });
+
+    const { StorageFilesMock } = require('../../testUtils/StorageFilesMock');
+    const storage = new StorageFilesMock();
+    storage.emptyFiles();
+
+    s3Mock = mockClient(S3Client);
+
+    s3Mock.on(HeadObjectCommand).callsFake(input => {
+      if (!storage.fileExists(input.Key)) {
+        throw new Error('File does not exist');
+      }
+      return {};
+    });
+
+    s3Mock.on(GetObjectCommand).callsFake(input => {
+      if (storage.fileExists(input.Key)) {
+        return {
+          Body: Readable.from(storage.readFile(input.Key)),
+        };
+      }
+
+      throw new Error(`The file ${input.Key} does not exist!`);
+    });
+
+    s3Mock.on(HeadBucketCommand).callsFake(input => {
+      if (input.Bucket === 'errorBucket') {
+        throw new Error('Bucket does not exist');
+      }
+      return {};
+    });
+
+    s3Mock.on(ListObjectsV2Command).callsFake(input => {
+      if (
+        input.Bucket === 'delete_stale_files_success' ||
+        input.Bucket === 'delete_stale_files_error'
+      ) {
+        return {
+          Contents: [{ Key: 'stale_file.png' }],
+        };
+      }
+      return {};
+    });
+
+    s3Mock.on(DeleteObjectCommand).callsFake(input => {
+      if (input.Bucket === 'delete_stale_files_error') {
+        throw new Error('Message');
+      }
+      return {};
+    });
+
+    s3Mock.on(UploadPartCommand).rejects();
+    s3Mock.on(PutObjectCommand).callsFake(input => {
+      storage.writeFile(input.Key, input.Body);
+    });
  });

  afterEach(() => {
    mockFs.restore();
+    process.env = env;
  });

  describe('getReadiness', () => {
@@ -478,7 +465,30 @@ describe('AwsS3Publish', () => {

      await expect(fails).rejects.toMatchObject({
        message: expect.stringMatching(
-          'TechDocs metadata fetch failed; caused by Error: Unable to read stream; caused by Error: The file invalid/triplet/path/techdocs_metadata.json does not exist',
+          'TechDocs metadata fetch failed; caused by Error: The file invalid/triplet/path/techdocs_metadata.json does not exist',
+        ),
+      });
+    });
+
+    it('should return an error if the techdocs_metadata.json file cannot be read from stream', async () => {
+      s3Mock.on(GetObjectCommand).callsFake(_ => {
+        return {
+          Body: new ErrorReadable('No stream!'),
+      }});
+
+      const publisher = createPublisherFromConfig();
+
+      const invalidEntityName = {
+        namespace: 'invalid',
+        kind: 'triplet',
+        name: 'path',
+      };
+
+      const fails = publisher.fetchTechDocsMetadata(invalidEntityName);
+
+      await expect(fails).rejects.toMatchObject({
+        message: expect.stringMatching(
+          'TechDocs metadata fetch failed; caused by Error: Unable to read stream; caused by Error: No stream!',
        ),
      });
    });
@@ -597,5 +607,21 @@ describe('AwsS3Publish', () => {
        'File Not Found',
      );
    });
+
+    it('should return 404 if file cannot be read from stream', async () => {
+      s3Mock.on(GetObjectCommand).callsFake(_ => {
+        return {
+          Body: new ErrorReadable('No stream!'),
+      }});
+
+      const response = await request(app).get(
+        `/${entityTripletPath}/not-found.html`,
+      );
+      expect(response.status).toBe(404);
+
+      expect(Buffer.from(response.text).toString('utf8')).toEqual(
+        'File Not Found',
+      );
+    });
  });
 });
@@ -16,9 +16,23 @@
 import { Entity, CompoundEntityRef } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { assertError, ForwardedError } from '@backstage/errors';
-import aws, { Credentials } from 'aws-sdk';
-import { ListObjectsV2Output } from 'aws-sdk/clients/s3';
-import { CredentialsOptions } from 'aws-sdk/lib/credentials';
+import {
+  GetObjectCommand,
+  CopyObjectCommand,
+  DeleteObjectCommand,
+  HeadBucketCommand,
+  HeadObjectCommand,
+  PutObjectCommandInput,
+  ListObjectsV2CommandOutput,
+  ListObjectsV2Command,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import {
+  fromNodeProviderChain,
+  fromTemporaryCredentials,
+} from '@aws-sdk/credential-providers';
+import { Upload } from '@aws-sdk/lib-storage';
+import { CredentialProvider } from '@aws-sdk/types';
 import express from 'express';
 import fs from 'fs-extra';
 import JSON5 from 'json5';
@@ -60,7 +74,7 @@ const streamToBuffer = (stream: Readable): Promise<Buffer> => {
 };

 export class AwsS3Publish implements PublisherBase {
-  private readonly storageClient: aws.S3;
+  private readonly storageClient: S3Client;
  private readonly bucketName: string;
  private readonly legacyPathCasing: boolean;
  private readonly logger: Logger;
@@ -68,7 +82,7 @@ export class AwsS3Publish implements PublisherBase {
  private readonly sse?: 'aws:kms' | 'AES256';

  constructor(options: {
-    storageClient: aws.S3;
+    storageClient: S3Client;
    bucketName: string;
    legacyPathCasing: boolean;
    logger: Logger;
@@ -103,6 +117,10 @@ export class AwsS3Publish implements PublisherBase {
      | 'AES256'
      | undefined;

+    // AWS Region is an optional config. If missing, default AWS env variable AWS_REGION
+    // or AWS shared credentials file at ~/.aws/credentials will be used.
+    const region = config.getOptionalString('techdocs.publisher.awsS3.region');
+
    // Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used.
    // 1. AWS environment variables
    // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
@@ -113,11 +131,7 @@ export class AwsS3Publish implements PublisherBase {
    const credentialsConfig = config.getOptionalConfig(
      'techdocs.publisher.awsS3.credentials',
    );
-    const credentials = AwsS3Publish.buildCredentials(credentialsConfig);
-
-    // AWS Region is an optional config. If missing, default AWS env variable AWS_REGION
-    // or AWS shared credentials file at ~/.aws/credentials will be used.
-    const region = config.getOptionalString('techdocs.publisher.awsS3.region');
+    const credentials = AwsS3Publish.buildCredentials(credentialsConfig, region);

    // AWS endpoint is an optional config. If missing, the default endpoint is built from
    // the configured region.
@@ -131,8 +145,9 @@ export class AwsS3Publish implements PublisherBase {
      'techdocs.publisher.awsS3.s3ForcePathStyle',
    );

-    const storageClient = new aws.S3({
-      credentials,
+    const storageClient = new S3Client({
+      customUserAgent: 'backstage-aws-techdocs-s3-publisher',
+      credentialDefaultProvider: () => credentials,
      ...(region && { region }),
      ...(endpoint && { endpoint }),
      ...(s3ForcePathStyle && { s3ForcePathStyle }),
@@ -153,31 +168,39 @@ export class AwsS3Publish implements PublisherBase {
    });
  }

-  private static buildCredentials(
-    config?: Config,
-  ): Credentials | CredentialsOptions | undefined {
+  private static buildStaticCredentials(
+    accessKeyId: string,
+    secretAccessKey: string,
+  ): CredentialProvider {
+    return async () => {
+      return Promise.resolve({
+        accessKeyId,
+        secretAccessKey,
+      });
+    };
+  }
+
+  private static buildCredentials(config?: Config, region?: string): CredentialProvider {
    if (!config) {
-      return undefined;
+      return fromNodeProviderChain();
    }

    const accessKeyId = config.getOptionalString('accessKeyId');
    const secretAccessKey = config.getOptionalString('secretAccessKey');
-    let explicitCredentials: Credentials | undefined;
-    if (accessKeyId && secretAccessKey) {
-      explicitCredentials = new Credentials({
-        accessKeyId,
-        secretAccessKey,
-      });
-    }
+    const explicitCredentials: CredentialProvider =
+      accessKeyId && secretAccessKey
+        ? AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey)
+        : fromNodeProviderChain();

    const roleArn = config.getOptionalString('roleArn');
    if (roleArn) {
-      return new aws.ChainableTemporaryCredentials({
+      return fromTemporaryCredentials({
        masterCredentials: explicitCredentials,
        params: {
          RoleSessionName: 'backstage-aws-techdocs-s3-publisher',
          RoleArn: roleArn,
        },
+        clientConfig: { region },
      });
    }

@@ -190,9 +213,9 @@ export class AwsS3Publish implements PublisherBase {
   */
  async getReadiness(): Promise<ReadinessResponse> {
    try {
-      await this.storageClient
-        .headBucket({ Bucket: this.bucketName })
-        .promise();
+      await this.storageClient.send(
+        new HeadBucketCommand({ Bucket: this.bucketName }),
+      );

      this.logger.info(
        `Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
@@ -258,7 +281,7 @@ export class AwsS3Publish implements PublisherBase {
          const relativeFilePath = path.relative(directory, absoluteFilePath);
          const fileStream = fs.createReadStream(absoluteFilePath);

-          const params = {
+          const params: PutObjectCommandInput = {
            Bucket: this.bucketName,
            Key: getCloudPathForLocalPath(
              entity,
@@ -268,10 +291,15 @@ export class AwsS3Publish implements PublisherBase {
            ),
            Body: fileStream,
            ...(sse && { ServerSideEncryption: sse }),
-          } as aws.S3.PutObjectRequest;
+          };

-          objects.push(params.Key);
-          return this.storageClient.upload(params).promise();
+          objects.push(params.Key!);
+
+          const upload = new Upload({
+            client: this.storageClient,
+            params,
+          });
+          return upload.done();
        },
        absoluteFilesToUpload,
        { concurrencyLimit: 10 },
@@ -301,12 +329,12 @@ export class AwsS3Publish implements PublisherBase {

      await bulkStorageOperation(
        async relativeFilePath => {
-          return await this.storageClient
-            .deleteObject({
+          return await this.storageClient.send(
+            new DeleteObjectCommand({
              Bucket: this.bucketName,
              Key: relativeFilePath,
-            })
-            .promise();
+            }),
+          );
        },
        staleFiles,
        { concurrencyLimit: 10 },
@@ -334,15 +362,17 @@ export class AwsS3Publish implements PublisherBase {

        const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);

-        const stream = this.storageClient
-          .getObject({
-            Bucket: this.bucketName,
-            Key: `${entityRootDir}/techdocs_metadata.json`,
-          })
-          .createReadStream();
-
        try {
-          const techdocsMetadataJson = await streamToBuffer(stream);
+          const resp = await this.storageClient.send(
+            new GetObjectCommand({
+              Bucket: this.bucketName,
+              Key: `${entityRootDir}/techdocs_metadata.json`,
+            }),
+          );
+
+          const techdocsMetadataJson = await streamToBuffer(
+            resp.Body as Readable,
+          );
          if (!techdocsMetadataJson) {
            throw new Error(
              `Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`,
@@ -384,10 +414,11 @@ export class AwsS3Publish implements PublisherBase {
      const fileExtension = path.extname(filePath);
      const responseHeaders = getHeadersForFileExtension(fileExtension);

-      const stream = this.storageClient
-        .getObject({ Bucket: this.bucketName, Key: filePath })
-        .createReadStream();
      try {
+        const resp = await this.storageClient.send(
+          new GetObjectCommand({ Bucket: this.bucketName, Key: filePath }),
+        );
+
        // Inject response headers
        for (const [headerKey, headerValue] of Object.entries(
          responseHeaders,
@@ -395,7 +426,7 @@ export class AwsS3Publish implements PublisherBase {
          res.setHeader(headerKey, headerValue);
        }

-        res.send(await streamToBuffer(stream));
+        res.send(await streamToBuffer(resp.Body as Readable));
      } catch (err) {
        assertError(err);
        this.logger.warn(
@@ -419,12 +450,12 @@ export class AwsS3Publish implements PublisherBase {

      const entityRootDir = path.posix.join(this.bucketRootPath, entityDir);

-      await this.storageClient
-        .headObject({
+      await this.storageClient.send(
+        new HeadObjectCommand({
          Bucket: this.bucketName,
          Key: `${entityRootDir}/index.html`,
-        })
-        .promise();
+        }),
+      );
      return Promise.resolve(true);
    } catch (e) {
      return Promise.resolve(false);
@@ -457,21 +488,21 @@ export class AwsS3Publish implements PublisherBase {

          try {
            this.logger.verbose(`Migrating ${file}`);
-            await this.storageClient
-              .copyObject({
+            await this.storageClient.send(
+              new CopyObjectCommand({
                Bucket: this.bucketName,
                CopySource: [this.bucketName, file].join('/'),
                Key: newPath,
-              })
-              .promise();
+              }),
+            );

            if (removeOriginal) {
-              await this.storageClient
-                .deleteObject({
+              await this.storageClient.send(
+                new DeleteObjectCommand({
                  Bucket: this.bucketName,
                  Key: file,
-                })
-                .promise();
+                }),
+              );
            }
          } catch (e) {
            assertError(e);
@@ -490,16 +521,16 @@ export class AwsS3Publish implements PublisherBase {
  ): Promise<string[]> {
    const objects: string[] = [];
    let nextContinuation: string | undefined;
-    let allObjects: ListObjectsV2Output;
+    let allObjects: ListObjectsV2CommandOutput;
    // Iterate through every file in the root of the publisher.
    do {
-      allObjects = await this.storageClient
-        .listObjectsV2({
+      allObjects = await this.storageClient.send(
+        new ListObjectsV2Command({
          Bucket: this.bucketName,
          ContinuationToken: nextContinuation,
          ...(prefix ? { Prefix: prefix } : {}),
-        })
-        .promise();
+        }),
+      );
      objects.push(
        ...(allObjects.Contents || []).map(f => f.Key || '').filter(f => !!f),
      );