From 1b37850698be2ae713774686d5a285adec55e2dd Mon Sep 17 00:00:00 2001 From: Lucas Guarisco Date: Wed, 1 Mar 2023 15:40:11 -0300 Subject: [PATCH] Modify AwsS3DiscoveryProcessor to use aws-sdk v3 Signed-off-by: Lucas Guarisco --- .../catalog-backend-module-aws/package.json | 3 ++ .../AwsS3DiscoveryProcessor.test.ts | 41 +++++++++++-------- yarn.lock | 5 ++- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index 6db1b8aa11..4883d34425 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -45,6 +45,7 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@aws-sdk/client-s3": "^3.281.0", "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", @@ -64,9 +65,11 @@ "winston": "^3.2.1" }, "devDependencies": { + "@aws-sdk/util-stream-node": "^3.272.0", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", + "aws-sdk-client-mock": "^2.0.1", "aws-sdk-mock": "^5.2.1", "luxon": "^3.0.0", "yaml": "^2.0.0" diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts index 6aab7b2bec..ef540da7f5 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts @@ -22,29 +22,26 @@ import { CatalogProcessorResult, processingResult, } from '@backstage/plugin-catalog-backend'; -import AWSMock from 'aws-sdk-mock'; -import aws from 'aws-sdk'; +import { + S3Client, + ListObjectsV2Command, + ListObjectsV2Output, + GetObjectCommand, +} from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import { sdkStreamMixin } from '@aws-sdk/util-stream-node'; +import fs from 'fs'; import path from 'path'; import YAML from 'yaml'; -AWSMock.setSDKInstance(aws); -const object: aws.S3.Types.Object = { +const s3Client = mockClient(S3Client); +const object: Object = { Key: 'awsS3-mock-object.txt', }; -const objectList: aws.S3.ObjectList = [object]; -const output: aws.S3.Types.ListObjectsV2Output = { +const objectList: Object[] = [object]; +const output: ListObjectsV2Output = { Contents: objectList, }; -AWSMock.mock('S3', 'listObjectsV2', output); -AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3-mock-object.txt'), - ), - ), -); const logger = getVoidLogger(); const reader = UrlReaders.default({ @@ -61,6 +58,18 @@ describe('readLocation', () => { target: 'https://testbucket.s3.us-east-2.amazonaws.com', }; + beforeEach(() => { + s3Client.reset(); + s3Client.on(ListObjectsV2Command).resolves(output); + s3Client.on(GetObjectCommand).resolves({ + Body: sdkStreamMixin( + fs.createReadStream( + path.resolve(__dirname, '__fixtures__/awsS3-mock-object.txt'), + ), + ), + }); + }); + it('should load from url', async () => { const generated = (await new Promise(emit => processor.readLocation( diff --git a/yarn.lock b/yarn.lock index acda53d655..82321cb4ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -445,7 +445,7 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-s3@npm:^3.208.0, @aws-sdk/client-s3@npm:^3.276.0": +"@aws-sdk/client-s3@npm:^3.208.0, @aws-sdk/client-s3@npm:^3.276.0, @aws-sdk/client-s3@npm:^3.281.0": version: 3.281.0 resolution: "@aws-sdk/client-s3@npm:3.281.0" dependencies: @@ -5177,6 +5177,8 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-aws@workspace:plugins/catalog-backend-module-aws" dependencies: + "@aws-sdk/client-s3": ^3.281.0 + "@aws-sdk/util-stream-node": ^3.272.0 "@backstage/backend-common": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" @@ -5193,6 +5195,7 @@ __metadata: "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 aws-sdk: ^2.840.0 + aws-sdk-client-mock: ^2.0.1 aws-sdk-mock: ^5.2.1 lodash: ^4.17.21 luxon: ^3.0.0