From 7e6186cf6b522c0130d683a8cd6dcaa5cde2f018 Mon Sep 17 00:00:00 2001 From: Elliot Greenwood Date: Wed, 27 Oct 2021 13:35:00 +0100 Subject: [PATCH 1/2] Add NotModifiedError for the AWSS3UrlReader Signed-off-by: Elliot Greenwood --- .../src/reading/AwsS3UrlReader.test.ts | 209 ++++++++++++------ .../src/reading/AwsS3UrlReader.ts | 6 +- 2 files changed, 142 insertions(+), 73 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 301543e8c9..a3ae8b647f 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -27,6 +27,7 @@ import { UrlReaderPredicateTuple } from './types'; import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; +import { NotModifiedError } from '@backstage/errors'; const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), @@ -131,29 +132,37 @@ describe('AwsS3UrlReader', () => { }); describe('read', () => { - AWSMock.setSDKInstance(aws); - AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3/awsS3-mock-object.yaml'), + let awsS3UrlReader: AwsS3UrlReader; + + beforeAll(() => { + AWSMock.setSDKInstance(aws); + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), ), - ), - ); - const s3 = new aws.S3(); - const awsS3UrlReader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), + ); + + const s3 = new aws.S3(); + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), ), - ), - { s3, treeResponseFactory }, - ); + { s3, treeResponseFactory }, + ); + }); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.read( @@ -176,32 +185,39 @@ describe('AwsS3UrlReader', () => { }); describe('readUrl', () => { - AWSMock.setSDKInstance(aws); + let awsS3UrlReader: AwsS3UrlReader; - AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3/awsS3-mock-object.yaml'), + beforeAll(() => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), ), - ), - ); + ); - const s3 = new aws.S3(); + const s3 = new aws.S3(); - const awsS3UrlReader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), ), - ), - { s3, treeResponseFactory }, - ); + { s3, treeResponseFactory }, + ); + }); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readUrl( @@ -223,40 +239,89 @@ describe('AwsS3UrlReader', () => { ); }); }); + + describe('readUrl with etag', () => { + let awsS3UrlReader: AwsS3UrlReader; + + beforeAll(() => { + AWSMock.setSDKInstance(aws); + + AWSMock.mock('S3', 'getObject', (_, callback) => { + callback({ statusCode: 304 }, null); + }); + + const s3 = new aws.S3(); + + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + { s3, treeResponseFactory }, + ); + }); + + it('returns contents of an object in a bucket', async () => { + await expect( + awsS3UrlReader.readUrl( + 'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml', + { + etag: 'abc123', + }, + ), + ).rejects.toThrow(NotModifiedError); + }); + }); + describe('readTree', () => { - const object: aws.S3.Types.Object = { - Key: 'awsS3-mock-object.yaml', - }; - const objectList: aws.S3.ObjectList = [object]; - const output: aws.S3.Types.ListObjectsV2Output = { - Contents: objectList, - }; - AWSMock.setSDKInstance(aws); - AWSMock.mock('S3', 'listObjectsV2', output); + let awsS3UrlReader: AwsS3UrlReader; - AWSMock.mock( - 'S3', - 'getObject', - Buffer.from( - require('fs').readFileSync( - path.resolve(__dirname, '__fixtures__/awsS3/awsS3-mock-object.yaml'), - ), - ), - ); + beforeAll(() => { + const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', + }; - const s3 = new aws.S3(); - const awsS3UrlReader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: '.amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), + const objectList: aws.S3.ObjectList = [object]; + const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, + }; + + AWSMock.setSDKInstance(aws); + AWSMock.mock('S3', 'listObjectsV2', output); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + __dirname, + '__fixtures__/awsS3/awsS3-mock-object.yaml', + ), + ), ), - ), - { s3, treeResponseFactory }, - ); + ); + + const s3 = new aws.S3(); + awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: '.amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + { s3, treeResponseFactory }, + ); + }); + it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readTree( 'https://test.s3.us-east-2.amazonaws.com', diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 7fd0f6ba19..05a14e26ec 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -27,7 +27,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; -import { ForwardedError } from '@backstage/errors'; +import { ForwardedError, NotModifiedError } from '@backstage/errors'; import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( @@ -163,6 +163,10 @@ export class AwsS3UrlReader implements UrlReader { etag: etag, }; } catch (e) { + if (e.statusCode === 304) { + throw new NotModifiedError(); + } + throw new ForwardedError('Could not retrieve file from S3', e); } } From 8c4cad0bf2e51533109fe6f5147fde33cac16618 Mon Sep 17 00:00:00 2001 From: Elliot Greenwood Date: Wed, 27 Oct 2021 14:33:08 +0100 Subject: [PATCH 2/2] Add changeset for backstage common Signed-off-by: Elliot Greenwood --- .changeset/few-waves-dream.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/few-waves-dream.md diff --git a/.changeset/few-waves-dream.md b/.changeset/few-waves-dream.md new file mode 100644 index 0000000000..72123171b8 --- /dev/null +++ b/.changeset/few-waves-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +AWSS3UrlReader now throws a `NotModifiedError` (exported from @backstage/backend-common) when s3 returns a 304 response.