From 38ba6e9294a9c31c54c0d443d4f3e4e2e987ef1f Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 17 Aug 2021 19:38:27 -0700 Subject: [PATCH] Finish AwsS3ReadTreeProcessor tests Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 51 ++++++++++ packages/backend-common/src/reading/index.ts | 1 + plugins/catalog-backend/package.json | 6 ++ .../processors/AwsS3ReadTreeProcessor.test.ts | 97 +++++++++++++++++++ .../awsS3/awsS3-mock-object.yaml | 1 + 5 files changed, 156 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 78cdcbb7b9..9be6de7a30 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -235,4 +235,55 @@ describe('AwsS3UrlReader', () => { ); }); }); + describe('readTree', () => { + const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', + }; + const objectList: aws.S3.ObjectList = [object]; + const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, + }; + AWSMock.setSDKInstance(aws); + AWSMock.mock('S3', 'listObjectsV2', output); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: '.amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + treeResponseFactory, + ); + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readTree( + 'https://test.s3.us-east-2.amazonaws.com', + ); + const files = await response.files(); + const body = await files[0].content(); + + expect(body.toString()).toBe('site_name: Test\n'); + }); + }); }); diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 207f01b37f..e0e77963f2 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; +export { AwsS3UrlReader } from './AwsS3UrlReader'; export type { ReaderFactory, ReadTreeOptions, diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 7f5ec0b2d6..15a7e65570 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -63,8 +63,14 @@ "yup": "^0.29.3" }, "devDependencies": { +<<<<<<< HEAD "@backstage/backend-test-utils": "^0.1.6", "@backstage/cli": "^0.7.9", +======= + "aws-sdk-mock": "^5.2.1", + "@backstage/backend-test-utils": "^0.1.5", + "@backstage/cli": "^0.7.8", +>>>>>>> Finish AwsS3ReadTreeProcessor tests "@backstage/test-utils": "^0.1.17", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts new file mode 100644 index 0000000000..6b99506e26 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + AwsS3UrlReader, + DefaultReadTreeResponseFactory, +} from '@backstage/backend-common'; + +import { ConfigReader } from '@backstage/config'; +import { + AwsS3Integration, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; +import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; +import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; +import { defaultEntityDataParser } from './util/parse'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + +AWSMock.setSDKInstance(aws); +const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', +}; +const objectList: aws.S3.ObjectList = [object]; +const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, +}; +AWSMock.mock('S3', 'listObjectsV2', output); +AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'ingestion', + 'processors', + '__fixtures__', + 'fileReaderProcessor', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), +); + +const s3 = new aws.S3(); +const reader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + treeResponseFactory, +); + +describe('readLocation', () => { + const processor = new AwsS3ReadTreeProcessor(reader); + const spec = { + type: 'aws-read-tree', + target: 'https://testbucket.s3.us-east-2.amazonaws.com', + }; + + it('should load from url', async () => { + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit, defaultEntityDataParser), + )) as CatalogProcessorEntityResult; + expect(generated.type).toBe('entity'); + expect(generated.location).toEqual({ + target: 'awsS3-mock-object.yaml', + type: 'aws-read-tree', + }); + expect(generated.entity).toEqual({ site_name: 'Test' }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml @@ -0,0 +1 @@ +site_name: Test