diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md index 7ae1fece63..47ce867137 100644 --- a/docs/integrations/aws-s3/readtree.md +++ b/docs/integrations/aws-s3/readtree.md @@ -21,8 +21,8 @@ configuration: ```yaml catalog: locations: - - type: aws-read-tree + - type: s3-bucket target: https://sample-bucket.s3.us-east-2.amazonaws.com/ ``` -Note the `aws-read-tree` type, as this is not a regular `url` processor. +Note the `s3-bucket` type, as this is not a regular `url` processor. diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 5b37c2fb9f..f2f38d9aab 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -23,7 +23,6 @@ export * from './logging'; export * from './middleware'; export * from './paths'; export * from './reading'; -export * from './reading/tree'; export * from './scm'; export * from './service'; export * from './util'; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index a512d7f64b..593a66cfcc 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -72,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory); + const reader = new AwsS3UrlReader(integration, { + s3, + treeResponseFactory, + }); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -81,8 +84,10 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, - private readonly s3: S3, - private readonly treeResponseFactory: ReadTreeResponseFactory, + private readonly deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, ) {} /** @@ -148,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader { }; } - const response = this.s3.getObject(params); + const response = this.deps.s3.getObject(params); const buffer = await getRawBody(response.createReadStream()); const etag = (await response.promise()).ETag; @@ -184,19 +189,11 @@ export class AwsS3UrlReader implements UrlReader { ContinuationToken: continuationToken, }; } - const { Contents, IsTruncated, NextContinuationToken } = await this.s3 - .listObjectsV2(params) - .promise(); -<<<<<<< HEAD -<<<<<<< HEAD -======= - ->>>>>>> First go -======= ->>>>>>> Preparing for PR + const { Contents, IsTruncated, NextContinuationToken } = + await this.deps.s3.listObjectsV2(params).promise(); const responses = await Promise.all( (Contents || []).map(({ Key }) => { - const s3Response = this.s3 + const s3Response = this.deps.s3 .getObject({ Bucket: bucket, Key: String(Key) }) .createReadStream(); Object.defineProperty(s3Response, 'path', { @@ -216,7 +213,7 @@ export class AwsS3UrlReader implements UrlReader { awsS3Readables = awsS3Readables.concat(responses); } - return await this.treeResponseFactory.fromReadableArray({ + return await this.deps.treeResponseFactory.fromReadableArray({ stream: awsS3Readables, etag: '', }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts index 6b99506e26..0b8ea6f814 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -13,16 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - AwsS3UrlReader, - DefaultReadTreeResponseFactory, -} from '@backstage/backend-common'; - +import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { - AwsS3Integration, - readAwsS3IntegrationConfig, -} from '@backstage/integration'; import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; import { defaultEntityDataParser } from './util/parse'; @@ -30,10 +22,6 @@ import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; -const treeResponseFactory = DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), -}); - AWSMock.setSDKInstance(aws); const object: aws.S3.Types.Object = { Key: 'awsS3-mock-object.yaml', @@ -61,25 +49,18 @@ AWSMock.mock( ), ); -const s3 = new aws.S3(); -const reader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), - ), - ), - s3, - treeResponseFactory, -); +const logger = getVoidLogger(); +const reader = UrlReaders.default({ + logger, + config: new ConfigReader({ + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }), +}); describe('readLocation', () => { const processor = new AwsS3ReadTreeProcessor(reader); const spec = { - type: 'aws-read-tree', + type: 's3-bucket', target: 'https://testbucket.s3.us-east-2.amazonaws.com', }; @@ -90,7 +71,7 @@ describe('readLocation', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ target: 'awsS3-mock-object.yaml', - type: 'aws-read-tree', + type: 's3-bucket', }); expect(generated.entity).toEqual({ site_name: 'Test' }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts index 8e65714a4c..9d8e1cdce5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts @@ -32,7 +32,7 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'aws-read-tree') { + if (location.type !== 's3-bucket') { return false; }