diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index d119115930..da9e77d9ca 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -26,6 +26,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { Readable } from 'stream'; const parseURL = ( url: string, @@ -158,8 +159,62 @@ export class AwsS3UrlReader implements UrlReader { } } - async readTree(): Promise { - throw new Error('AwsS3Reader does not implement readTree'); + async readTree(url: string): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + let moreKeys = true; + let awsS3Readables: Readable[] = []; + let continuationToken = ''; + + while (moreKeys) { + let params; + if (continuationToken === '') { + params = { + Bucket: bucket, + Prefix: path, + }; + } else { + params = { + Bucket: bucket, + Prefix: path, + ContinuationToken: continuationToken, + }; + } + const { Contents, IsTruncated, NextContinuationToken } = await this.s3 + .listObjectsV2(params) + .promise(); + + const responses = await Promise.all( + (Contents || []).map(({ Key }) => { + const s3Response = this.s3 + .getObject({ Bucket: bucket, Key: String(Key) }) + .createReadStream(); + Object.defineProperty(s3Response, 'path', { + value: String(Key), + writable: false, + }); + return s3Response; + }), + ); + + if (IsTruncated) { + continuationToken = String(NextContinuationToken); + } else { + continuationToken = ''; + moreKeys = false; + } + awsS3Readables = awsS3Readables.concat(responses); + } + + return await this.treeResponseFactory.fromReadableArray({ + stream: awsS3Readables, + etag: '', + }); + } catch (e) { + throw new Error(`Could not retrieve file tree from S3: ${e.message}`); + } } async search(): Promise { diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 912fddf965..19d7ed2fab 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -20,9 +20,11 @@ import { ReadTreeResponse, FromArchiveOptions, ReadTreeResponseFactory, + FromReadableArrayOptions, } from '../types'; import { TarArchiveResponse } from './TarArchiveResponse'; import { ZipArchiveResponse } from './ZipArchiveResponse'; +import { ReadableArrayResponse } from './ReadableArrayResponse'; export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { static create(options: { config: Config }): DefaultReadTreeResponseFactory { @@ -53,4 +55,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { options.filter, ); } + + async fromReadableArray( + options: FromReadableArrayOptions, + ): Promise { + return new ReadableArrayResponse(options.stream, options.etag); + } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts new file mode 100644 index 0000000000..8b5d3ff0b5 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import getRawBody from 'raw-body'; +import { Readable } from 'stream'; +import { ReadTreeResponse, ReadTreeResponseFile } from '../types'; + +/** + * Wraps a array of Readable objects into a tree response reader. + */ +export class ReadableArrayResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: Readable[], + public readonly etag: string, + ) { + this.etag = etag; + } + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + async files(): Promise { + this.onlyOnce(); + + const files = Array(); + + for (let i = 0; i < this.stream.length; i++) { + if (!(this.stream[i] as any).path.endsWith('/')) { + files.push({ + path: (this.stream[i] as any).path, + content: () => getRawBody(this.stream[i]), + }); + } + } + + return files; + } + + archive(): Promise { + throw new Error('Method not implemented.'); + } + + dir(): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 93f287d4fe..8d896408c3 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -167,9 +167,19 @@ export type FromArchiveOptions = { filter?: (path: string, info?: { size: number }) => boolean; }; +export type FromReadableArrayOptions = { + // An array of readable streams + stream: Readable[]; + // etag of the file tree + etag: string; +}; + export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; } /**