From b8086dedd7d46c8b1616bd88ef90f314c5ec6d25 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Fri, 13 Aug 2021 16:30:05 -0700
Subject: [PATCH 01/25] First go

Signed-off-by: Sean Tan <seant@splunk.com>
---
 .../src/reading/AwsS3UrlReader.ts             | 59 ++++++++++++++++-
 .../reading/tree/ReadTreeResponseFactory.ts   |  8 +++
 .../src/reading/tree/ReadableArrayResponse.ts | 66 +++++++++++++++++++
 packages/backend-common/src/reading/types.ts  | 10 +++
 4 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 packages/backend-common/src/reading/tree/ReadableArrayResponse.ts

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index d119115930..da9e77d9ca 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -26,6 +26,7 @@ import {
 } from './types';
 import getRawBody from 'raw-body';
 import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+import { Readable } from 'stream';
 
 const parseURL = (
   url: string,
@@ -158,8 +159,62 @@ export class AwsS3UrlReader implements UrlReader {
     }
   }
 
-  async readTree(): Promise<ReadTreeResponse> {
-    throw new Error('AwsS3Reader does not implement readTree');
+  async readTree(url: string): Promise<ReadTreeResponse> {
+    try {
+      const { path, bucket, region } = parseURL(url);
+      aws.config.update({ region: region });
+
+      let moreKeys = true;
+      let awsS3Readables: Readable[] = [];
+      let continuationToken = '';
+
+      while (moreKeys) {
+        let params;
+        if (continuationToken === '') {
+          params = {
+            Bucket: bucket,
+            Prefix: path,
+          };
+        } else {
+          params = {
+            Bucket: bucket,
+            Prefix: path,
+            ContinuationToken: continuationToken,
+          };
+        }
+        const { Contents, IsTruncated, NextContinuationToken } = await this.s3
+          .listObjectsV2(params)
+          .promise();
+
+        const responses = await Promise.all(
+          (Contents || []).map(({ Key }) => {
+            const s3Response = this.s3
+              .getObject({ Bucket: bucket, Key: String(Key) })
+              .createReadStream();
+            Object.defineProperty(s3Response, 'path', {
+              value: String(Key),
+              writable: false,
+            });
+            return s3Response;
+          }),
+        );
+
+        if (IsTruncated) {
+          continuationToken = String(NextContinuationToken);
+        } else {
+          continuationToken = '';
+          moreKeys = false;
+        }
+        awsS3Readables = awsS3Readables.concat(responses);
+      }
+
+      return await this.treeResponseFactory.fromReadableArray({
+        stream: awsS3Readables,
+        etag: '',
+      });
+    } catch (e) {
+      throw new Error(`Could not retrieve file tree from S3: ${e.message}`);
+    }
   }
 
   async search(): Promise<SearchResponse> {
diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
index 912fddf965..19d7ed2fab 100644
--- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
+++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
@@ -20,9 +20,11 @@ import {
   ReadTreeResponse,
   FromArchiveOptions,
   ReadTreeResponseFactory,
+  FromReadableArrayOptions,
 } from '../types';
 import { TarArchiveResponse } from './TarArchiveResponse';
 import { ZipArchiveResponse } from './ZipArchiveResponse';
+import { ReadableArrayResponse } from './ReadableArrayResponse';
 
 export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
   static create(options: { config: Config }): DefaultReadTreeResponseFactory {
@@ -53,4 +55,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
       options.filter,
     );
   }
+
+  async fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse> {
+    return new ReadableArrayResponse(options.stream, options.etag);
+  }
 }
diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts
new file mode 100644
index 0000000000..8b5d3ff0b5
--- /dev/null
+++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import getRawBody from 'raw-body';
+import { Readable } from 'stream';
+import { ReadTreeResponse, ReadTreeResponseFile } from '../types';
+
+/**
+ * Wraps a array of Readable objects into a tree response reader.
+ */
+export class ReadableArrayResponse implements ReadTreeResponse {
+  private read = false;
+
+  constructor(
+    private readonly stream: Readable[],
+    public readonly etag: string,
+  ) {
+    this.etag = etag;
+  }
+
+  // Make sure the input stream is only read once
+  private onlyOnce() {
+    if (this.read) {
+      throw new Error('Response has already been read');
+    }
+    this.read = true;
+  }
+
+  async files(): Promise<ReadTreeResponseFile[]> {
+    this.onlyOnce();
+
+    const files = Array<ReadTreeResponseFile>();
+
+    for (let i = 0; i < this.stream.length; i++) {
+      if (!(this.stream[i] as any).path.endsWith('/')) {
+        files.push({
+          path: (this.stream[i] as any).path,
+          content: () => getRawBody(this.stream[i]),
+        });
+      }
+    }
+
+    return files;
+  }
+
+  archive(): Promise<NodeJS.ReadableStream> {
+    throw new Error('Method not implemented.');
+  }
+
+  dir(): Promise<string> {
+    throw new Error('Method not implemented.');
+  }
+}
diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts
index 93f287d4fe..8d896408c3 100644
--- a/packages/backend-common/src/reading/types.ts
+++ b/packages/backend-common/src/reading/types.ts
@@ -167,9 +167,19 @@ export type FromArchiveOptions = {
   filter?: (path: string, info?: { size: number }) => boolean;
 };
 
+export type FromReadableArrayOptions = {
+  // An array of readable streams
+  stream: Readable[];
+  // etag of the file tree
+  etag: string;
+};
+
 export interface ReadTreeResponseFactory {
   fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
   fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
+  fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse>;
 }
 
 /**

From 522b55fdcccbe2fb308c66e4f9bc42d5d777c333 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Fri, 13 Aug 2021 16:46:42 -0700
Subject: [PATCH 02/25] More work Signed-off-by: Sean Tan <seant@splunk.com>

---
 .../src/reading/AwsS3UrlReader.ts             |  6 +-
 .../processors/AwsS3ReadTreeProcessor.ts      | 74 +++++++++++++++++++
 2 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index da9e77d9ca..f7904ca70f 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadTreeResponseFactory,
   ReadUrlOptions,
   ReadUrlResponse,
   SearchResponse,
@@ -62,7 +63,7 @@ const parseURL = (
 };
 
 export class AwsS3UrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);
 
     return integrations.awsS3.list().map(integration => {
@@ -71,7 +72,7 @@ export class AwsS3UrlReader implements UrlReader {
         apiVersion: '2006-03-01',
         credentials: creds,
       });
-      const reader = new AwsS3UrlReader(integration, s3);
+      const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory);
       const predicate = (url: URL) =>
         url.host.endsWith(integration.config.host);
       return { reader, predicate };
@@ -81,6 +82,7 @@ export class AwsS3UrlReader implements UrlReader {
   constructor(
     private readonly integration: AwsS3Integration,
     private readonly s3: S3,
+    private readonly treeResponseFactory: ReadTreeResponseFactory,
   ) {}
 
   /**
diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts
new file mode 100644
index 0000000000..8e65714a4c
--- /dev/null
+++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import * as result from './results';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorParser,
+} from './types';
+
+export class AwsS3ReadTreeProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean> {
+    if (location.type !== 'aws-read-tree') {
+      return false;
+    }
+
+    try {
+      const output = await this.doRead(location.target);
+      for (const item of output) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url },
+        })) {
+          emit(parseResult);
+        }
+      }
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`;
+
+      if (error.name === 'NotFoundError') {
+        if (!optional) {
+          emit(result.notFoundError(location, message));
+        }
+      } else {
+        emit(result.generalError(location, message));
+      }
+    }
+    return true;
+  }
+
+  private async doRead(
+    location: string,
+  ): Promise<{ data: Buffer; url: string }[]> {
+    const response = await this.reader.readTree(location);
+    const responseFiles = await response.files();
+    const output = responseFiles.map(async file => ({
+      url: file.path,
+      data: await file.content(),
+    }));
+    return Promise.all(output);
+  }
+}

From 9972778cd747a037be835fe11271a575c21e3bbe Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Fri, 13 Aug 2021 17:24:14 -0700
Subject: [PATCH 03/25] Merge work done on previous branch Signed-off-by: Sean
 Tan <seant@splunk.com>

---
 app-config.yaml                                        |  2 ++
 .../backend-common/src/reading/AwsS3UrlReader.test.ts  | 10 +++++++---
 packages/backend/src/plugins/catalog.ts                |  3 ++-
 .../catalog-backend/src/ingestion/processors/index.ts  |  1 +
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 9f365becef..fcfd92a66f 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -259,6 +259,8 @@ catalog:
     # Backstage example groups and users
     - type: file
       target: ../catalog-model/examples/acme-corp.yaml
+    - type: aws-read-tree
+      target: https://seant-splunk.s3.us-east-2.amazonaws.com
 
 scaffolder:
   # Use to customize default commit author info used when new components are created
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index 084f7e3e13..78bfdbb0e2 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
 
+const treeResponseFactory = DefaultReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});
+
 describe('AwsS3UrlReader', () => {
   const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
     return AwsS3UrlReader.factory({
       config: new ConfigReader(config),
       logger: getVoidLogger(),
-      treeResponseFactory: DefaultReadTreeResponseFactory.create({
-        config: new ConfigReader({}),
-      }),
+      treeResponseFactory,
     });
   };
 
@@ -154,6 +156,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ),
       s3,
+      treeResponseFactory,
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -207,6 +210,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ),
       s3,
+      treeResponseFactory,
     );
 
     it('returns contents of an object in a bucket', async () => {
diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 055595dcb5..9c94b3851e 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor';
 import {
   CatalogBuilder,
   createRouter,
@@ -25,6 +25,7 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader));
   const {
     entitiesCatalog,
     locationAnalyzer,
diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts
index 6a44ea0a8f..23d14dde01 100644
--- a/plugins/catalog-backend/src/ingestion/processors/index.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/index.ts
@@ -19,6 +19,7 @@ import * as results from './results';
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+export { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
 export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';

From 20334fc3cb5f0c8a518ddd56bcb30ee2c0aa58ac Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Mon, 16 Aug 2021 17:14:47 -0700
Subject: [PATCH 04/25] Add ReadableArrayResponse tests Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 .../src/reading/AwsS3UrlReader.test.ts        |  2 +
 .../{ => awsS3}/awsS3-mock-object.yaml        |  0
 .../awsS3/awsS3-mock-object2.yaml             |  1 +
 .../tree/ReadableArrayResponse.test.ts        | 62 +++++++++++++++++++
 4 files changed, 65 insertions(+)
 rename packages/backend-common/src/reading/__fixtures__/{ => awsS3}/awsS3-mock-object.yaml (100%)
 create mode 100644 packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml
 create mode 100644 packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index 78bfdbb0e2..78cdcbb7b9 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -139,6 +139,7 @@ describe('AwsS3UrlReader', () => {
             'src',
             'reading',
             '__fixtures__',
+            'awsS3',
             'awsS3-mock-object.yaml',
           ),
         ),
@@ -191,6 +192,7 @@ describe('AwsS3UrlReader', () => {
             'src',
             'reading',
             '__fixtures__',
+            'awsS3',
             'awsS3-mock-object.yaml',
           ),
         ),
diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml
similarity index 100%
rename from packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml
rename to packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml
diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml
new file mode 100644
index 0000000000..b2c132961b
--- /dev/null
+++ b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml
@@ -0,0 +1 @@
+site_name: Test2
diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts
new file mode 100644
index 0000000000..c2e3c26269
--- /dev/null
+++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ReadableArrayResponse } from './ReadableArrayResponse';
+import path from 'path';
+import { Readable } from 'stream';
+import fs from 'fs';
+
+const arr: Readable[] = [];
+const file1 = path.resolve(
+  'src',
+  'reading',
+  '__fixtures__',
+  'awsS3',
+  'awsS3-mock-object.yaml',
+);
+const file2 = path.resolve(
+  'src',
+  'reading',
+  '__fixtures__',
+  'awsS3',
+  'awsS3-mock-object2.yaml',
+);
+const stream1 = fs.createReadStream(file1);
+const stream2 = fs.createReadStream(file2);
+arr.push(stream1);
+arr.push(stream2);
+
+describe('ReadableArrayResponse', () => {
+  it('should read files', async () => {
+    const res = new ReadableArrayResponse(arr, 'etag');
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: file1,
+        content: expect.any(Function),
+      },
+      {
+        path: file2,
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      'site_name: Test',
+      'site_name: Test2',
+    ]);
+  });
+});

From 6d6392a1d389e741eeca7dfa31147c5cea56af87 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Tue, 17 Aug 2021 19:38:27 -0700
Subject: [PATCH 05/25] Finish AwsS3ReadTreeProcessor tests Signed-off-by: Sean
 Tan <seant@splunk.com>

---
 .../src/reading/AwsS3UrlReader.test.ts        | 51 ++++++++++
 packages/backend-common/src/reading/index.ts  |  1 +
 plugins/catalog-backend/package.json          |  1 +
 .../processors/AwsS3ReadTreeProcessor.test.ts | 97 +++++++++++++++++++
 .../awsS3/awsS3-mock-object.yaml              |  1 +
 5 files changed, 151 insertions(+)
 create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
 create mode 100644 plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index 78cdcbb7b9..9be6de7a30 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -235,4 +235,55 @@ describe('AwsS3UrlReader', () => {
       );
     });
   });
+  describe('readTree', () => {
+    const object: aws.S3.Types.Object = {
+      Key: 'awsS3-mock-object.yaml',
+    };
+    const objectList: aws.S3.ObjectList = [object];
+    const output: aws.S3.Types.ListObjectsV2Output = {
+      Contents: objectList,
+    };
+    AWSMock.setSDKInstance(aws);
+    AWSMock.mock('S3', 'listObjectsV2', output);
+
+    AWSMock.mock(
+      'S3',
+      'getObject',
+      Buffer.from(
+        require('fs').readFileSync(
+          path.resolve(
+            'src',
+            'reading',
+            '__fixtures__',
+            'awsS3',
+            'awsS3-mock-object.yaml',
+          ),
+        ),
+      ),
+    );
+
+    const s3 = new aws.S3();
+    const awsS3UrlReader = new AwsS3UrlReader(
+      new AwsS3Integration(
+        readAwsS3IntegrationConfig(
+          new ConfigReader({
+            host: '.amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          }),
+        ),
+      ),
+      s3,
+      treeResponseFactory,
+    );
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.readTree(
+        'https://test.s3.us-east-2.amazonaws.com',
+      );
+      const files = await response.files();
+      const body = await files[0].content();
+
+      expect(body.toString()).toBe('site_name: Test\n');
+    });
+  });
 });
diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts
index 4c601556d7..e50ba1e305 100644
--- a/packages/backend-common/src/reading/index.ts
+++ b/packages/backend-common/src/reading/index.ts
@@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader';
 export { BitbucketUrlReader } from './BitbucketUrlReader';
 export { GithubUrlReader } from './GithubUrlReader';
 export { GitlabUrlReader } from './GitlabUrlReader';
+export { AwsS3UrlReader } from './AwsS3UrlReader';
 export type {
   ReadTreeResponse,
   ReadTreeResponseFile,
diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json
index f095f39785..e490214385 100644
--- a/plugins/catalog-backend/package.json
+++ b/plugins/catalog-backend/package.json
@@ -61,6 +61,7 @@
     "yup": "^0.29.3"
   },
   "devDependencies": {
+    "aws-sdk-mock": "^5.2.1",
     "@backstage/backend-test-utils": "^0.1.5",
     "@backstage/cli": "^0.7.8",
     "@backstage/test-utils": "^0.1.17",
diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
new file mode 100644
index 0000000000..6b99506e26
--- /dev/null
+++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  AwsS3UrlReader,
+  DefaultReadTreeResponseFactory,
+} from '@backstage/backend-common';
+
+import { ConfigReader } from '@backstage/config';
+import {
+  AwsS3Integration,
+  readAwsS3IntegrationConfig,
+} from '@backstage/integration';
+import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
+import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types';
+import { defaultEntityDataParser } from './util/parse';
+import AWSMock from 'aws-sdk-mock';
+import aws from 'aws-sdk';
+import path from 'path';
+
+const treeResponseFactory = DefaultReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});
+
+AWSMock.setSDKInstance(aws);
+const object: aws.S3.Types.Object = {
+  Key: 'awsS3-mock-object.yaml',
+};
+const objectList: aws.S3.ObjectList = [object];
+const output: aws.S3.Types.ListObjectsV2Output = {
+  Contents: objectList,
+};
+AWSMock.mock('S3', 'listObjectsV2', output);
+AWSMock.mock(
+  'S3',
+  'getObject',
+  Buffer.from(
+    require('fs').readFileSync(
+      path.resolve(
+        'src',
+        'ingestion',
+        'processors',
+        '__fixtures__',
+        'fileReaderProcessor',
+        'awsS3',
+        'awsS3-mock-object.yaml',
+      ),
+    ),
+  ),
+);
+
+const s3 = new aws.S3();
+const reader = new AwsS3UrlReader(
+  new AwsS3Integration(
+    readAwsS3IntegrationConfig(
+      new ConfigReader({
+        host: 'amazonaws.com',
+        accessKeyId: 'fake-access-key',
+        secretAccessKey: 'fake-secret-key',
+      }),
+    ),
+  ),
+  s3,
+  treeResponseFactory,
+);
+
+describe('readLocation', () => {
+  const processor = new AwsS3ReadTreeProcessor(reader);
+  const spec = {
+    type: 'aws-read-tree',
+    target: 'https://testbucket.s3.us-east-2.amazonaws.com',
+  };
+
+  it('should load from url', async () => {
+    const generated = (await new Promise<CatalogProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit, defaultEntityDataParser),
+    )) as CatalogProcessorEntityResult;
+    expect(generated.type).toBe('entity');
+    expect(generated.location).toEqual({
+      target: 'awsS3-mock-object.yaml',
+      type: 'aws-read-tree',
+    });
+    expect(generated.entity).toEqual({ site_name: 'Test' });
+  });
+});
diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml
new file mode 100644
index 0000000000..7470c0e8a3
--- /dev/null
+++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml
@@ -0,0 +1 @@
+site_name: Test

From 2e9e29cb7a28bca82f405124c4f07e852a842cc9 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:28:27 -0700
Subject: [PATCH 06/25] Preparing for PR Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 docs/integrations/aws-s3/readtree.md          | 28 +++++++++++++++++++
 microsite/sidebars.json                       |  2 +-
 mkdocs.yml                                    |  1 +
 packages/backend-common/src/index.ts          |  1 +
 .../src/reading/AwsS3UrlReader.ts             |  1 -
 5 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 docs/integrations/aws-s3/readtree.md

diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md
new file mode 100644
index 0000000000..7ae1fece63
--- /dev/null
+++ b/docs/integrations/aws-s3/readtree.md
@@ -0,0 +1,28 @@
+---
+id: read-tree
+title: AWS S3 Read Tree
+sidebar_label: Read Tree
+# prettier-ignore
+description: Discovering multiple catalog entities from an AWS S3 Bucket
+---
+
+The AWS S3 integration has a special processor for using AwsS3UrlReader's
+readTree() method. The method readTree() allows a user to retrieve more than one
+file hosted inside an S3 bucket. The processor passes a valid S3 URL and
+credentials to AwsS3UrlReader's readTree(). A user can supply a URL which points
+to root of a bucket or a subdirectory inside the bucket. This can be useful as
+an alternative to static locations or manually adding things to the catalog.
+
+To use the discovery processor, you'll need an AWS S3 integration
+[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and
+optionally a `roleArn`. Then you can add a location target to the catalog
+configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: aws-read-tree
+      target: https://sample-bucket.s3.us-east-2.amazonaws.com/
+```
+
+Note the `aws-read-tree` type, as this is not a regular `url` processor.
diff --git a/microsite/sidebars.json b/microsite/sidebars.json
index d787fb28aa..3517c8d70c 100644
--- a/microsite/sidebars.json
+++ b/microsite/sidebars.json
@@ -107,7 +107,7 @@
       {
         "type": "subcategory",
         "label": "AWS S3",
-        "ids": ["integrations/aws-s3/locations"]
+        "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/readtree"]
       },
       {
         "type": "subcategory",
diff --git a/mkdocs.yml b/mkdocs.yml
index 73ad52341e..3b4dad765c 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -81,6 +81,7 @@ nav:
       - Overview: 'integrations/index.md'
       - AWS S3:
           - Locations: 'integrations/aws-s3/locations.md'
+          - Read Tree: 'integrations/aws-s3/readtree.md'
       - Azure:
           - Locations: 'integrations/azure/locations.md'
           - Org Data: 'integrations/azure/org.md'
diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts
index f2f38d9aab..5b37c2fb9f 100644
--- a/packages/backend-common/src/index.ts
+++ b/packages/backend-common/src/index.ts
@@ -23,6 +23,7 @@ export * from './logging';
 export * from './middleware';
 export * from './paths';
 export * from './reading';
+export * from './reading/tree';
 export * from './scm';
 export * from './service';
 export * from './util';
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index f7904ca70f..615a095e50 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -187,7 +187,6 @@ export class AwsS3UrlReader implements UrlReader {
         const { Contents, IsTruncated, NextContinuationToken } = await this.s3
           .listObjectsV2(params)
           .promise();
-
         const responses = await Promise.all(
           (Contents || []).map(({ Key }) => {
             const s3Response = this.s3

From 96b818fdb2ea5f16f279af38b33c46f684d46099 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:40:25 -0700
Subject: [PATCH 07/25] Add readTree() to AwsS3UrlReader and
 AwsS3ReadTreeProcessor Signed-off-by: Sean Tan <seant@splunk.com>

---
 .changeset/shiny-emus-perform.md | 7 +++++++
 app-config.yaml                  | 2 --
 2 files changed, 7 insertions(+), 2 deletions(-)
 create mode 100644 .changeset/shiny-emus-perform.md

diff --git a/.changeset/shiny-emus-perform.md b/.changeset/shiny-emus-perform.md
new file mode 100644
index 0000000000..76a40504c8
--- /dev/null
+++ b/.changeset/shiny-emus-perform.md
@@ -0,0 +1,7 @@
+---
+'@backstage/backend-common': patch
+'@backstage/integration': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add readTree() to AwsS3UrlReader and aws-read-tree Catalog Processor
diff --git a/app-config.yaml b/app-config.yaml
index fcfd92a66f..9f365becef 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -259,8 +259,6 @@ catalog:
     # Backstage example groups and users
     - type: file
       target: ../catalog-model/examples/acme-corp.yaml
-    - type: aws-read-tree
-      target: https://seant-splunk.s3.us-east-2.amazonaws.com
 
 scaffolder:
   # Use to customize default commit author info used when new components are created

From 030e3e30e6421b7e447fd02bb23c5d332c922cd2 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:46:48 -0700
Subject: [PATCH 08/25] Remove local changes made to catalog.ts Signed-off-by:
 Sean Tan <seant@splunk.com>

---
 packages/backend/src/plugins/catalog.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 9c94b3851e..75709eefc7 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor';
 import {
   CatalogBuilder,
   createRouter,
@@ -25,7 +24,6 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   const builder = await CatalogBuilder.create(env);
-  builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader));
   const {
     entitiesCatalog,
     locationAnalyzer,

From 55262fb1389f0b161226229757940f19bf6663fe Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:49:11 -0700
Subject: [PATCH 09/25] Fix one more error in catalog.ts Signed-off-by: Sean
 Tan <seant@splunk.com>

Signed-off-by: Sean Tan <seant@splunk.com>
---
 packages/backend/src/plugins/catalog.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 75709eefc7..055595dcb5 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import {
   CatalogBuilder,
   createRouter,

From bedf145c2c700850c56100472c57541a8424e8bf Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 15:51:31 -0700
Subject: [PATCH 10/25] Regenerate api reports Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 packages/backend-common/api-report.md | 58 ++++++++++++++++++++++++---
 plugins/catalog-backend/api-report.md | 14 +++++++
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md
index 736899d1ed..adbc32f67c 100644
--- a/packages/backend-common/api-report.md
+++ b/packages/backend-common/api-report.md
@@ -6,6 +6,7 @@
 /// <reference types="node" />
 /// <reference types="webpack-env" />
 
+import { AwsS3Integration } from '@backstage/integration';
 import { AzureIntegration } from '@backstage/integration';
 import { BitbucketIntegration } from '@backstage/integration';
 import { Config } from '@backstage/config';
@@ -28,10 +29,40 @@ import { Readable } from 'stream';
 import { ReadCommitResult } from 'isomorphic-git';
 import { RequestHandler } from 'express';
 import { Router } from 'express';
+import { S3 } from 'aws-sdk';
 import { Server } from 'http';
 import * as winston from 'winston';
 import { Writable } from 'stream';
 
+// Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class AwsS3UrlReader implements UrlReader {
+  // Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts
+  constructor(
+    integration: AwsS3Integration,
+    s3: S3,
+    treeResponseFactory: ReadTreeResponseFactory,
+  );
+  // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  static factory: ReaderFactory;
+  // (undocumented)
+  read(url: string): Promise<Buffer>;
+  // (undocumented)
+  readTree(url: string): Promise<ReadTreeResponse>;
+  // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts
+  // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+  // (undocumented)
+  search(): Promise<SearchResponse>;
+  // (undocumented)
+  toString(): string;
+}
+
 // Warning: (ae-missing-release-tag) "AzureUrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -42,8 +73,6 @@ export class AzureUrlReader implements UrlReader {
       treeResponseFactory: ReadTreeResponseFactory;
     },
   );
-  // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts
-  //
   // (undocumented)
   static factory: ReaderFactory;
   // (undocumented)
@@ -52,9 +81,6 @@ export class AzureUrlReader implements UrlReader {
   //
   // (undocumented)
   readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
-  // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts
-  // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts
-  //
   // (undocumented)
   readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
   // Warning: (ae-forgotten-export) The symbol "SearchOptions" needs to be exported by the entry point index.d.ts
@@ -173,6 +199,27 @@ export class DatabaseManager {
   static fromConfig(config: Config): DatabaseManager;
 }
 
+// Warning: (ae-missing-release-tag) "DefaultReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
+  constructor(workDir: string);
+  // (undocumented)
+  static create(options: { config: Config }): DefaultReadTreeResponseFactory;
+  // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse>;
+  // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
+  // (undocumented)
+  fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
+}
+
 // Warning: (ae-missing-release-tag) "DockerContainerRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -592,7 +639,6 @@ export function useHotMemoize<T>(_module: NodeModule, valueFactory: () => T): T;
 //
 // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts
 // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here
-// src/reading/AzureUrlReader.d.ts:9:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts
 // src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts
 // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md
index a6e67215c9..3ccebf48f9 100644
--- a/plugins/catalog-backend/api-report.md
+++ b/plugins/catalog-backend/api-report.md
@@ -121,6 +121,20 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
+// Warning: (ae-missing-release-tag) "AwsS3ReadTreeProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class AwsS3ReadTreeProcessor implements CatalogProcessor {
+  constructor(reader: UrlReader);
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean>;
+}
+
 // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

From c2d7efb782fcc8a7e4cc4124427aa12c54399b17 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Fri, 13 Aug 2021 16:30:05 -0700
Subject: [PATCH 11/25] First go

Signed-off-by: Sean Tan <seant@splunk.com>
---
 packages/backend-common/src/reading/AwsS3UrlReader.ts | 4 ++++
 packages/backend-common/src/reading/types.ts          | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index 615a095e50..6c2d55972b 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -187,6 +187,10 @@ export class AwsS3UrlReader implements UrlReader {
         const { Contents, IsTruncated, NextContinuationToken } = await this.s3
           .listObjectsV2(params)
           .promise();
+<<<<<<< HEAD
+=======
+
+>>>>>>> First go
         const responses = await Promise.all(
           (Contents || []).map(({ Key }) => {
             const s3Response = this.s3
diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts
index 8d896408c3..448e678435 100644
--- a/packages/backend-common/src/reading/types.ts
+++ b/packages/backend-common/src/reading/types.ts
@@ -174,6 +174,13 @@ export type FromReadableArrayOptions = {
   etag: string;
 };
 
+export type FromReadableArrayOptions = {
+  // An array of readable streams
+  stream: Readable[];
+  // etag of the file tree
+  etag: string;
+};
+
 export interface ReadTreeResponseFactory {
   fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
   fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;

From 2cb8041f700d92e6d6de8ab44e8e6bcd6cbe8061 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Fri, 13 Aug 2021 17:24:14 -0700
Subject: [PATCH 12/25] Merge work done on previous branch Signed-off-by: Sean
 Tan <seant@splunk.com>

---
 app-config.yaml                         | 2 ++
 packages/backend/src/plugins/catalog.ts | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app-config.yaml b/app-config.yaml
index 9f365becef..fcfd92a66f 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -259,6 +259,8 @@ catalog:
     # Backstage example groups and users
     - type: file
       target: ../catalog-model/examples/acme-corp.yaml
+    - type: aws-read-tree
+      target: https://seant-splunk.s3.us-east-2.amazonaws.com
 
 scaffolder:
   # Use to customize default commit author info used when new components are created
diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 055595dcb5..9c94b3851e 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor';
 import {
   CatalogBuilder,
   createRouter,
@@ -25,6 +25,7 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   const builder = await CatalogBuilder.create(env);
+  builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader));
   const {
     entitiesCatalog,
     locationAnalyzer,

From 1297baca45647a322be21b8aa0d2642f64c4e395 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:28:27 -0700
Subject: [PATCH 13/25] Preparing for PR Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 packages/backend-common/src/reading/AwsS3UrlReader.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index 6c2d55972b..a512d7f64b 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -188,9 +188,12 @@ export class AwsS3UrlReader implements UrlReader {
           .listObjectsV2(params)
           .promise();
 <<<<<<< HEAD
+<<<<<<< HEAD
 =======
 
 >>>>>>> First go
+=======
+>>>>>>> Preparing for PR
         const responses = await Promise.all(
           (Contents || []).map(({ Key }) => {
             const s3Response = this.s3

From 689ab3204d2da1b8b66cbd29e0c8059afa043cb8 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:40:25 -0700
Subject: [PATCH 14/25] Add readTree() to AwsS3UrlReader and
 AwsS3ReadTreeProcessor Signed-off-by: Sean Tan <seant@splunk.com>

---
 app-config.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index fcfd92a66f..9f365becef 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -259,8 +259,6 @@ catalog:
     # Backstage example groups and users
     - type: file
       target: ../catalog-model/examples/acme-corp.yaml
-    - type: aws-read-tree
-      target: https://seant-splunk.s3.us-east-2.amazonaws.com
 
 scaffolder:
   # Use to customize default commit author info used when new components are created

From ee9e520c157c6fb0f23929f1f535363622ba0bc6 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:46:48 -0700
Subject: [PATCH 15/25] Remove local changes made to catalog.ts Signed-off-by:
 Sean Tan <seant@splunk.com>

---
 packages/backend/src/plugins/catalog.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 9c94b3851e..75709eefc7 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor';
 import {
   CatalogBuilder,
   createRouter,
@@ -25,7 +24,6 @@ export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   const builder = await CatalogBuilder.create(env);
-  builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader));
   const {
     entitiesCatalog,
     locationAnalyzer,

From 393ec8b3c6c35833fb4d6c85f510bf58c76ec843 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 14:49:11 -0700
Subject: [PATCH 16/25] Fix one more error in catalog.ts Signed-off-by: Sean
 Tan <seant@splunk.com>

Signed-off-by: Sean Tan <seant@splunk.com>
---
 packages/backend/src/plugins/catalog.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 75709eefc7..055595dcb5 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import {
   CatalogBuilder,
   createRouter,

From 8bde9ef1f12b47ecc4b11e0ee60f5013c026de0b Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 15:57:23 -0700
Subject: [PATCH 17/25] Duplicate entry Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 packages/backend-common/src/reading/types.ts | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts
index 448e678435..c0acc04701 100644
--- a/packages/backend-common/src/reading/types.ts
+++ b/packages/backend-common/src/reading/types.ts
@@ -173,14 +173,6 @@ export type FromReadableArrayOptions = {
   // etag of the file tree
   etag: string;
 };
-
-export type FromReadableArrayOptions = {
-  // An array of readable streams
-  stream: Readable[];
-  // etag of the file tree
-  etag: string;
-};
-
 export interface ReadTreeResponseFactory {
   fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
   fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;

From b992dacf4f110d4c9cf495b3f4a24ff53a175f5e Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 18 Aug 2021 16:00:31 -0700
Subject: [PATCH 18/25] Remove random file change Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 .../software-templates/migrating-from-v1alpha1-to-v1beta2.md    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md
index 28001cce0b..25ff2a2f3a 100644
--- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md
+++ b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md
@@ -325,7 +325,7 @@ spec:
   output:
     links:
       - url: '{{steps.publish.output.remoteUrl}}'
-        title: 'Go to Repo'
+        text: 'Go to Repo'
 ```
 
 ## Questions?

From 7f5ce313f82ed9ba6535deacee2224508cd87521 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Mon, 23 Aug 2021 12:57:42 -0700
Subject: [PATCH 19/25] Begin changes on PR Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 docs/integrations/aws-s3/readtree.md          |  4 +-
 packages/backend-common/src/index.ts          |  1 -
 .../src/reading/AwsS3UrlReader.ts             | 29 +++++++-------
 .../processors/AwsS3ReadTreeProcessor.test.ts | 39 +++++--------------
 .../processors/AwsS3ReadTreeProcessor.ts      |  2 +-
 5 files changed, 26 insertions(+), 49 deletions(-)

diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md
index 7ae1fece63..47ce867137 100644
--- a/docs/integrations/aws-s3/readtree.md
+++ b/docs/integrations/aws-s3/readtree.md
@@ -21,8 +21,8 @@ configuration:
 ```yaml
 catalog:
   locations:
-    - type: aws-read-tree
+    - type: s3-bucket
       target: https://sample-bucket.s3.us-east-2.amazonaws.com/
 ```
 
-Note the `aws-read-tree` type, as this is not a regular `url` processor.
+Note the `s3-bucket` type, as this is not a regular `url` processor.
diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts
index 5b37c2fb9f..f2f38d9aab 100644
--- a/packages/backend-common/src/index.ts
+++ b/packages/backend-common/src/index.ts
@@ -23,7 +23,6 @@ export * from './logging';
 export * from './middleware';
 export * from './paths';
 export * from './reading';
-export * from './reading/tree';
 export * from './scm';
 export * from './service';
 export * from './util';
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index a512d7f64b..593a66cfcc 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -72,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader {
         apiVersion: '2006-03-01',
         credentials: creds,
       });
-      const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory);
+      const reader = new AwsS3UrlReader(integration, {
+        s3,
+        treeResponseFactory,
+      });
       const predicate = (url: URL) =>
         url.host.endsWith(integration.config.host);
       return { reader, predicate };
@@ -81,8 +84,10 @@ export class AwsS3UrlReader implements UrlReader {
 
   constructor(
     private readonly integration: AwsS3Integration,
-    private readonly s3: S3,
-    private readonly treeResponseFactory: ReadTreeResponseFactory,
+    private readonly deps: {
+      s3: S3;
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
   ) {}
 
   /**
@@ -148,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader {
         };
       }
 
-      const response = this.s3.getObject(params);
+      const response = this.deps.s3.getObject(params);
       const buffer = await getRawBody(response.createReadStream());
       const etag = (await response.promise()).ETag;
 
@@ -184,19 +189,11 @@ export class AwsS3UrlReader implements UrlReader {
             ContinuationToken: continuationToken,
           };
         }
-        const { Contents, IsTruncated, NextContinuationToken } = await this.s3
-          .listObjectsV2(params)
-          .promise();
-<<<<<<< HEAD
-<<<<<<< HEAD
-=======
-
->>>>>>> First go
-=======
->>>>>>> Preparing for PR
+        const { Contents, IsTruncated, NextContinuationToken } =
+          await this.deps.s3.listObjectsV2(params).promise();
         const responses = await Promise.all(
           (Contents || []).map(({ Key }) => {
-            const s3Response = this.s3
+            const s3Response = this.deps.s3
               .getObject({ Bucket: bucket, Key: String(Key) })
               .createReadStream();
             Object.defineProperty(s3Response, 'path', {
@@ -216,7 +213,7 @@ export class AwsS3UrlReader implements UrlReader {
         awsS3Readables = awsS3Readables.concat(responses);
       }
 
-      return await this.treeResponseFactory.fromReadableArray({
+      return await this.deps.treeResponseFactory.fromReadableArray({
         stream: awsS3Readables,
         etag: '',
       });
diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
index 6b99506e26..0b8ea6f814 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
@@ -13,16 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import {
-  AwsS3UrlReader,
-  DefaultReadTreeResponseFactory,
-} from '@backstage/backend-common';
-
+import { getVoidLogger, UrlReaders } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
-import {
-  AwsS3Integration,
-  readAwsS3IntegrationConfig,
-} from '@backstage/integration';
 import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor';
 import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types';
 import { defaultEntityDataParser } from './util/parse';
@@ -30,10 +22,6 @@ import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
 
-const treeResponseFactory = DefaultReadTreeResponseFactory.create({
-  config: new ConfigReader({}),
-});
-
 AWSMock.setSDKInstance(aws);
 const object: aws.S3.Types.Object = {
   Key: 'awsS3-mock-object.yaml',
@@ -61,25 +49,18 @@ AWSMock.mock(
   ),
 );
 
-const s3 = new aws.S3();
-const reader = new AwsS3UrlReader(
-  new AwsS3Integration(
-    readAwsS3IntegrationConfig(
-      new ConfigReader({
-        host: 'amazonaws.com',
-        accessKeyId: 'fake-access-key',
-        secretAccessKey: 'fake-secret-key',
-      }),
-    ),
-  ),
-  s3,
-  treeResponseFactory,
-);
+const logger = getVoidLogger();
+const reader = UrlReaders.default({
+  logger,
+  config: new ConfigReader({
+    backend: { reading: { allow: [{ host: 'localhost' }] } },
+  }),
+});
 
 describe('readLocation', () => {
   const processor = new AwsS3ReadTreeProcessor(reader);
   const spec = {
-    type: 'aws-read-tree',
+    type: 's3-bucket',
     target: 'https://testbucket.s3.us-east-2.amazonaws.com',
   };
 
@@ -90,7 +71,7 @@ describe('readLocation', () => {
     expect(generated.type).toBe('entity');
     expect(generated.location).toEqual({
       target: 'awsS3-mock-object.yaml',
-      type: 'aws-read-tree',
+      type: 's3-bucket',
     });
     expect(generated.entity).toEqual({ site_name: 'Test' });
   });
diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts
index 8e65714a4c..9d8e1cdce5 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts
@@ -32,7 +32,7 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor {
     emit: CatalogProcessorEmit,
     parser: CatalogProcessorParser,
   ): Promise<boolean> {
-    if (location.type !== 'aws-read-tree') {
+    if (location.type !== 's3-bucket') {
       return false;
     }
 

From 4d4b302905b099576e0c14bdeb7cd408368f8dc8 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Tue, 24 Aug 2021 17:29:21 -0700
Subject: [PATCH 20/25] Finish ReadableArrayResponse changes Signed-off-by:
 Sean Tan <seant@splunk.com>

---
 .../src/reading/AwsS3UrlReader.test.ts        |  9 ++--
 .../reading/tree/ReadTreeResponseFactory.ts   |  6 ++-
 .../tree/ReadableArrayResponse.test.ts        | 34 +++++++++---
 .../src/reading/tree/ReadableArrayResponse.ts | 54 ++++++++++++++++---
 4 files changed, 83 insertions(+), 20 deletions(-)

diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
index 9be6de7a30..736248b10b 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts
@@ -156,8 +156,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
-      treeResponseFactory,
+      { s3, treeResponseFactory },
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -211,8 +210,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
-      treeResponseFactory,
+      { s3, treeResponseFactory },
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -273,8 +271,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
-      treeResponseFactory,
+      { s3, treeResponseFactory },
     );
     it('returns contents of an object in a bucket', async () => {
       const response = await awsS3UrlReader.readTree(
diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
index 19d7ed2fab..7d20385635 100644
--- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
+++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts
@@ -59,6 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
   async fromReadableArray(
     options: FromReadableArrayOptions,
   ): Promise<ReadTreeResponse> {
-    return new ReadableArrayResponse(options.stream, options.etag);
+    return new ReadableArrayResponse(
+      options.stream,
+      this.workDir,
+      options.etag,
+    );
   }
 }
diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts
index c2e3c26269..bf457f02e4 100644
--- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts
+++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts
@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 import { ReadableArrayResponse } from './ReadableArrayResponse';
-import path from 'path';
+import path, { resolve as resolvePath } from 'path';
+
 import { Readable } from 'stream';
-import fs from 'fs';
+import fs from 'fs-extra';
 
 const arr: Readable[] = [];
+const arr2: Readable[] = [];
 const file1 = path.resolve(
   'src',
   'reading',
@@ -33,14 +35,15 @@ const file2 = path.resolve(
   'awsS3',
   'awsS3-mock-object2.yaml',
 );
-const stream1 = fs.createReadStream(file1);
-const stream2 = fs.createReadStream(file2);
-arr.push(stream1);
-arr.push(stream2);
 
 describe('ReadableArrayResponse', () => {
   it('should read files', async () => {
-    const res = new ReadableArrayResponse(arr, 'etag');
+    const stream1 = fs.createReadStream(file1);
+    const stream2 = fs.createReadStream(file2);
+    arr.push(stream1);
+    arr.push(stream2);
+
+    const res = new ReadableArrayResponse(arr, '/tmp', 'etag');
     const files = await res.files();
 
     expect(files).toEqual([
@@ -59,4 +62,21 @@ describe('ReadableArrayResponse', () => {
       'site_name: Test2',
     ]);
   });
+
+  it('should extract entire archive into directory', async () => {
+    const stream1 = fs.createReadStream(file1);
+    const stream2 = fs.createReadStream(file2);
+
+    arr2.push(stream1);
+    arr2.push(stream2);
+
+    const res = new ReadableArrayResponse(arr2, '/tmp', 'etag');
+    const dir = await res.dir();
+    await expect(
+      fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'),
+    ).resolves.toBe('site_name: Test\n');
+    await expect(
+      fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'),
+    ).resolves.toBe('site_name: Test2\n');
+  });
 });
diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts
index 8b5d3ff0b5..bf74a19603 100644
--- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts
+++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts
@@ -14,9 +14,21 @@
  * limitations under the License.
  */
 
+import concatStream from 'concat-stream';
+import platformPath, { basename } from 'path';
+
 import getRawBody from 'raw-body';
-import { Readable } from 'stream';
-import { ReadTreeResponse, ReadTreeResponseFile } from '../types';
+import fs from 'fs-extra';
+import { promisify } from 'util';
+import tar from 'tar';
+import { pipeline as pipelineCb, Readable } from 'stream';
+import {
+  ReadTreeResponse,
+  ReadTreeResponseFile,
+  ReadTreeResponseDirOptions,
+} from '../types';
+
+const pipeline = promisify(pipelineCb);
 
 /**
  * Wraps a array of Readable objects into a tree response reader.
@@ -26,6 +38,7 @@ export class ReadableArrayResponse implements ReadTreeResponse {
 
   constructor(
     private readonly stream: Readable[],
+    private readonly workDir: string,
     public readonly etag: string,
   ) {
     this.etag = etag;
@@ -56,11 +69,40 @@ export class ReadableArrayResponse implements ReadTreeResponse {
     return files;
   }
 
-  archive(): Promise<NodeJS.ReadableStream> {
-    throw new Error('Method not implemented.');
+  async archive(): Promise<NodeJS.ReadableStream> {
+    const tmpDir = await this.dir();
+
+    try {
+      const data = await new Promise<Buffer>(async resolve => {
+        await pipeline(
+          tar.create({ cwd: tmpDir }, ['']),
+          concatStream(resolve),
+        );
+      });
+      return Readable.from(data);
+    } finally {
+      await fs.remove(tmpDir);
+    }
   }
 
-  dir(): Promise<string> {
-    throw new Error('Method not implemented.');
+  async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
+    this.onlyOnce();
+
+    const dir =
+      options?.targetDir ??
+      (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
+
+    for (let i = 0; i < this.stream.length; i++) {
+      if (!(this.stream[i] as any).path.endsWith('/')) {
+        await pipeline(
+          this.stream[i],
+          fs.createWriteStream(
+            platformPath.join(dir, basename((this.stream[i] as any).path)),
+          ),
+        );
+      }
+    }
+
+    return dir;
   }
 }

From 160a50029faa706d4e2dbfd77f25f11dbbc71dcb Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Tue, 24 Aug 2021 18:39:08 -0700
Subject: [PATCH 21/25] Finish up changes Signed-off-by: Sean Tan
 <seant@splunk.com>

---
 packages/backend-common/api-report.md         | 29 ++++---------------
 .../src/reading/AwsS3UrlReader.ts             |  2 +-
 2 files changed, 6 insertions(+), 25 deletions(-)

diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md
index adbc32f67c..ac35fa698a 100644
--- a/packages/backend-common/api-report.md
+++ b/packages/backend-common/api-report.md
@@ -38,11 +38,12 @@ import { Writable } from 'stream';
 //
 // @public (undocumented)
 export class AwsS3UrlReader implements UrlReader {
-  // Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts
   constructor(
     integration: AwsS3Integration,
-    s3: S3,
-    treeResponseFactory: ReadTreeResponseFactory,
+    deps: {
+      s3: S3;
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
   );
   // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts
   //
@@ -199,27 +200,6 @@ export class DatabaseManager {
   static fromConfig(config: Config): DatabaseManager;
 }
 
-// Warning: (ae-missing-release-tag) "DefaultReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
-export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
-  constructor(workDir: string);
-  // (undocumented)
-  static create(options: { config: Config }): DefaultReadTreeResponseFactory;
-  // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts
-  //
-  // (undocumented)
-  fromReadableArray(
-    options: FromReadableArrayOptions,
-  ): Promise<ReadTreeResponse>;
-  // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts
-  //
-  // (undocumented)
-  fromTarArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
-  // (undocumented)
-  fromZipArchive(options: FromArchiveOptions): Promise<ReadTreeResponse>;
-}
-
 // Warning: (ae-missing-release-tag) "DockerContainerRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -639,6 +619,7 @@ export function useHotMemoize<T>(_module: NodeModule, valueFactory: () => T): T;
 //
 // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts
 // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here
+// src/reading/AwsS3UrlReader.d.ts:11:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts
 // src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts
 // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
 // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts
index 593a66cfcc..c48d16cc69 100644
--- a/packages/backend-common/src/reading/AwsS3UrlReader.ts
+++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts
@@ -169,7 +169,6 @@ export class AwsS3UrlReader implements UrlReader {
   async readTree(url: string): Promise<ReadTreeResponse> {
     try {
       const { path, bucket, region } = parseURL(url);
-      aws.config.update({ region: region });
 
       let moreKeys = true;
       let awsS3Readables: Readable[] = [];
@@ -189,6 +188,7 @@ export class AwsS3UrlReader implements UrlReader {
             ContinuationToken: continuationToken,
           };
         }
+        aws.config.update({ region: region });
         const { Contents, IsTruncated, NextContinuationToken } =
           await this.deps.s3.listObjectsV2(params).promise();
         const responses = await Promise.all(

From 5b1aebca114089b5b857190dc156b032a1ceb7bf Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Tue, 24 Aug 2021 19:34:48 -0700
Subject: [PATCH 22/25] Change FileReaderProcessor tests to reflect new test
 file 'aws-s3-mock-object.yaml' Signed-off-by: Sean Tan <seant@splunk.com>

---
 .../src/ingestion/processors/FileReaderProcessor.test.ts        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts
index 41c7e4f732..c1dfbb0af7 100644
--- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts
@@ -77,7 +77,7 @@ describe('FileReaderProcessor', () => {
       emit,
     );
 
-    expect(emit).toBeCalledTimes(2);
+    expect(emit).toBeCalledTimes(3);
     expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' });
     expect(emit.mock.calls[0][0].location).toEqual({
       type: 'file',

From 9bbe5dd98cae931fb68793043fef940c8d41b21f Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Wed, 25 Aug 2021 10:48:27 -0700
Subject: [PATCH 23/25] Change AwsS3ReadTreeProcessor tests as to not interfere
 with FileReaderProcessor tests Signed-off-by: Sean Tan <seant@splunk.com>

---
 .../src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts | 6 +++---
 .../src/ingestion/processors/FileReaderProcessor.test.ts    | 2 +-
 .../awsS3/{awsS3-mock-object.yaml => awsS3-mock-object.txt} | 0
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/{awsS3-mock-object.yaml => awsS3-mock-object.txt} (100%)

diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
index 0b8ea6f814..ebc258a29e 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts
@@ -24,7 +24,7 @@ import path from 'path';
 
 AWSMock.setSDKInstance(aws);
 const object: aws.S3.Types.Object = {
-  Key: 'awsS3-mock-object.yaml',
+  Key: 'awsS3-mock-object.txt',
 };
 const objectList: aws.S3.ObjectList = [object];
 const output: aws.S3.Types.ListObjectsV2Output = {
@@ -43,7 +43,7 @@ AWSMock.mock(
         '__fixtures__',
         'fileReaderProcessor',
         'awsS3',
-        'awsS3-mock-object.yaml',
+        'awsS3-mock-object.txt',
       ),
     ),
   ),
@@ -70,7 +70,7 @@ describe('readLocation', () => {
     )) as CatalogProcessorEntityResult;
     expect(generated.type).toBe('entity');
     expect(generated.location).toEqual({
-      target: 'awsS3-mock-object.yaml',
+      target: 'awsS3-mock-object.txt',
       type: 's3-bucket',
     });
     expect(generated.entity).toEqual({ site_name: 'Test' });
diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts
index c1dfbb0af7..41c7e4f732 100644
--- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts
@@ -77,7 +77,7 @@ describe('FileReaderProcessor', () => {
       emit,
     );
 
-    expect(emit).toBeCalledTimes(3);
+    expect(emit).toBeCalledTimes(2);
     expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' });
     expect(emit.mock.calls[0][0].location).toEqual({
       type: 'file',
diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt
similarity index 100%
rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml
rename to plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt

From c63ba3eff0b03fe804718bc5eab28145306eebe7 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Thu, 26 Aug 2021 10:31:09 -0700
Subject: [PATCH 24/25] Resolve conflict with package.json Signed-off-by: Sean
 Tan <seant@splunk.com>

---
 plugins/catalog-backend/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json
index e490214385..d8fabc8d7a 100644
--- a/plugins/catalog-backend/package.json
+++ b/plugins/catalog-backend/package.json
@@ -62,8 +62,8 @@
   },
   "devDependencies": {
     "aws-sdk-mock": "^5.2.1",
-    "@backstage/backend-test-utils": "^0.1.5",
-    "@backstage/cli": "^0.7.8",
+    "@backstage/backend-test-utils": "^0.1.6",
+    "@backstage/cli": "^0.7.9",
     "@backstage/test-utils": "^0.1.17",
     "@types/core-js": "^2.5.4",
     "@types/git-url-parse": "^9.0.0",

From bd119f21cc075b9d09eb0470566eee996364fbb0 Mon Sep 17 00:00:00 2001
From: Sean Tan <seant@splunk.com>
Date: Thu, 26 Aug 2021 10:38:09 -0700
Subject: [PATCH 25/25] Copy package.json from master and add 'aws-sdk-mock' to
 devDependencies Signed-off-by: Sean Tan <seant@splunk.com>

---
 plugins/catalog-backend/package.json | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json
index d8fabc8d7a..34c9caccbc 100644
--- a/plugins/catalog-backend/package.json
+++ b/plugins/catalog-backend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "0.13.2",
+  "version": "0.13.5",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
@@ -29,17 +29,18 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
-    "@backstage/backend-common": "^0.8.9",
+    "@backstage/backend-common": "^0.9.0",
     "@backstage/catalog-client": "^0.3.18",
     "@backstage/catalog-model": "^0.9.0",
-    "@backstage/config": "^0.1.5",
+    "@backstage/config": "^0.1.8",
     "@backstage/errors": "^0.1.1",
-    "@backstage/integration": "^0.6.0",
+    "@backstage/integration": "^0.6.2",
     "@backstage/plugin-search-backend-node": "^0.4.0",
     "@backstage/search-common": "^0.1.2",
     "@octokit/graphql": "^4.5.8",
     "@types/express": "^4.17.6",
     "aws-sdk": "^2.840.0",
+    "aws-sdk-mock": "^5.2.1",
     "codeowners-utils": "^1.0.2",
     "core-js": "^3.6.5",
     "cross-fetch": "^3.0.6",
@@ -51,8 +52,10 @@
     "glob": "^7.1.6",
     "knex": "^0.95.1",
     "lodash": "^4.17.15",
+    "luxon": "^2.0.2",
     "morgan": "^1.10.0",
     "p-limit": "^3.0.2",
+    "prom-client": "^13.2.0",
     "qs": "^6.9.4",
     "uuid": "^8.0.0",
     "winston": "^3.2.1",
@@ -61,7 +64,6 @@
     "yup": "^0.29.3"
   },
   "devDependencies": {
-    "aws-sdk-mock": "^5.2.1",
     "@backstage/backend-test-utils": "^0.1.6",
     "@backstage/cli": "^0.7.9",
     "@backstage/test-utils": "^0.1.17",