Merge pull request #6882 from splunk/seant-splunk/awsS3_readTree_processor

Add readTree() to AwsS3UrlReader, ReadableArrayResponse type, and AwsS3ReadTreeProcessor

.changeset/curvy-books-help.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-common': patch
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add AWS S3 Discovery Processor. Add readTree() to AwsS3UrlReader. Add ReadableArrayResponse type that implements ReadTreeResponse to use in AwsS3UrlReader's readTree()

docs/integrations/aws-s3/discovery.md

@@ -0,0 +1,28 @@
+---
+id: discovery
+title: AWS S3 Discovery
+sidebar_label: Discovery
+# prettier-ignore
+description: Automatically discovering catalog entities from an AWS S3 Bucket
+---
+
+The AWS S3 integration has a special discovery processor for discovering catalog
+entities located in an S3 Bucket. If you have a bucket that contains multiple
+catalog-info files and want to automatically discover them, you can use this
+processor. The processor will crawl your S3 bucket and register entities
+matching the configured path. This can be useful as an alternative to static
+locations or manually adding things to the catalog.
+
+To use the discovery processor, you'll need an AWS S3 integration
+[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and
+optionally a `roleArn`. Then you can add a location target to the catalog
+configuration:
+
+```yaml
+catalog:
+  locations:
+    - type: s3-discovery
+      target: https://sample-bucket.s3.us-east-2.amazonaws.com/
+```
+
+Note the `s3-discovery` type, as this is not a regular `url` processor.

microsite/sidebars.json

@@ -109,7 +109,10 @@
       {
         "type": "subcategory",
         "label": "AWS S3",
-        "ids": ["integrations/aws-s3/locations"]
+        "ids": [
+          "integrations/aws-s3/locations",
+          "integrations/aws-s3/discovery"
+        ]
       },
       {
         "type": "subcategory",

mkdocs.yml

@@ -82,6 +82,7 @@ nav:
       - Overview: 'integrations/index.md'
       - AWS S3:
           - Locations: 'integrations/aws-s3/locations.md'
+          - Discovery: 'integrations/aws-s3/discovery.md'
       - Azure:
           - Locations: 'integrations/azure/locations.md'
           - Org Data: 'integrations/azure/org.md'

packages/backend-common/api-report.md

@@ -6,6 +6,7 @@
 /// <reference types="node" />
 /// <reference types="webpack-env" />
 
+import { AwsS3Integration } from '@backstage/integration';
 import { AzureIntegration } from '@backstage/integration';
 import { BitbucketIntegration } from '@backstage/integration';
 import { Config } from '@backstage/config';
@@ -26,10 +27,36 @@ import { Readable } from 'stream';
 import { ReadCommitResult } from 'isomorphic-git';
 import { RequestHandler } from 'express';
 import { Router } from 'express';
+import { S3 } from 'aws-sdk';
 import { Server } from 'http';
 import * as winston from 'winston';
 import { Writable } from 'stream';
 
+// Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class AwsS3UrlReader implements UrlReader {
+  constructor(
+    integration: AwsS3Integration,
+    deps: {
+      s3: S3;
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
+  );
+  // (undocumented)
+  static factory: ReaderFactory;
+  // (undocumented)
+  read(url: string): Promise<Buffer>;
+  // (undocumented)
+  readTree(url: string): Promise<ReadTreeResponse>;
+  // (undocumented)
+  readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+  // (undocumented)
+  search(): Promise<SearchResponse>;
+  // (undocumented)
+  toString(): string;
+}
+
 // @public (undocumented)
 export class AzureUrlReader implements UrlReader {
   constructor(
@@ -399,6 +426,12 @@ export type ReadTreeResponseDirOptions = {
 
 // @public (undocumented)
 export interface ReadTreeResponseFactory {
+  // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse>;
   // (undocumented)
   fromTarArchive(
     options: ReadTreeResponseFactoryOptions,

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
 
+const treeResponseFactory = DefaultReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});
+
 describe('AwsS3UrlReader', () => {
   const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
     return AwsS3UrlReader.factory({
       config: new ConfigReader(config),
       logger: getVoidLogger(),
-      treeResponseFactory: DefaultReadTreeResponseFactory.create({
-        config: new ConfigReader({}),
-      }),
+      treeResponseFactory,
     });
   };
 
@@ -137,6 +139,7 @@ describe('AwsS3UrlReader', () => {
             'src',
             'reading',
             '__fixtures__',
+            'awsS3',
             'awsS3-mock-object.yaml',
           ),
         ),
@@ -153,7 +156,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
+      { s3, treeResponseFactory },
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -188,6 +191,7 @@ describe('AwsS3UrlReader', () => {
             'src',
             'reading',
             '__fixtures__',
+            'awsS3',
             'awsS3-mock-object.yaml',
           ),
         ),
@@ -206,7 +210,7 @@ describe('AwsS3UrlReader', () => {
           }),
         ),
       ),
-      s3,
+      { s3, treeResponseFactory },
     );
 
     it('returns contents of an object in a bucket', async () => {
@@ -229,4 +233,54 @@ describe('AwsS3UrlReader', () => {
       );
     });
   });
+  describe('readTree', () => {
+    const object: aws.S3.Types.Object = {
+      Key: 'awsS3-mock-object.yaml',
+    };
+    const objectList: aws.S3.ObjectList = [object];
+    const output: aws.S3.Types.ListObjectsV2Output = {
+      Contents: objectList,
+    };
+    AWSMock.setSDKInstance(aws);
+    AWSMock.mock('S3', 'listObjectsV2', output);
+
+    AWSMock.mock(
+      'S3',
+      'getObject',
+      Buffer.from(
+        require('fs').readFileSync(
+          path.resolve(
+            'src',
+            'reading',
+            '__fixtures__',
+            'awsS3',
+            'awsS3-mock-object.yaml',
+          ),
+        ),
+      ),
+    );
+
+    const s3 = new aws.S3();
+    const awsS3UrlReader = new AwsS3UrlReader(
+      new AwsS3Integration(
+        readAwsS3IntegrationConfig(
+          new ConfigReader({
+            host: '.amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          }),
+        ),
+      ),
+      { s3, treeResponseFactory },
+    );
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.readTree(
+        'https://test.s3.us-east-2.amazonaws.com',
+      );
+      const files = await response.files();
+      const body = await files[0].content();
+
+      expect(body.toString()).toBe('site_name: Test\n');
+    });
+  });
 });

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadTreeResponseFactory,
   ReadUrlOptions,
   ReadUrlResponse,
   SearchResponse,
@@ -26,6 +27,7 @@ import {
 } from './types';
 import getRawBody from 'raw-body';
 import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
 
 const parseURL = (
   url: string,
@@ -61,7 +63,7 @@ const parseURL = (
 };
 
 export class AwsS3UrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);
 
     return integrations.awsS3.list().map(integration => {
@@ -70,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader {
         apiVersion: '2006-03-01',
         credentials: creds,
       });
-      const reader = new AwsS3UrlReader(integration, s3);
+      const reader = new AwsS3UrlReader(integration, {
+        s3,
+        treeResponseFactory,
+      });
       const predicate = (url: URL) =>
         url.host.endsWith(integration.config.host);
       return { reader, predicate };
@@ -79,7 +84,10 @@ export class AwsS3UrlReader implements UrlReader {
 
   constructor(
     private readonly integration: AwsS3Integration,
-    private readonly s3: S3,
+    private readonly deps: {
+      s3: S3;
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
   ) {}
 
   /**
@@ -145,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader {
         };
       }
 
-      const response = this.s3.getObject(params);
+      const response = this.deps.s3.getObject(params);
       const buffer = await getRawBody(response.createReadStream());
       const etag = (await response.promise()).ETag;
 
@@ -158,8 +166,45 @@ export class AwsS3UrlReader implements UrlReader {
     }
   }
 
-  async readTree(): Promise<ReadTreeResponse> {
-    throw new Error('AwsS3Reader does not implement readTree');
+  async readTree(url: string): Promise<ReadTreeResponse> {
+    try {
+      const { path, bucket, region } = parseURL(url);
+      const allObjects: ObjectList = [];
+      const responses = [];
+      let continuationToken: string | undefined;
+      let output: ListObjectsV2Output;
+      do {
+        aws.config.update({ region: region });
+        output = await this.deps.s3
+          .listObjectsV2({
+            Bucket: bucket,
+            ContinuationToken: continuationToken,
+            Prefix: path,
+          })
+          .promise();
+        if (output.Contents) {
+          output.Contents.forEach(contents => {
+            allObjects.push(contents);
+          });
+        }
+        continuationToken = output.NextContinuationToken;
+      } while (continuationToken);
+
+      for (let i = 0; i < allObjects.length; i++) {
+        const object = this.deps.s3.getObject({
+          Bucket: bucket,
+          Key: String(allObjects[i].Key),
+        });
+        responses.push({
+          data: object.createReadStream(),
+          path: String(allObjects[i].Key),
+        });
+      }
+
+      return await this.deps.treeResponseFactory.fromReadableArray(responses);
+    } catch (e) {
+      throw new Error(`Could not retrieve file tree from S3: ${e.message}`);
+    }
   }
 
   async search(): Promise<SearchResponse> {

packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml

@@ -0,0 +1 @@
+site_name: Test2

packages/backend-common/src/reading/index.ts

@@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader';
 export { BitbucketUrlReader } from './BitbucketUrlReader';
 export { GithubUrlReader } from './GithubUrlReader';
 export { GitlabUrlReader } from './GitlabUrlReader';
+export { AwsS3UrlReader } from './AwsS3UrlReader';
 export type {
   ReaderFactory,
   ReadTreeOptions,

packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts

@@ -20,9 +20,11 @@ import {
   ReadTreeResponse,
   ReadTreeResponseFactoryOptions,
   ReadTreeResponseFactory,
+  FromReadableArrayOptions,
 } from '../types';
 import { TarArchiveResponse } from './TarArchiveResponse';
 import { ZipArchiveResponse } from './ZipArchiveResponse';
+import { ReadableArrayResponse } from './ReadableArrayResponse';
 
 export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
   static create(options: { config: Config }): DefaultReadTreeResponseFactory {
@@ -57,4 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory {
       options.filter,
     );
   }
+
+  async fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse> {
+    return new ReadableArrayResponse(options, this.workDir, '');
+  }
 }

packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ReadableArrayResponse } from './ReadableArrayResponse';
+import path, { resolve as resolvePath } from 'path';
+import fs from 'fs-extra';
+import { FromReadableArrayOptions } from '../types';
+
+const arr: FromReadableArrayOptions = [];
+const arr2: FromReadableArrayOptions = [];
+const path1 = path.resolve(
+  'src',
+  'reading',
+  '__fixtures__',
+  'awsS3',
+  'awsS3-mock-object.yaml',
+);
+const path2 = path.resolve(
+  'src',
+  'reading',
+  '__fixtures__',
+  'awsS3',
+  'awsS3-mock-object2.yaml',
+);
+
+describe('ReadableArrayResponse', () => {
+  it('should read files', async () => {
+    const stream1 = fs.createReadStream(path1);
+    const stream2 = fs.createReadStream(path2);
+    arr.push({ data: stream1, path: path1 });
+    arr.push({ data: stream2, path: path2 });
+
+    const res = new ReadableArrayResponse(arr, '/tmp', 'etag');
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: path1,
+        content: expect.any(Function),
+      },
+      {
+        path: path2,
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      'site_name: Test',
+      'site_name: Test2',
+    ]);
+  });
+
+  it('should extract entire archive into directory', async () => {
+    const stream1 = fs.createReadStream(path1);
+    const stream2 = fs.createReadStream(path2);
+
+    arr2.push({ data: stream1, path: path1 });
+    arr2.push({ data: stream2, path: path2 });
+
+    const res = new ReadableArrayResponse(arr2, '/tmp', 'etag');
+    const dir = await res.dir();
+    await expect(
+      fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'),
+    ).resolves.toBe('site_name: Test\n');
+    await expect(
+      fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'),
+    ).resolves.toBe('site_name: Test2\n');
+  });
+});

packages/backend-common/src/reading/tree/ReadableArrayResponse.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import concatStream from 'concat-stream';
+import platformPath, { basename } from 'path';
+
+import getRawBody from 'raw-body';
+import fs from 'fs-extra';
+import { promisify } from 'util';
+import tar from 'tar';
+import { pipeline as pipelineCb, Readable } from 'stream';
+import {
+  ReadTreeResponse,
+  ReadTreeResponseFile,
+  ReadTreeResponseDirOptions,
+  FromReadableArrayOptions,
+} from '../types';
+
+const pipeline = promisify(pipelineCb);
+
+/**
+ * Wraps a array of Readable objects into a tree response reader.
+ */
+export class ReadableArrayResponse implements ReadTreeResponse {
+  private read = false;
+
+  constructor(
+    private readonly stream: FromReadableArrayOptions,
+    private readonly workDir: string,
+    public readonly etag: string,
+  ) {
+    this.etag = etag;
+  }
+
+  // Make sure the input stream is only read once
+  private onlyOnce() {
+    if (this.read) {
+      throw new Error('Response has already been read');
+    }
+    this.read = true;
+  }
+
+  async files(): Promise<ReadTreeResponseFile[]> {
+    this.onlyOnce();
+
+    const files = Array<ReadTreeResponseFile>();
+
+    for (let i = 0; i < this.stream.length; i++) {
+      if (!this.stream[i].path.endsWith('/')) {
+        files.push({
+          path: this.stream[i].path,
+          content: () => getRawBody(this.stream[i].data),
+        });
+      }
+    }
+
+    return files;
+  }
+
+  async archive(): Promise<NodeJS.ReadableStream> {
+    const tmpDir = await this.dir();
+
+    try {
+      const data = await new Promise<Buffer>(async resolve => {
+        await pipeline(
+          tar.create({ cwd: tmpDir }, ['']),
+          concatStream(resolve),
+        );
+      });
+      return Readable.from(data);
+    } finally {
+      await fs.remove(tmpDir);
+    }
+  }
+
+  async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
+    this.onlyOnce();
+
+    const dir =
+      options?.targetDir ??
+      (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
+
+    for (let i = 0; i < this.stream.length; i++) {
+      if (!this.stream[i].path.endsWith('/')) {
+        await pipeline(
+          this.stream[i].data,
+          fs.createWriteStream(
+            platformPath.join(dir, basename(this.stream[i].path)),
+          ),
+        );
+      }
+    }
+
+    return dir;
+  }
+}

packages/backend-common/src/reading/types.ts

@@ -183,6 +183,13 @@ export type ReadTreeResponseFactoryOptions = {
   // Filter passed on from the ReadTreeOptions
   filter?: (path: string, info?: { size: number }) => boolean;
 };
+/** @public */
+export type FromReadableArrayOptions = Array<{
+  // Data in the form of a readable
+  data: Readable;
+  // A string containing the filepath of the data
+  path: string;
+}>;
 
 /** @public */
 export interface ReadTreeResponseFactory {
@@ -192,6 +199,9 @@ export interface ReadTreeResponseFactory {
   fromZipArchive(
     options: ReadTreeResponseFactoryOptions,
   ): Promise<ReadTreeResponse>;
+  fromReadableArray(
+    options: FromReadableArrayOptions,
+  ): Promise<ReadTreeResponse>;
 }
 
 /**

plugins/catalog-backend/api-report.md

@@ -158,6 +158,20 @@ export type AwsOrganizationProviderConfig = {
   roleArn?: string;
 };
 
+// Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class AwsS3DiscoveryProcessor implements CatalogProcessor {
+  constructor(reader: UrlReader);
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean>;
+}
+
 // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/catalog-backend/package.json

@@ -73,6 +73,7 @@
     "@types/supertest": "^2.0.8",
     "@types/uuid": "^8.0.0",
     "@types/yup": "^0.29.8",
+    "aws-sdk-mock": "^5.2.1",
     "msw": "^0.29.0",
     "sqlite3": "^5.0.1",
     "supertest": "^6.1.3",

plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getVoidLogger, UrlReaders } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
+import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types';
+import { defaultEntityDataParser } from './util/parse';
+import AWSMock from 'aws-sdk-mock';
+import aws from 'aws-sdk';
+import path from 'path';
+
+AWSMock.setSDKInstance(aws);
+const object: aws.S3.Types.Object = {
+  Key: 'awsS3-mock-object.txt',
+};
+const objectList: aws.S3.ObjectList = [object];
+const output: aws.S3.Types.ListObjectsV2Output = {
+  Contents: objectList,
+};
+AWSMock.mock('S3', 'listObjectsV2', output);
+AWSMock.mock(
+  'S3',
+  'getObject',
+  Buffer.from(
+    require('fs').readFileSync(
+      path.resolve(
+        'src',
+        'ingestion',
+        'processors',
+        '__fixtures__',
+        'fileReaderProcessor',
+        'awsS3',
+        'awsS3-mock-object.txt',
+      ),
+    ),
+  ),
+);
+
+const logger = getVoidLogger();
+const reader = UrlReaders.default({
+  logger,
+  config: new ConfigReader({
+    backend: { reading: { allow: [{ host: 'localhost' }] } },
+  }),
+});
+
+describe('readLocation', () => {
+  const processor = new AwsS3DiscoveryProcessor(reader);
+  const spec = {
+    type: 's3-discovery',
+    target: 'https://testbucket.s3.us-east-2.amazonaws.com',
+  };
+
+  it('should load from url', async () => {
+    const generated = (await new Promise<CatalogProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit, defaultEntityDataParser),
+    )) as CatalogProcessorEntityResult;
+    expect(generated.type).toBe('entity');
+    expect(generated.location).toEqual({
+      target: 'awsS3-mock-object.txt',
+      type: 's3-discovery',
+    });
+    expect(generated.entity).toEqual({ site_name: 'Test' });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import limiterFactory from 'p-limit';
+import * as result from './results';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorParser,
+} from './types';
+
+export class AwsS3DiscoveryProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean> {
+    if (location.type !== 's3-discovery') {
+      return false;
+    }
+
+    try {
+      const output = await this.doRead(location.target);
+      for (const item of output) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url },
+        })) {
+          emit(parseResult);
+        }
+      }
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`;
+
+      if (error.name === 'NotFoundError') {
+        if (!optional) {
+          emit(result.notFoundError(location, message));
+        }
+      } else {
+        emit(result.generalError(location, message));
+      }
+    }
+    return true;
+  }
+
+  private async doRead(
+    location: string,
+  ): Promise<{ data: Buffer; url: string }[]> {
+    const limiter = limiterFactory(5);
+    const response = await this.reader.readTree(location);
+    const responseFiles = await response.files();
+    const output = responseFiles.map(async file => ({
+      url: file.path,
+      data: await limiter(file.content),
+    }));
+    return Promise.all(output);
+  }
+}

plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt

@@ -0,0 +1 @@
+site_name: Test

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -20,6 +20,7 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess
 export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
 export type { AwsOrganizationProviderConfig } from './awsOrganization/config';
+export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
 export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';