More work

Signed-off-by: Sean Tan <seant@splunk.com>

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadTreeResponseFactory,
   ReadUrlOptions,
   ReadUrlResponse,
   SearchResponse,
@@ -62,7 +63,7 @@ const parseURL = (
 };
 
 export class AwsS3UrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);
 
     return integrations.awsS3.list().map(integration => {
@@ -71,7 +72,7 @@ export class AwsS3UrlReader implements UrlReader {
         apiVersion: '2006-03-01',
         credentials: creds,
       });
-      const reader = new AwsS3UrlReader(integration, s3);
+      const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory);
       const predicate = (url: URL) =>
         url.host.endsWith(integration.config.host);
       return { reader, predicate };
@@ -81,6 +82,7 @@ export class AwsS3UrlReader implements UrlReader {
   constructor(
     private readonly integration: AwsS3Integration,
     private readonly s3: S3,
+    private readonly treeResponseFactory: ReadTreeResponseFactory,
   ) {}
 
   /**

plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import * as result from './results';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorParser,
+} from './types';
+
+export class AwsS3ReadTreeProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
+  async readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean> {
+    if (location.type !== 'aws-read-tree') {
+      return false;
+    }
+
+    try {
+      const output = await this.doRead(location.target);
+      for (const item of output) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url },
+        })) {
+          emit(parseResult);
+        }
+      }
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`;
+
+      if (error.name === 'NotFoundError') {
+        if (!optional) {
+          emit(result.notFoundError(location, message));
+        }
+      } else {
+        emit(result.generalError(location, message));
+      }
+    }
+    return true;
+  }
+
+  private async doRead(
+    location: string,
+  ): Promise<{ data: Buffer; url: string }[]> {
+    const response = await this.reader.readTree(location);
+    const responseFiles = await response.files();
+    const output = responseFiles.map(async file => ({
+      url: file.path,
+      data: await file.content(),
+    }));
+    return Promise.all(output);
+  }
+}