diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index da9e77d9ca..f7904ca70f 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import { ReaderFactory, ReadTreeResponse, + ReadTreeResponseFactory, ReadUrlOptions, ReadUrlResponse, SearchResponse, @@ -62,7 +63,7 @@ const parseURL = ( }; export class AwsS3UrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); return integrations.awsS3.list().map(integration => { @@ -71,7 +72,7 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3); + const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -81,6 +82,7 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, private readonly s3: S3, + private readonly treeResponseFactory: ReadTreeResponseFactory, ) {} /** diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts new file mode 100644 index 0000000000..8e65714a4c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import * as result from './results'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; + +export class AwsS3ReadTreeProcessor implements CatalogProcessor { + constructor(private readonly reader: UrlReader) {} + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise { + if (location.type !== 'aws-read-tree') { + return false; + } + + try { + const output = await this.doRead(location.target); + for (const item of output) { + for await (const parseResult of parser({ + data: item.data, + location: { type: location.type, target: item.url }, + })) { + emit(parseResult); + } + } + } catch (error) { + const message = `Unable to read ${location.type}, ${error}`; + + if (error.name === 'NotFoundError') { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + return true; + } + + private async doRead( + location: string, + ): Promise<{ data: Buffer; url: string }[]> { + const response = await this.reader.readTree(location); + const responseFiles = await response.files(); + const output = responseFiles.map(async file => ({ + url: file.path, + data: await file.content(), + })); + return Promise.all(output); + } +}