diff --git a/.changeset/curvy-books-help.md b/.changeset/curvy-books-help.md new file mode 100644 index 0000000000..de450b12f0 --- /dev/null +++ b/.changeset/curvy-books-help.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-catalog-backend': patch +--- + +Add AWS S3 Discovery Processor. Add readTree() to AwsS3UrlReader. Add ReadableArrayResponse type that implements ReadTreeResponse to use in AwsS3UrlReader's readTree() diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md new file mode 100644 index 0000000000..d50aceacb8 --- /dev/null +++ b/docs/integrations/aws-s3/discovery.md @@ -0,0 +1,28 @@ +--- +id: discovery +title: AWS S3 Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from an AWS S3 Bucket +--- + +The AWS S3 integration has a special discovery processor for discovering catalog +entities located in an S3 Bucket. If you have a bucket that contains multiple +catalog-info files and want to automatically discover them, you can use this +processor. The processor will crawl your S3 bucket and register entities +matching the configured path. This can be useful as an alternative to static +locations or manually adding things to the catalog. + +To use the discovery processor, you'll need an AWS S3 integration +[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and +optionally a `roleArn`. Then you can add a location target to the catalog +configuration: + +```yaml +catalog: + locations: + - type: s3-discovery + target: https://sample-bucket.s3.us-east-2.amazonaws.com/ +``` + +Note the `s3-discovery` type, as this is not a regular `url` processor. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 93af2c0be3..2d14a288d8 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -109,7 +109,10 @@ { "type": "subcategory", "label": "AWS S3", - "ids": ["integrations/aws-s3/locations"] + "ids": [ + "integrations/aws-s3/locations", + "integrations/aws-s3/discovery" + ] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 3ffd4aa97d..48d1d49dcf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -82,6 +82,7 @@ nav: - Overview: 'integrations/index.md' - AWS S3: - Locations: 'integrations/aws-s3/locations.md' + - Discovery: 'integrations/aws-s3/discovery.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 9f8cf55a38..e2d68055bf 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -6,6 +6,7 @@ /// /// +import { AwsS3Integration } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BitbucketIntegration } from '@backstage/integration'; import { Config } from '@backstage/config'; @@ -26,10 +27,36 @@ import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; import { RequestHandler } from 'express'; import { Router } from 'express'; +import { S3 } from 'aws-sdk'; import { Server } from 'http'; import * as winston from 'winston'; import { Writable } from 'stream'; +// Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3UrlReader implements UrlReader { + constructor( + integration: AwsS3Integration, + deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string): Promise; + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public (undocumented) export class AzureUrlReader implements UrlReader { constructor( @@ -399,6 +426,12 @@ export type ReadTreeResponseDirOptions = { // @public (undocumented) export interface ReadTreeResponseFactory { + // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; // (undocumented) fromTarArchive( options: ReadTreeResponseFactoryOptions, diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 084f7e3e13..736248b10b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + describe('AwsS3UrlReader', () => { const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { return AwsS3UrlReader.factory({ config: new ConfigReader(config), logger: getVoidLogger(), - treeResponseFactory: DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), - }), + treeResponseFactory, }); }; @@ -137,6 +139,7 @@ describe('AwsS3UrlReader', () => { 'src', 'reading', '__fixtures__', + 'awsS3', 'awsS3-mock-object.yaml', ), ), @@ -153,7 +156,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -188,6 +191,7 @@ describe('AwsS3UrlReader', () => { 'src', 'reading', '__fixtures__', + 'awsS3', 'awsS3-mock-object.yaml', ), ), @@ -206,7 +210,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -229,4 +233,54 @@ describe('AwsS3UrlReader', () => { ); }); }); + describe('readTree', () => { + const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', + }; + const objectList: aws.S3.ObjectList = [object]; + const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, + }; + AWSMock.setSDKInstance(aws); + AWSMock.mock('S3', 'listObjectsV2', output); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: '.amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + { s3, treeResponseFactory }, + ); + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readTree( + 'https://test.s3.us-east-2.amazonaws.com', + ); + const files = await response.files(); + const body = await files[0].content(); + + expect(body.toString()).toBe('site_name: Test\n'); + }); + }); }); diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index d119115930..cb03ec4f71 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import { ReaderFactory, ReadTreeResponse, + ReadTreeResponseFactory, ReadUrlOptions, ReadUrlResponse, SearchResponse, @@ -26,6 +27,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( url: string, @@ -61,7 +63,7 @@ const parseURL = ( }; export class AwsS3UrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); return integrations.awsS3.list().map(integration => { @@ -70,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3); + const reader = new AwsS3UrlReader(integration, { + s3, + treeResponseFactory, + }); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -79,7 +84,10 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, - private readonly s3: S3, + private readonly deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, ) {} /** @@ -145,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader { }; } - const response = this.s3.getObject(params); + const response = this.deps.s3.getObject(params); const buffer = await getRawBody(response.createReadStream()); const etag = (await response.promise()).ETag; @@ -158,8 +166,45 @@ export class AwsS3UrlReader implements UrlReader { } } - async readTree(): Promise { - throw new Error('AwsS3Reader does not implement readTree'); + async readTree(url: string): Promise { + try { + const { path, bucket, region } = parseURL(url); + const allObjects: ObjectList = []; + const responses = []; + let continuationToken: string | undefined; + let output: ListObjectsV2Output; + do { + aws.config.update({ region: region }); + output = await this.deps.s3 + .listObjectsV2({ + Bucket: bucket, + ContinuationToken: continuationToken, + Prefix: path, + }) + .promise(); + if (output.Contents) { + output.Contents.forEach(contents => { + allObjects.push(contents); + }); + } + continuationToken = output.NextContinuationToken; + } while (continuationToken); + + for (let i = 0; i < allObjects.length; i++) { + const object = this.deps.s3.getObject({ + Bucket: bucket, + Key: String(allObjects[i].Key), + }); + responses.push({ + data: object.createReadStream(), + path: String(allObjects[i].Key), + }); + } + + return await this.deps.treeResponseFactory.fromReadableArray(responses); + } catch (e) { + throw new Error(`Could not retrieve file tree from S3: ${e.message}`); + } } async search(): Promise { diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml similarity index 100% rename from packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml rename to packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml new file mode 100644 index 0000000000..b2c132961b --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml @@ -0,0 +1 @@ +site_name: Test2 diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 097f604987..77096795ba 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; +export { AwsS3UrlReader } from './AwsS3UrlReader'; export type { ReaderFactory, ReadTreeOptions, diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 5fdd633102..11a5ef168a 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -20,9 +20,11 @@ import { ReadTreeResponse, ReadTreeResponseFactoryOptions, ReadTreeResponseFactory, + FromReadableArrayOptions, } from '../types'; import { TarArchiveResponse } from './TarArchiveResponse'; import { ZipArchiveResponse } from './ZipArchiveResponse'; +import { ReadableArrayResponse } from './ReadableArrayResponse'; export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { static create(options: { config: Config }): DefaultReadTreeResponseFactory { @@ -57,4 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { options.filter, ); } + + async fromReadableArray( + options: FromReadableArrayOptions, + ): Promise { + return new ReadableArrayResponse(options, this.workDir, ''); + } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts new file mode 100644 index 0000000000..736dc0195c --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -0,0 +1,81 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ReadableArrayResponse } from './ReadableArrayResponse'; +import path, { resolve as resolvePath } from 'path'; +import fs from 'fs-extra'; +import { FromReadableArrayOptions } from '../types'; + +const arr: FromReadableArrayOptions = []; +const arr2: FromReadableArrayOptions = []; +const path1 = path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', +); +const path2 = path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object2.yaml', +); + +describe('ReadableArrayResponse', () => { + it('should read files', async () => { + const stream1 = fs.createReadStream(path1); + const stream2 = fs.createReadStream(path2); + arr.push({ data: stream1, path: path1 }); + arr.push({ data: stream2, path: path2 }); + + const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); + const files = await res.files(); + + expect(files).toEqual([ + { + path: path1, + content: expect.any(Function), + }, + { + path: path2, + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + 'site_name: Test2', + ]); + }); + + it('should extract entire archive into directory', async () => { + const stream1 = fs.createReadStream(path1); + const stream2 = fs.createReadStream(path2); + + arr2.push({ data: stream1, path: path1 }); + arr2.push({ data: stream2, path: path2 }); + + const res = new ReadableArrayResponse(arr2, '/tmp', 'etag'); + const dir = await res.dir(); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test2\n'); + }); +}); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts new file mode 100644 index 0000000000..eabaa3bc56 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import concatStream from 'concat-stream'; +import platformPath, { basename } from 'path'; + +import getRawBody from 'raw-body'; +import fs from 'fs-extra'; +import { promisify } from 'util'; +import tar from 'tar'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import { + ReadTreeResponse, + ReadTreeResponseFile, + ReadTreeResponseDirOptions, + FromReadableArrayOptions, +} from '../types'; + +const pipeline = promisify(pipelineCb); + +/** + * Wraps a array of Readable objects into a tree response reader. + */ +export class ReadableArrayResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: FromReadableArrayOptions, + private readonly workDir: string, + public readonly etag: string, + ) { + this.etag = etag; + } + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + async files(): Promise { + this.onlyOnce(); + + const files = Array(); + + for (let i = 0; i < this.stream.length; i++) { + if (!this.stream[i].path.endsWith('/')) { + files.push({ + path: this.stream[i].path, + content: () => getRawBody(this.stream[i].data), + }); + } + } + + return files; + } + + async archive(): Promise { + const tmpDir = await this.dir(); + + try { + const data = await new Promise(async resolve => { + await pipeline( + tar.create({ cwd: tmpDir }, ['']), + concatStream(resolve), + ); + }); + return Readable.from(data); + } finally { + await fs.remove(tmpDir); + } + } + + async dir(options?: ReadTreeResponseDirOptions): Promise { + this.onlyOnce(); + + const dir = + options?.targetDir ?? + (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); + + for (let i = 0; i < this.stream.length; i++) { + if (!this.stream[i].path.endsWith('/')) { + await pipeline( + this.stream[i].data, + fs.createWriteStream( + platformPath.join(dir, basename(this.stream[i].path)), + ), + ); + } + } + + return dir; + } +} diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index b7e1ff823f..ff799f548a 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -183,6 +183,13 @@ export type ReadTreeResponseFactoryOptions = { // Filter passed on from the ReadTreeOptions filter?: (path: string, info?: { size: number }) => boolean; }; +/** @public */ +export type FromReadableArrayOptions = Array<{ + // Data in the form of a readable + data: Readable; + // A string containing the filepath of the data + path: string; +}>; /** @public */ export interface ReadTreeResponseFactory { @@ -192,6 +199,9 @@ export interface ReadTreeResponseFactory { fromZipArchive( options: ReadTreeResponseFactoryOptions, ): Promise; + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; } /** diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 3b3adbf43e..ecc7c38b80 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -158,6 +158,20 @@ export type AwsOrganizationProviderConfig = { roleArn?: string; }; +// Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3DiscoveryProcessor implements CatalogProcessor { + constructor(reader: UrlReader); + // (undocumented) + readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise; +} + // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 21f7258720..49bf99508a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -73,6 +73,7 @@ "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@types/yup": "^0.29.8", + "aws-sdk-mock": "^5.2.1", "msw": "^0.29.0", "sqlite3": "^5.0.1", "supertest": "^6.1.3", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts new file mode 100644 index 0000000000..435ad368c7 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; +import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; +import { defaultEntityDataParser } from './util/parse'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +AWSMock.setSDKInstance(aws); +const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.txt', +}; +const objectList: aws.S3.ObjectList = [object]; +const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, +}; +AWSMock.mock('S3', 'listObjectsV2', output); +AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'ingestion', + 'processors', + '__fixtures__', + 'fileReaderProcessor', + 'awsS3', + 'awsS3-mock-object.txt', + ), + ), + ), +); + +const logger = getVoidLogger(); +const reader = UrlReaders.default({ + logger, + config: new ConfigReader({ + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }), +}); + +describe('readLocation', () => { + const processor = new AwsS3DiscoveryProcessor(reader); + const spec = { + type: 's3-discovery', + target: 'https://testbucket.s3.us-east-2.amazonaws.com', + }; + + it('should load from url', async () => { + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit, defaultEntityDataParser), + )) as CatalogProcessorEntityResult; + expect(generated.type).toBe('entity'); + expect(generated.location).toEqual({ + target: 'awsS3-mock-object.txt', + type: 's3-discovery', + }); + expect(generated.entity).toEqual({ site_name: 'Test' }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts new file mode 100644 index 0000000000..c394e9b047 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts @@ -0,0 +1,76 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import limiterFactory from 'p-limit'; +import * as result from './results'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; + +export class AwsS3DiscoveryProcessor implements CatalogProcessor { + constructor(private readonly reader: UrlReader) {} + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise { + if (location.type !== 's3-discovery') { + return false; + } + + try { + const output = await this.doRead(location.target); + for (const item of output) { + for await (const parseResult of parser({ + data: item.data, + location: { type: location.type, target: item.url }, + })) { + emit(parseResult); + } + } + } catch (error) { + const message = `Unable to read ${location.type}, ${error}`; + + if (error.name === 'NotFoundError') { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + return true; + } + + private async doRead( + location: string, + ): Promise<{ data: Buffer; url: string }[]> { + const limiter = limiterFactory(5); + const response = await this.reader.readTree(location); + const responseFiles = await response.files(); + const output = responseFiles.map(async file => ({ + url: file.path, + data: await limiter(file.content), + })); + return Promise.all(output); + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt @@ -0,0 +1 @@ +site_name: Test diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index fbdadba667..4f3c502a8e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -20,6 +20,7 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; export type { AwsOrganizationProviderConfig } from './awsOrganization/config'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor';