From b8086dedd7d46c8b1616bd88ef90f314c5ec6d25 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 16:30:05 -0700 Subject: [PATCH 01/61] First go Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.ts | 59 ++++++++++++++++- .../reading/tree/ReadTreeResponseFactory.ts | 8 +++ .../src/reading/tree/ReadableArrayResponse.ts | 66 +++++++++++++++++++ packages/backend-common/src/reading/types.ts | 10 +++ 4 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 packages/backend-common/src/reading/tree/ReadableArrayResponse.ts diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index d119115930..da9e77d9ca 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -26,6 +26,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { Readable } from 'stream'; const parseURL = ( url: string, @@ -158,8 +159,62 @@ export class AwsS3UrlReader implements UrlReader { } } - async readTree(): Promise { - throw new Error('AwsS3Reader does not implement readTree'); + async readTree(url: string): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + let moreKeys = true; + let awsS3Readables: Readable[] = []; + let continuationToken = ''; + + while (moreKeys) { + let params; + if (continuationToken === '') { + params = { + Bucket: bucket, + Prefix: path, + }; + } else { + params = { + Bucket: bucket, + Prefix: path, + ContinuationToken: continuationToken, + }; + } + const { Contents, IsTruncated, NextContinuationToken } = await this.s3 + .listObjectsV2(params) + .promise(); + + const responses = await Promise.all( + (Contents || []).map(({ Key }) => { + const s3Response = this.s3 + .getObject({ Bucket: bucket, Key: String(Key) }) + .createReadStream(); + Object.defineProperty(s3Response, 'path', { + value: String(Key), + writable: false, + }); + return s3Response; + }), + ); + + if (IsTruncated) { + continuationToken = String(NextContinuationToken); + } else { + continuationToken = ''; + moreKeys = false; + } + awsS3Readables = awsS3Readables.concat(responses); + } + + return await this.treeResponseFactory.fromReadableArray({ + stream: awsS3Readables, + etag: '', + }); + } catch (e) { + throw new Error(`Could not retrieve file tree from S3: ${e.message}`); + } } async search(): Promise { diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 912fddf965..19d7ed2fab 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -20,9 +20,11 @@ import { ReadTreeResponse, FromArchiveOptions, ReadTreeResponseFactory, + FromReadableArrayOptions, } from '../types'; import { TarArchiveResponse } from './TarArchiveResponse'; import { ZipArchiveResponse } from './ZipArchiveResponse'; +import { ReadableArrayResponse } from './ReadableArrayResponse'; export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { static create(options: { config: Config }): DefaultReadTreeResponseFactory { @@ -53,4 +55,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { options.filter, ); } + + async fromReadableArray( + options: FromReadableArrayOptions, + ): Promise { + return new ReadableArrayResponse(options.stream, options.etag); + } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts new file mode 100644 index 0000000000..8b5d3ff0b5 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import getRawBody from 'raw-body'; +import { Readable } from 'stream'; +import { ReadTreeResponse, ReadTreeResponseFile } from '../types'; + +/** + * Wraps a array of Readable objects into a tree response reader. + */ +export class ReadableArrayResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: Readable[], + public readonly etag: string, + ) { + this.etag = etag; + } + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + async files(): Promise { + this.onlyOnce(); + + const files = Array(); + + for (let i = 0; i < this.stream.length; i++) { + if (!(this.stream[i] as any).path.endsWith('/')) { + files.push({ + path: (this.stream[i] as any).path, + content: () => getRawBody(this.stream[i]), + }); + } + } + + return files; + } + + archive(): Promise { + throw new Error('Method not implemented.'); + } + + dir(): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 93f287d4fe..8d896408c3 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -167,9 +167,19 @@ export type FromArchiveOptions = { filter?: (path: string, info?: { size: number }) => boolean; }; +export type FromReadableArrayOptions = { + // An array of readable streams + stream: Readable[]; + // etag of the file tree + etag: string; +}; + export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; } /** From 522b55fdcccbe2fb308c66e4f9bc42d5d777c333 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 16:46:42 -0700 Subject: [PATCH 02/61] More work Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.ts | 6 +- .../processors/AwsS3ReadTreeProcessor.ts | 74 +++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index da9e77d9ca..f7904ca70f 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import { ReaderFactory, ReadTreeResponse, + ReadTreeResponseFactory, ReadUrlOptions, ReadUrlResponse, SearchResponse, @@ -62,7 +63,7 @@ const parseURL = ( }; export class AwsS3UrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); return integrations.awsS3.list().map(integration => { @@ -71,7 +72,7 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3); + const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -81,6 +82,7 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, private readonly s3: S3, + private readonly treeResponseFactory: ReadTreeResponseFactory, ) {} /** diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts new file mode 100644 index 0000000000..8e65714a4c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import * as result from './results'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; + +export class AwsS3ReadTreeProcessor implements CatalogProcessor { + constructor(private readonly reader: UrlReader) {} + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise { + if (location.type !== 'aws-read-tree') { + return false; + } + + try { + const output = await this.doRead(location.target); + for (const item of output) { + for await (const parseResult of parser({ + data: item.data, + location: { type: location.type, target: item.url }, + })) { + emit(parseResult); + } + } + } catch (error) { + const message = `Unable to read ${location.type}, ${error}`; + + if (error.name === 'NotFoundError') { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + return true; + } + + private async doRead( + location: string, + ): Promise<{ data: Buffer; url: string }[]> { + const response = await this.reader.readTree(location); + const responseFiles = await response.files(); + const output = responseFiles.map(async file => ({ + url: file.path, + data: await file.content(), + })); + return Promise.all(output); + } +} From 9972778cd747a037be835fe11271a575c21e3bbe Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 17:24:14 -0700 Subject: [PATCH 03/61] Merge work done on previous branch Signed-off-by: Sean Tan --- app-config.yaml | 2 ++ .../backend-common/src/reading/AwsS3UrlReader.test.ts | 10 +++++++--- packages/backend/src/plugins/catalog.ts | 3 ++- .../catalog-backend/src/ingestion/processors/index.ts | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 9f365becef..fcfd92a66f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,6 +259,8 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml + - type: aws-read-tree + target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 084f7e3e13..78bfdbb0e2 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + describe('AwsS3UrlReader', () => { const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { return AwsS3UrlReader.factory({ config: new ConfigReader(config), logger: getVoidLogger(), - treeResponseFactory: DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), - }), + treeResponseFactory, }); }; @@ -154,6 +156,7 @@ describe('AwsS3UrlReader', () => { ), ), s3, + treeResponseFactory, ); it('returns contents of an object in a bucket', async () => { @@ -207,6 +210,7 @@ describe('AwsS3UrlReader', () => { ), ), s3, + treeResponseFactory, ); it('returns contents of an object in a bucket', async () => { diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 055595dcb5..9c94b3851e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,6 +25,7 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); + builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 6a44ea0a8f..23d14dde01 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -19,6 +19,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; +export { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; From 20334fc3cb5f0c8a518ddd56bcb30ee2c0aa58ac Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 16 Aug 2021 17:14:47 -0700 Subject: [PATCH 04/61] Add ReadableArrayResponse tests Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 2 + .../{ => awsS3}/awsS3-mock-object.yaml | 0 .../awsS3/awsS3-mock-object2.yaml | 1 + .../tree/ReadableArrayResponse.test.ts | 62 +++++++++++++++++++ 4 files changed, 65 insertions(+) rename packages/backend-common/src/reading/__fixtures__/{ => awsS3}/awsS3-mock-object.yaml (100%) create mode 100644 packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml create mode 100644 packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 78bfdbb0e2..78cdcbb7b9 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -139,6 +139,7 @@ describe('AwsS3UrlReader', () => { 'src', 'reading', '__fixtures__', + 'awsS3', 'awsS3-mock-object.yaml', ), ), @@ -191,6 +192,7 @@ describe('AwsS3UrlReader', () => { 'src', 'reading', '__fixtures__', + 'awsS3', 'awsS3-mock-object.yaml', ), ), diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml similarity index 100% rename from packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml rename to packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml new file mode 100644 index 0000000000..b2c132961b --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml @@ -0,0 +1 @@ +site_name: Test2 diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts new file mode 100644 index 0000000000..c2e3c26269 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ReadableArrayResponse } from './ReadableArrayResponse'; +import path from 'path'; +import { Readable } from 'stream'; +import fs from 'fs'; + +const arr: Readable[] = []; +const file1 = path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', +); +const file2 = path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object2.yaml', +); +const stream1 = fs.createReadStream(file1); +const stream2 = fs.createReadStream(file2); +arr.push(stream1); +arr.push(stream2); + +describe('ReadableArrayResponse', () => { + it('should read files', async () => { + const res = new ReadableArrayResponse(arr, 'etag'); + const files = await res.files(); + + expect(files).toEqual([ + { + path: file1, + content: expect.any(Function), + }, + { + path: file2, + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + 'site_name: Test2', + ]); + }); +}); From 6d6392a1d389e741eeca7dfa31147c5cea56af87 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 17 Aug 2021 19:38:27 -0700 Subject: [PATCH 05/61] Finish AwsS3ReadTreeProcessor tests Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 51 ++++++++++ packages/backend-common/src/reading/index.ts | 1 + plugins/catalog-backend/package.json | 1 + .../processors/AwsS3ReadTreeProcessor.test.ts | 97 +++++++++++++++++++ .../awsS3/awsS3-mock-object.yaml | 1 + 5 files changed, 151 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 78cdcbb7b9..9be6de7a30 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -235,4 +235,55 @@ describe('AwsS3UrlReader', () => { ); }); }); + describe('readTree', () => { + const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', + }; + const objectList: aws.S3.ObjectList = [object]; + const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, + }; + AWSMock.setSDKInstance(aws); + AWSMock.mock('S3', 'listObjectsV2', output); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: '.amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + treeResponseFactory, + ); + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readTree( + 'https://test.s3.us-east-2.amazonaws.com', + ); + const files = await response.files(); + const body = await files[0].content(); + + expect(body.toString()).toBe('site_name: Test\n'); + }); + }); }); diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 4c601556d7..e50ba1e305 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; +export { AwsS3UrlReader } from './AwsS3UrlReader'; export type { ReadTreeResponse, ReadTreeResponseFile, diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f095f39785..e490214385 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -61,6 +61,7 @@ "yup": "^0.29.3" }, "devDependencies": { + "aws-sdk-mock": "^5.2.1", "@backstage/backend-test-utils": "^0.1.5", "@backstage/cli": "^0.7.8", "@backstage/test-utils": "^0.1.17", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts new file mode 100644 index 0000000000..6b99506e26 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + AwsS3UrlReader, + DefaultReadTreeResponseFactory, +} from '@backstage/backend-common'; + +import { ConfigReader } from '@backstage/config'; +import { + AwsS3Integration, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; +import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; +import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; +import { defaultEntityDataParser } from './util/parse'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + +AWSMock.setSDKInstance(aws); +const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', +}; +const objectList: aws.S3.ObjectList = [object]; +const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, +}; +AWSMock.mock('S3', 'listObjectsV2', output); +AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'ingestion', + 'processors', + '__fixtures__', + 'fileReaderProcessor', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), +); + +const s3 = new aws.S3(); +const reader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + treeResponseFactory, +); + +describe('readLocation', () => { + const processor = new AwsS3ReadTreeProcessor(reader); + const spec = { + type: 'aws-read-tree', + target: 'https://testbucket.s3.us-east-2.amazonaws.com', + }; + + it('should load from url', async () => { + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit, defaultEntityDataParser), + )) as CatalogProcessorEntityResult; + expect(generated.type).toBe('entity'); + expect(generated.location).toEqual({ + target: 'awsS3-mock-object.yaml', + type: 'aws-read-tree', + }); + expect(generated.entity).toEqual({ site_name: 'Test' }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml @@ -0,0 +1 @@ +site_name: Test From 2e9e29cb7a28bca82f405124c4f07e852a842cc9 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:28:27 -0700 Subject: [PATCH 06/61] Preparing for PR Signed-off-by: Sean Tan --- docs/integrations/aws-s3/readtree.md | 28 +++++++++++++++++++ microsite/sidebars.json | 2 +- mkdocs.yml | 1 + packages/backend-common/src/index.ts | 1 + .../src/reading/AwsS3UrlReader.ts | 1 - 5 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 docs/integrations/aws-s3/readtree.md diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md new file mode 100644 index 0000000000..7ae1fece63 --- /dev/null +++ b/docs/integrations/aws-s3/readtree.md @@ -0,0 +1,28 @@ +--- +id: read-tree +title: AWS S3 Read Tree +sidebar_label: Read Tree +# prettier-ignore +description: Discovering multiple catalog entities from an AWS S3 Bucket +--- + +The AWS S3 integration has a special processor for using AwsS3UrlReader's +readTree() method. The method readTree() allows a user to retrieve more than one +file hosted inside an S3 bucket. The processor passes a valid S3 URL and +credentials to AwsS3UrlReader's readTree(). A user can supply a URL which points +to root of a bucket or a subdirectory inside the bucket. This can be useful as +an alternative to static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need an AWS S3 integration +[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and +optionally a `roleArn`. Then you can add a location target to the catalog +configuration: + +```yaml +catalog: + locations: + - type: aws-read-tree + target: https://sample-bucket.s3.us-east-2.amazonaws.com/ +``` + +Note the `aws-read-tree` type, as this is not a regular `url` processor. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index d787fb28aa..3517c8d70c 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -107,7 +107,7 @@ { "type": "subcategory", "label": "AWS S3", - "ids": ["integrations/aws-s3/locations"] + "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/readtree"] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 73ad52341e..3b4dad765c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - Overview: 'integrations/index.md' - AWS S3: - Locations: 'integrations/aws-s3/locations.md' + - Read Tree: 'integrations/aws-s3/readtree.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index f2f38d9aab..5b37c2fb9f 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -23,6 +23,7 @@ export * from './logging'; export * from './middleware'; export * from './paths'; export * from './reading'; +export * from './reading/tree'; export * from './scm'; export * from './service'; export * from './util'; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index f7904ca70f..615a095e50 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -187,7 +187,6 @@ export class AwsS3UrlReader implements UrlReader { const { Contents, IsTruncated, NextContinuationToken } = await this.s3 .listObjectsV2(params) .promise(); - const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3 From 96b818fdb2ea5f16f279af38b33c46f684d46099 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:40:25 -0700 Subject: [PATCH 07/61] Add readTree() to AwsS3UrlReader and AwsS3ReadTreeProcessor Signed-off-by: Sean Tan --- .changeset/shiny-emus-perform.md | 7 +++++++ app-config.yaml | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/shiny-emus-perform.md diff --git a/.changeset/shiny-emus-perform.md b/.changeset/shiny-emus-perform.md new file mode 100644 index 0000000000..76a40504c8 --- /dev/null +++ b/.changeset/shiny-emus-perform.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +'@backstage/plugin-catalog-backend': patch +--- + +Add readTree() to AwsS3UrlReader and aws-read-tree Catalog Processor diff --git a/app-config.yaml b/app-config.yaml index fcfd92a66f..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,8 +259,6 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml - - type: aws-read-tree - target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created From 030e3e30e6421b7e447fd02bb23c5d332c922cd2 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:46:48 -0700 Subject: [PATCH 08/61] Remove local changes made to catalog.ts Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 9c94b3851e..75709eefc7 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,7 +24,6 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); - builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, From 55262fb1389f0b161226229757940f19bf6663fe Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:49:11 -0700 Subject: [PATCH 09/61] Fix one more error in catalog.ts Signed-off-by: Sean Tan Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 75709eefc7..055595dcb5 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { CatalogBuilder, createRouter, From bedf145c2c700850c56100472c57541a8424e8bf Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 15:51:31 -0700 Subject: [PATCH 10/61] Regenerate api reports Signed-off-by: Sean Tan --- packages/backend-common/api-report.md | 58 ++++++++++++++++++++++++--- plugins/catalog-backend/api-report.md | 14 +++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 736899d1ed..adbc32f67c 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -6,6 +6,7 @@ /// /// +import { AwsS3Integration } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BitbucketIntegration } from '@backstage/integration'; import { Config } from '@backstage/config'; @@ -28,10 +29,40 @@ import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; import { RequestHandler } from 'express'; import { Router } from 'express'; +import { S3 } from 'aws-sdk'; import { Server } from 'http'; import * as winston from 'winston'; import { Writable } from 'stream'; +// Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3UrlReader implements UrlReader { + // Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts + constructor( + integration: AwsS3Integration, + s3: S3, + treeResponseFactory: ReadTreeResponseFactory, + ); + // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string): Promise; + // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts + // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts + // + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // Warning: (ae-missing-release-tag) "AzureUrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -42,8 +73,6 @@ export class AzureUrlReader implements UrlReader { treeResponseFactory: ReadTreeResponseFactory; }, ); - // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts - // // (undocumented) static factory: ReaderFactory; // (undocumented) @@ -52,9 +81,6 @@ export class AzureUrlReader implements UrlReader { // // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts - // // (undocumented) readUrl(url: string, _options?: ReadUrlOptions): Promise; // Warning: (ae-forgotten-export) The symbol "SearchOptions" needs to be exported by the entry point index.d.ts @@ -173,6 +199,27 @@ export class DatabaseManager { static fromConfig(config: Config): DatabaseManager; } +// Warning: (ae-missing-release-tag) "DefaultReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { + constructor(workDir: string); + // (undocumented) + static create(options: { config: Config }): DefaultReadTreeResponseFactory; + // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; + // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromTarArchive(options: FromArchiveOptions): Promise; + // (undocumented) + fromZipArchive(options: FromArchiveOptions): Promise; +} + // Warning: (ae-missing-release-tag) "DockerContainerRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -592,7 +639,6 @@ export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; // // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here -// src/reading/AzureUrlReader.d.ts:9:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts // src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a6e67215c9..3ccebf48f9 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -121,6 +121,20 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "AwsS3ReadTreeProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3ReadTreeProcessor implements CatalogProcessor { + constructor(reader: UrlReader); + // (undocumented) + readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise; +} + // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From c2d7efb782fcc8a7e4cc4124427aa12c54399b17 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 16:30:05 -0700 Subject: [PATCH 11/61] First go Signed-off-by: Sean Tan --- packages/backend-common/src/reading/AwsS3UrlReader.ts | 4 ++++ packages/backend-common/src/reading/types.ts | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 615a095e50..6c2d55972b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -187,6 +187,10 @@ export class AwsS3UrlReader implements UrlReader { const { Contents, IsTruncated, NextContinuationToken } = await this.s3 .listObjectsV2(params) .promise(); +<<<<<<< HEAD +======= + +>>>>>>> First go const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3 diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 8d896408c3..448e678435 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -174,6 +174,13 @@ export type FromReadableArrayOptions = { etag: string; }; +export type FromReadableArrayOptions = { + // An array of readable streams + stream: Readable[]; + // etag of the file tree + etag: string; +}; + export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; From 2cb8041f700d92e6d6de8ab44e8e6bcd6cbe8061 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 17:24:14 -0700 Subject: [PATCH 12/61] Merge work done on previous branch Signed-off-by: Sean Tan --- app-config.yaml | 2 ++ packages/backend/src/plugins/catalog.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index 9f365becef..fcfd92a66f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,6 +259,8 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml + - type: aws-read-tree + target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 055595dcb5..9c94b3851e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,6 +25,7 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); + builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, From 1297baca45647a322be21b8aa0d2642f64c4e395 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:28:27 -0700 Subject: [PATCH 13/61] Preparing for PR Signed-off-by: Sean Tan --- packages/backend-common/src/reading/AwsS3UrlReader.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 6c2d55972b..a512d7f64b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -188,9 +188,12 @@ export class AwsS3UrlReader implements UrlReader { .listObjectsV2(params) .promise(); <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> First go +======= +>>>>>>> Preparing for PR const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3 From 689ab3204d2da1b8b66cbd29e0c8059afa043cb8 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:40:25 -0700 Subject: [PATCH 14/61] Add readTree() to AwsS3UrlReader and AwsS3ReadTreeProcessor Signed-off-by: Sean Tan --- app-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index fcfd92a66f..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,8 +259,6 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml - - type: aws-read-tree - target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created From ee9e520c157c6fb0f23929f1f535363622ba0bc6 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:46:48 -0700 Subject: [PATCH 15/61] Remove local changes made to catalog.ts Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 9c94b3851e..75709eefc7 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,7 +24,6 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); - builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, From 393ec8b3c6c35833fb4d6c85f510bf58c76ec843 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:49:11 -0700 Subject: [PATCH 16/61] Fix one more error in catalog.ts Signed-off-by: Sean Tan Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 75709eefc7..055595dcb5 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { CatalogBuilder, createRouter, From 8bde9ef1f12b47ecc4b11e0ee60f5013c026de0b Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 15:57:23 -0700 Subject: [PATCH 17/61] Duplicate entry Signed-off-by: Sean Tan --- packages/backend-common/src/reading/types.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 448e678435..c0acc04701 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -173,14 +173,6 @@ export type FromReadableArrayOptions = { // etag of the file tree etag: string; }; - -export type FromReadableArrayOptions = { - // An array of readable streams - stream: Readable[]; - // etag of the file tree - etag: string; -}; - export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; From b992dacf4f110d4c9cf495b3f4a24ff53a175f5e Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 16:00:31 -0700 Subject: [PATCH 18/61] Remove random file change Signed-off-by: Sean Tan --- .../software-templates/migrating-from-v1alpha1-to-v1beta2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md index 28001cce0b..25ff2a2f3a 100644 --- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md +++ b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md @@ -325,7 +325,7 @@ spec: output: links: - url: '{{steps.publish.output.remoteUrl}}' - title: 'Go to Repo' + text: 'Go to Repo' ``` ## Questions? From 7f5ce313f82ed9ba6535deacee2224508cd87521 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 23 Aug 2021 12:57:42 -0700 Subject: [PATCH 19/61] Begin changes on PR Signed-off-by: Sean Tan --- docs/integrations/aws-s3/readtree.md | 4 +- packages/backend-common/src/index.ts | 1 - .../src/reading/AwsS3UrlReader.ts | 29 +++++++------- .../processors/AwsS3ReadTreeProcessor.test.ts | 39 +++++-------------- .../processors/AwsS3ReadTreeProcessor.ts | 2 +- 5 files changed, 26 insertions(+), 49 deletions(-) diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md index 7ae1fece63..47ce867137 100644 --- a/docs/integrations/aws-s3/readtree.md +++ b/docs/integrations/aws-s3/readtree.md @@ -21,8 +21,8 @@ configuration: ```yaml catalog: locations: - - type: aws-read-tree + - type: s3-bucket target: https://sample-bucket.s3.us-east-2.amazonaws.com/ ``` -Note the `aws-read-tree` type, as this is not a regular `url` processor. +Note the `s3-bucket` type, as this is not a regular `url` processor. diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 5b37c2fb9f..f2f38d9aab 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -23,7 +23,6 @@ export * from './logging'; export * from './middleware'; export * from './paths'; export * from './reading'; -export * from './reading/tree'; export * from './scm'; export * from './service'; export * from './util'; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index a512d7f64b..593a66cfcc 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -72,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory); + const reader = new AwsS3UrlReader(integration, { + s3, + treeResponseFactory, + }); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -81,8 +84,10 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, - private readonly s3: S3, - private readonly treeResponseFactory: ReadTreeResponseFactory, + private readonly deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, ) {} /** @@ -148,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader { }; } - const response = this.s3.getObject(params); + const response = this.deps.s3.getObject(params); const buffer = await getRawBody(response.createReadStream()); const etag = (await response.promise()).ETag; @@ -184,19 +189,11 @@ export class AwsS3UrlReader implements UrlReader { ContinuationToken: continuationToken, }; } - const { Contents, IsTruncated, NextContinuationToken } = await this.s3 - .listObjectsV2(params) - .promise(); -<<<<<<< HEAD -<<<<<<< HEAD -======= - ->>>>>>> First go -======= ->>>>>>> Preparing for PR + const { Contents, IsTruncated, NextContinuationToken } = + await this.deps.s3.listObjectsV2(params).promise(); const responses = await Promise.all( (Contents || []).map(({ Key }) => { - const s3Response = this.s3 + const s3Response = this.deps.s3 .getObject({ Bucket: bucket, Key: String(Key) }) .createReadStream(); Object.defineProperty(s3Response, 'path', { @@ -216,7 +213,7 @@ export class AwsS3UrlReader implements UrlReader { awsS3Readables = awsS3Readables.concat(responses); } - return await this.treeResponseFactory.fromReadableArray({ + return await this.deps.treeResponseFactory.fromReadableArray({ stream: awsS3Readables, etag: '', }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts index 6b99506e26..0b8ea6f814 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -13,16 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - AwsS3UrlReader, - DefaultReadTreeResponseFactory, -} from '@backstage/backend-common'; - +import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { - AwsS3Integration, - readAwsS3IntegrationConfig, -} from '@backstage/integration'; import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; import { defaultEntityDataParser } from './util/parse'; @@ -30,10 +22,6 @@ import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; -const treeResponseFactory = DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), -}); - AWSMock.setSDKInstance(aws); const object: aws.S3.Types.Object = { Key: 'awsS3-mock-object.yaml', @@ -61,25 +49,18 @@ AWSMock.mock( ), ); -const s3 = new aws.S3(); -const reader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), - ), - ), - s3, - treeResponseFactory, -); +const logger = getVoidLogger(); +const reader = UrlReaders.default({ + logger, + config: new ConfigReader({ + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }), +}); describe('readLocation', () => { const processor = new AwsS3ReadTreeProcessor(reader); const spec = { - type: 'aws-read-tree', + type: 's3-bucket', target: 'https://testbucket.s3.us-east-2.amazonaws.com', }; @@ -90,7 +71,7 @@ describe('readLocation', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ target: 'awsS3-mock-object.yaml', - type: 'aws-read-tree', + type: 's3-bucket', }); expect(generated.entity).toEqual({ site_name: 'Test' }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts index 8e65714a4c..9d8e1cdce5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts @@ -32,7 +32,7 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'aws-read-tree') { + if (location.type !== 's3-bucket') { return false; } From 4d4b302905b099576e0c14bdeb7cd408368f8dc8 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 24 Aug 2021 17:29:21 -0700 Subject: [PATCH 20/61] Finish ReadableArrayResponse changes Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 9 ++-- .../reading/tree/ReadTreeResponseFactory.ts | 6 ++- .../tree/ReadableArrayResponse.test.ts | 34 +++++++++--- .../src/reading/tree/ReadableArrayResponse.ts | 54 ++++++++++++++++--- 4 files changed, 83 insertions(+), 20 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 9be6de7a30..736248b10b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -156,8 +156,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -211,8 +210,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -273,8 +271,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readTree( diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 19d7ed2fab..7d20385635 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -59,6 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { async fromReadableArray( options: FromReadableArrayOptions, ): Promise { - return new ReadableArrayResponse(options.stream, options.etag); + return new ReadableArrayResponse( + options.stream, + this.workDir, + options.etag, + ); } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index c2e3c26269..bf457f02e4 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ import { ReadableArrayResponse } from './ReadableArrayResponse'; -import path from 'path'; +import path, { resolve as resolvePath } from 'path'; + import { Readable } from 'stream'; -import fs from 'fs'; +import fs from 'fs-extra'; const arr: Readable[] = []; +const arr2: Readable[] = []; const file1 = path.resolve( 'src', 'reading', @@ -33,14 +35,15 @@ const file2 = path.resolve( 'awsS3', 'awsS3-mock-object2.yaml', ); -const stream1 = fs.createReadStream(file1); -const stream2 = fs.createReadStream(file2); -arr.push(stream1); -arr.push(stream2); describe('ReadableArrayResponse', () => { it('should read files', async () => { - const res = new ReadableArrayResponse(arr, 'etag'); + const stream1 = fs.createReadStream(file1); + const stream2 = fs.createReadStream(file2); + arr.push(stream1); + arr.push(stream2); + + const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -59,4 +62,21 @@ describe('ReadableArrayResponse', () => { 'site_name: Test2', ]); }); + + it('should extract entire archive into directory', async () => { + const stream1 = fs.createReadStream(file1); + const stream2 = fs.createReadStream(file2); + + arr2.push(stream1); + arr2.push(stream2); + + const res = new ReadableArrayResponse(arr2, '/tmp', 'etag'); + const dir = await res.dir(); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test2\n'); + }); }); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index 8b5d3ff0b5..bf74a19603 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -14,9 +14,21 @@ * limitations under the License. */ +import concatStream from 'concat-stream'; +import platformPath, { basename } from 'path'; + import getRawBody from 'raw-body'; -import { Readable } from 'stream'; -import { ReadTreeResponse, ReadTreeResponseFile } from '../types'; +import fs from 'fs-extra'; +import { promisify } from 'util'; +import tar from 'tar'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import { + ReadTreeResponse, + ReadTreeResponseFile, + ReadTreeResponseDirOptions, +} from '../types'; + +const pipeline = promisify(pipelineCb); /** * Wraps a array of Readable objects into a tree response reader. @@ -26,6 +38,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { constructor( private readonly stream: Readable[], + private readonly workDir: string, public readonly etag: string, ) { this.etag = etag; @@ -56,11 +69,40 @@ export class ReadableArrayResponse implements ReadTreeResponse { return files; } - archive(): Promise { - throw new Error('Method not implemented.'); + async archive(): Promise { + const tmpDir = await this.dir(); + + try { + const data = await new Promise(async resolve => { + await pipeline( + tar.create({ cwd: tmpDir }, ['']), + concatStream(resolve), + ); + }); + return Readable.from(data); + } finally { + await fs.remove(tmpDir); + } } - dir(): Promise { - throw new Error('Method not implemented.'); + async dir(options?: ReadTreeResponseDirOptions): Promise { + this.onlyOnce(); + + const dir = + options?.targetDir ?? + (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); + + for (let i = 0; i < this.stream.length; i++) { + if (!(this.stream[i] as any).path.endsWith('/')) { + await pipeline( + this.stream[i], + fs.createWriteStream( + platformPath.join(dir, basename((this.stream[i] as any).path)), + ), + ); + } + } + + return dir; } } From 160a50029faa706d4e2dbfd77f25f11dbbc71dcb Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 24 Aug 2021 18:39:08 -0700 Subject: [PATCH 21/61] Finish up changes Signed-off-by: Sean Tan --- packages/backend-common/api-report.md | 29 ++++--------------- .../src/reading/AwsS3UrlReader.ts | 2 +- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index adbc32f67c..ac35fa698a 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -38,11 +38,12 @@ import { Writable } from 'stream'; // // @public (undocumented) export class AwsS3UrlReader implements UrlReader { - // Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts constructor( integration: AwsS3Integration, - s3: S3, - treeResponseFactory: ReadTreeResponseFactory, + deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, ); // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts // @@ -199,27 +200,6 @@ export class DatabaseManager { static fromConfig(config: Config): DatabaseManager; } -// Warning: (ae-missing-release-tag) "DefaultReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { - constructor(workDir: string); - // (undocumented) - static create(options: { config: Config }): DefaultReadTreeResponseFactory; - // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts - // - // (undocumented) - fromReadableArray( - options: FromReadableArrayOptions, - ): Promise; - // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts - // - // (undocumented) - fromTarArchive(options: FromArchiveOptions): Promise; - // (undocumented) - fromZipArchive(options: FromArchiveOptions): Promise; -} - // Warning: (ae-missing-release-tag) "DockerContainerRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -639,6 +619,7 @@ export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; // // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here +// src/reading/AwsS3UrlReader.d.ts:11:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts // src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 593a66cfcc..c48d16cc69 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -169,7 +169,6 @@ export class AwsS3UrlReader implements UrlReader { async readTree(url: string): Promise { try { const { path, bucket, region } = parseURL(url); - aws.config.update({ region: region }); let moreKeys = true; let awsS3Readables: Readable[] = []; @@ -189,6 +188,7 @@ export class AwsS3UrlReader implements UrlReader { ContinuationToken: continuationToken, }; } + aws.config.update({ region: region }); const { Contents, IsTruncated, NextContinuationToken } = await this.deps.s3.listObjectsV2(params).promise(); const responses = await Promise.all( From 5b1aebca114089b5b857190dc156b032a1ceb7bf Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 24 Aug 2021 19:34:48 -0700 Subject: [PATCH 22/61] Change FileReaderProcessor tests to reflect new test file 'aws-s3-mock-object.yaml' Signed-off-by: Sean Tan --- .../src/ingestion/processors/FileReaderProcessor.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts index 41c7e4f732..c1dfbb0af7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts @@ -77,7 +77,7 @@ describe('FileReaderProcessor', () => { emit, ); - expect(emit).toBeCalledTimes(2); + expect(emit).toBeCalledTimes(3); expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', From 9bbe5dd98cae931fb68793043fef940c8d41b21f Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 25 Aug 2021 10:48:27 -0700 Subject: [PATCH 23/61] Change AwsS3ReadTreeProcessor tests as to not interfere with FileReaderProcessor tests Signed-off-by: Sean Tan --- .../src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts | 6 +++--- .../src/ingestion/processors/FileReaderProcessor.test.ts | 2 +- .../awsS3/{awsS3-mock-object.yaml => awsS3-mock-object.txt} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/{awsS3-mock-object.yaml => awsS3-mock-object.txt} (100%) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts index 0b8ea6f814..ebc258a29e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -24,7 +24,7 @@ import path from 'path'; AWSMock.setSDKInstance(aws); const object: aws.S3.Types.Object = { - Key: 'awsS3-mock-object.yaml', + Key: 'awsS3-mock-object.txt', }; const objectList: aws.S3.ObjectList = [object]; const output: aws.S3.Types.ListObjectsV2Output = { @@ -43,7 +43,7 @@ AWSMock.mock( '__fixtures__', 'fileReaderProcessor', 'awsS3', - 'awsS3-mock-object.yaml', + 'awsS3-mock-object.txt', ), ), ), @@ -70,7 +70,7 @@ describe('readLocation', () => { )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ - target: 'awsS3-mock-object.yaml', + target: 'awsS3-mock-object.txt', type: 's3-bucket', }); expect(generated.entity).toEqual({ site_name: 'Test' }); diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts index c1dfbb0af7..41c7e4f732 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts @@ -77,7 +77,7 @@ describe('FileReaderProcessor', () => { emit, ); - expect(emit).toBeCalledTimes(3); + expect(emit).toBeCalledTimes(2); expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml rename to plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt From c63ba3eff0b03fe804718bc5eab28145306eebe7 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Thu, 26 Aug 2021 10:31:09 -0700 Subject: [PATCH 24/61] Resolve conflict with package.json Signed-off-by: Sean Tan --- plugins/catalog-backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index e490214385..d8fabc8d7a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -62,8 +62,8 @@ }, "devDependencies": { "aws-sdk-mock": "^5.2.1", - "@backstage/backend-test-utils": "^0.1.5", - "@backstage/cli": "^0.7.8", + "@backstage/backend-test-utils": "^0.1.6", + "@backstage/cli": "^0.7.9", "@backstage/test-utils": "^0.1.17", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", From bd119f21cc075b9d09eb0470566eee996364fbb0 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Thu, 26 Aug 2021 10:38:09 -0700 Subject: [PATCH 25/61] Copy package.json from master and add 'aws-sdk-mock' to devDependencies Signed-off-by: Sean Tan --- plugins/catalog-backend/package.json | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index d8fabc8d7a..34c9caccbc 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.13.2", + "version": "0.13.5", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,17 +29,18 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.8.9", + "@backstage/backend-common": "^0.9.0", "@backstage/catalog-client": "^0.3.18", "@backstage/catalog-model": "^0.9.0", - "@backstage/config": "^0.1.5", + "@backstage/config": "^0.1.8", "@backstage/errors": "^0.1.1", - "@backstage/integration": "^0.6.0", + "@backstage/integration": "^0.6.2", "@backstage/plugin-search-backend-node": "^0.4.0", "@backstage/search-common": "^0.1.2", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "aws-sdk": "^2.840.0", + "aws-sdk-mock": "^5.2.1", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", @@ -51,8 +52,10 @@ "glob": "^7.1.6", "knex": "^0.95.1", "lodash": "^4.17.15", + "luxon": "^2.0.2", "morgan": "^1.10.0", "p-limit": "^3.0.2", + "prom-client": "^13.2.0", "qs": "^6.9.4", "uuid": "^8.0.0", "winston": "^3.2.1", @@ -61,7 +64,6 @@ "yup": "^0.29.3" }, "devDependencies": { - "aws-sdk-mock": "^5.2.1", "@backstage/backend-test-utils": "^0.1.6", "@backstage/cli": "^0.7.9", "@backstage/test-utils": "^0.1.17", From dfbbb4fefabe461647fe897d43e190941c6ab60e Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 16:30:05 -0700 Subject: [PATCH 26/61] First go Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.ts | 59 ++++++++++++++++- .../reading/tree/ReadTreeResponseFactory.ts | 8 +++ .../src/reading/tree/ReadableArrayResponse.ts | 66 +++++++++++++++++++ packages/backend-common/src/reading/types.ts | 10 +++ 4 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 packages/backend-common/src/reading/tree/ReadableArrayResponse.ts diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index d119115930..da9e77d9ca 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -26,6 +26,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { Readable } from 'stream'; const parseURL = ( url: string, @@ -158,8 +159,62 @@ export class AwsS3UrlReader implements UrlReader { } } - async readTree(): Promise { - throw new Error('AwsS3Reader does not implement readTree'); + async readTree(url: string): Promise { + try { + const { path, bucket, region } = parseURL(url); + aws.config.update({ region: region }); + + let moreKeys = true; + let awsS3Readables: Readable[] = []; + let continuationToken = ''; + + while (moreKeys) { + let params; + if (continuationToken === '') { + params = { + Bucket: bucket, + Prefix: path, + }; + } else { + params = { + Bucket: bucket, + Prefix: path, + ContinuationToken: continuationToken, + }; + } + const { Contents, IsTruncated, NextContinuationToken } = await this.s3 + .listObjectsV2(params) + .promise(); + + const responses = await Promise.all( + (Contents || []).map(({ Key }) => { + const s3Response = this.s3 + .getObject({ Bucket: bucket, Key: String(Key) }) + .createReadStream(); + Object.defineProperty(s3Response, 'path', { + value: String(Key), + writable: false, + }); + return s3Response; + }), + ); + + if (IsTruncated) { + continuationToken = String(NextContinuationToken); + } else { + continuationToken = ''; + moreKeys = false; + } + awsS3Readables = awsS3Readables.concat(responses); + } + + return await this.treeResponseFactory.fromReadableArray({ + stream: awsS3Readables, + etag: '', + }); + } catch (e) { + throw new Error(`Could not retrieve file tree from S3: ${e.message}`); + } } async search(): Promise { diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 912fddf965..19d7ed2fab 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -20,9 +20,11 @@ import { ReadTreeResponse, FromArchiveOptions, ReadTreeResponseFactory, + FromReadableArrayOptions, } from '../types'; import { TarArchiveResponse } from './TarArchiveResponse'; import { ZipArchiveResponse } from './ZipArchiveResponse'; +import { ReadableArrayResponse } from './ReadableArrayResponse'; export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { static create(options: { config: Config }): DefaultReadTreeResponseFactory { @@ -53,4 +55,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { options.filter, ); } + + async fromReadableArray( + options: FromReadableArrayOptions, + ): Promise { + return new ReadableArrayResponse(options.stream, options.etag); + } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts new file mode 100644 index 0000000000..8b5d3ff0b5 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import getRawBody from 'raw-body'; +import { Readable } from 'stream'; +import { ReadTreeResponse, ReadTreeResponseFile } from '../types'; + +/** + * Wraps a array of Readable objects into a tree response reader. + */ +export class ReadableArrayResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: Readable[], + public readonly etag: string, + ) { + this.etag = etag; + } + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + async files(): Promise { + this.onlyOnce(); + + const files = Array(); + + for (let i = 0; i < this.stream.length; i++) { + if (!(this.stream[i] as any).path.endsWith('/')) { + files.push({ + path: (this.stream[i] as any).path, + content: () => getRawBody(this.stream[i]), + }); + } + } + + return files; + } + + archive(): Promise { + throw new Error('Method not implemented.'); + } + + dir(): Promise { + throw new Error('Method not implemented.'); + } +} diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 93f287d4fe..8d896408c3 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -167,9 +167,19 @@ export type FromArchiveOptions = { filter?: (path: string, info?: { size: number }) => boolean; }; +export type FromReadableArrayOptions = { + // An array of readable streams + stream: Readable[]; + // etag of the file tree + etag: string; +}; + export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; } /** From e32934750becba0bb369c60fea817398ba3dea17 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 16:46:42 -0700 Subject: [PATCH 27/61] More work Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.ts | 6 +- .../processors/AwsS3ReadTreeProcessor.ts | 74 +++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index da9e77d9ca..f7904ca70f 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -19,6 +19,7 @@ import { CredentialsOptions } from 'aws-sdk/lib/credentials'; import { ReaderFactory, ReadTreeResponse, + ReadTreeResponseFactory, ReadUrlOptions, ReadUrlResponse, SearchResponse, @@ -62,7 +63,7 @@ const parseURL = ( }; export class AwsS3UrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); return integrations.awsS3.list().map(integration => { @@ -71,7 +72,7 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3); + const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -81,6 +82,7 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, private readonly s3: S3, + private readonly treeResponseFactory: ReadTreeResponseFactory, ) {} /** diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts new file mode 100644 index 0000000000..8e65714a4c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import * as result from './results'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; + +export class AwsS3ReadTreeProcessor implements CatalogProcessor { + constructor(private readonly reader: UrlReader) {} + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise { + if (location.type !== 'aws-read-tree') { + return false; + } + + try { + const output = await this.doRead(location.target); + for (const item of output) { + for await (const parseResult of parser({ + data: item.data, + location: { type: location.type, target: item.url }, + })) { + emit(parseResult); + } + } + } catch (error) { + const message = `Unable to read ${location.type}, ${error}`; + + if (error.name === 'NotFoundError') { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + return true; + } + + private async doRead( + location: string, + ): Promise<{ data: Buffer; url: string }[]> { + const response = await this.reader.readTree(location); + const responseFiles = await response.files(); + const output = responseFiles.map(async file => ({ + url: file.path, + data: await file.content(), + })); + return Promise.all(output); + } +} From 0058a553ce22c28cf4e0f8fd5b8cb3d39d057e1f Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 17:24:14 -0700 Subject: [PATCH 28/61] Merge work done on previous branch Signed-off-by: Sean Tan --- app-config.yaml | 2 ++ .../backend-common/src/reading/AwsS3UrlReader.test.ts | 10 +++++++--- packages/backend/src/plugins/catalog.ts | 3 ++- .../catalog-backend/src/ingestion/processors/index.ts | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 9f365becef..fcfd92a66f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,6 +259,8 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml + - type: aws-read-tree + target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 084f7e3e13..78bfdbb0e2 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -26,14 +26,16 @@ import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + describe('AwsS3UrlReader', () => { const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { return AwsS3UrlReader.factory({ config: new ConfigReader(config), logger: getVoidLogger(), - treeResponseFactory: DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), - }), + treeResponseFactory, }); }; @@ -154,6 +156,7 @@ describe('AwsS3UrlReader', () => { ), ), s3, + treeResponseFactory, ); it('returns contents of an object in a bucket', async () => { @@ -207,6 +210,7 @@ describe('AwsS3UrlReader', () => { ), ), s3, + treeResponseFactory, ); it('returns contents of an object in a bucket', async () => { diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 055595dcb5..9c94b3851e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,6 +25,7 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); + builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index fe4d374829..1cfb9980f2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -19,6 +19,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; +export { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; From 45374f1be4d599f193351eda8bbd31ab88a75690 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 16 Aug 2021 17:14:47 -0700 Subject: [PATCH 29/61] Add ReadableArrayResponse tests Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 2 + .../{ => awsS3}/awsS3-mock-object.yaml | 0 .../awsS3/awsS3-mock-object2.yaml | 1 + .../tree/ReadableArrayResponse.test.ts | 62 +++++++++++++++++++ 4 files changed, 65 insertions(+) rename packages/backend-common/src/reading/__fixtures__/{ => awsS3}/awsS3-mock-object.yaml (100%) create mode 100644 packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml create mode 100644 packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 78bfdbb0e2..78cdcbb7b9 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -139,6 +139,7 @@ describe('AwsS3UrlReader', () => { 'src', 'reading', '__fixtures__', + 'awsS3', 'awsS3-mock-object.yaml', ), ), @@ -191,6 +192,7 @@ describe('AwsS3UrlReader', () => { 'src', 'reading', '__fixtures__', + 'awsS3', 'awsS3-mock-object.yaml', ), ), diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml similarity index 100% rename from packages/backend-common/src/reading/__fixtures__/awsS3-mock-object.yaml rename to packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object.yaml diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml new file mode 100644 index 0000000000..b2c132961b --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object2.yaml @@ -0,0 +1 @@ +site_name: Test2 diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts new file mode 100644 index 0000000000..c2e3c26269 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ReadableArrayResponse } from './ReadableArrayResponse'; +import path from 'path'; +import { Readable } from 'stream'; +import fs from 'fs'; + +const arr: Readable[] = []; +const file1 = path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', +); +const file2 = path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object2.yaml', +); +const stream1 = fs.createReadStream(file1); +const stream2 = fs.createReadStream(file2); +arr.push(stream1); +arr.push(stream2); + +describe('ReadableArrayResponse', () => { + it('should read files', async () => { + const res = new ReadableArrayResponse(arr, 'etag'); + const files = await res.files(); + + expect(files).toEqual([ + { + path: file1, + content: expect.any(Function), + }, + { + path: file2, + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + 'site_name: Test2', + ]); + }); +}); From 38ba6e9294a9c31c54c0d443d4f3e4e2e987ef1f Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 17 Aug 2021 19:38:27 -0700 Subject: [PATCH 30/61] Finish AwsS3ReadTreeProcessor tests Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 51 ++++++++++ packages/backend-common/src/reading/index.ts | 1 + plugins/catalog-backend/package.json | 6 ++ .../processors/AwsS3ReadTreeProcessor.test.ts | 97 +++++++++++++++++++ .../awsS3/awsS3-mock-object.yaml | 1 + 5 files changed, 156 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 78cdcbb7b9..9be6de7a30 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -235,4 +235,55 @@ describe('AwsS3UrlReader', () => { ); }); }); + describe('readTree', () => { + const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', + }; + const objectList: aws.S3.ObjectList = [object]; + const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, + }; + AWSMock.setSDKInstance(aws); + AWSMock.mock('S3', 'listObjectsV2', output); + + AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'reading', + '__fixtures__', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), + ); + + const s3 = new aws.S3(); + const awsS3UrlReader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: '.amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + treeResponseFactory, + ); + it('returns contents of an object in a bucket', async () => { + const response = await awsS3UrlReader.readTree( + 'https://test.s3.us-east-2.amazonaws.com', + ); + const files = await response.files(); + const body = await files[0].content(); + + expect(body.toString()).toBe('site_name: Test\n'); + }); + }); }); diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 207f01b37f..e0e77963f2 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -18,6 +18,7 @@ export { AzureUrlReader } from './AzureUrlReader'; export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; +export { AwsS3UrlReader } from './AwsS3UrlReader'; export type { ReaderFactory, ReadTreeOptions, diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 7f5ec0b2d6..15a7e65570 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -63,8 +63,14 @@ "yup": "^0.29.3" }, "devDependencies": { +<<<<<<< HEAD "@backstage/backend-test-utils": "^0.1.6", "@backstage/cli": "^0.7.9", +======= + "aws-sdk-mock": "^5.2.1", + "@backstage/backend-test-utils": "^0.1.5", + "@backstage/cli": "^0.7.8", +>>>>>>> Finish AwsS3ReadTreeProcessor tests "@backstage/test-utils": "^0.1.17", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts new file mode 100644 index 0000000000..6b99506e26 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + AwsS3UrlReader, + DefaultReadTreeResponseFactory, +} from '@backstage/backend-common'; + +import { ConfigReader } from '@backstage/config'; +import { + AwsS3Integration, + readAwsS3IntegrationConfig, +} from '@backstage/integration'; +import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; +import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; +import { defaultEntityDataParser } from './util/parse'; +import AWSMock from 'aws-sdk-mock'; +import aws from 'aws-sdk'; +import path from 'path'; + +const treeResponseFactory = DefaultReadTreeResponseFactory.create({ + config: new ConfigReader({}), +}); + +AWSMock.setSDKInstance(aws); +const object: aws.S3.Types.Object = { + Key: 'awsS3-mock-object.yaml', +}; +const objectList: aws.S3.ObjectList = [object]; +const output: aws.S3.Types.ListObjectsV2Output = { + Contents: objectList, +}; +AWSMock.mock('S3', 'listObjectsV2', output); +AWSMock.mock( + 'S3', + 'getObject', + Buffer.from( + require('fs').readFileSync( + path.resolve( + 'src', + 'ingestion', + 'processors', + '__fixtures__', + 'fileReaderProcessor', + 'awsS3', + 'awsS3-mock-object.yaml', + ), + ), + ), +); + +const s3 = new aws.S3(); +const reader = new AwsS3UrlReader( + new AwsS3Integration( + readAwsS3IntegrationConfig( + new ConfigReader({ + host: 'amazonaws.com', + accessKeyId: 'fake-access-key', + secretAccessKey: 'fake-secret-key', + }), + ), + ), + s3, + treeResponseFactory, +); + +describe('readLocation', () => { + const processor = new AwsS3ReadTreeProcessor(reader); + const spec = { + type: 'aws-read-tree', + target: 'https://testbucket.s3.us-east-2.amazonaws.com', + }; + + it('should load from url', async () => { + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit, defaultEntityDataParser), + )) as CatalogProcessorEntityResult; + expect(generated.type).toBe('entity'); + expect(generated.location).toEqual({ + target: 'awsS3-mock-object.yaml', + type: 'aws-read-tree', + }); + expect(generated.entity).toEqual({ site_name: 'Test' }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml @@ -0,0 +1 @@ +site_name: Test From 0b45dece0980ff8ebda48ac2de07266b40599812 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:28:27 -0700 Subject: [PATCH 31/61] Preparing for PR Signed-off-by: Sean Tan --- docs/integrations/aws-s3/readtree.md | 28 +++++++++++++++++++ microsite/sidebars.json | 2 +- mkdocs.yml | 1 + packages/backend-common/src/index.ts | 1 + .../src/reading/AwsS3UrlReader.ts | 1 - 5 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 docs/integrations/aws-s3/readtree.md diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md new file mode 100644 index 0000000000..7ae1fece63 --- /dev/null +++ b/docs/integrations/aws-s3/readtree.md @@ -0,0 +1,28 @@ +--- +id: read-tree +title: AWS S3 Read Tree +sidebar_label: Read Tree +# prettier-ignore +description: Discovering multiple catalog entities from an AWS S3 Bucket +--- + +The AWS S3 integration has a special processor for using AwsS3UrlReader's +readTree() method. The method readTree() allows a user to retrieve more than one +file hosted inside an S3 bucket. The processor passes a valid S3 URL and +credentials to AwsS3UrlReader's readTree(). A user can supply a URL which points +to root of a bucket or a subdirectory inside the bucket. This can be useful as +an alternative to static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need an AWS S3 integration +[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and +optionally a `roleArn`. Then you can add a location target to the catalog +configuration: + +```yaml +catalog: + locations: + - type: aws-read-tree + target: https://sample-bucket.s3.us-east-2.amazonaws.com/ +``` + +Note the `aws-read-tree` type, as this is not a regular `url` processor. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index d787fb28aa..3517c8d70c 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -107,7 +107,7 @@ { "type": "subcategory", "label": "AWS S3", - "ids": ["integrations/aws-s3/locations"] + "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/readtree"] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 73ad52341e..3b4dad765c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - Overview: 'integrations/index.md' - AWS S3: - Locations: 'integrations/aws-s3/locations.md' + - Read Tree: 'integrations/aws-s3/readtree.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index f2f38d9aab..5b37c2fb9f 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -23,6 +23,7 @@ export * from './logging'; export * from './middleware'; export * from './paths'; export * from './reading'; +export * from './reading/tree'; export * from './scm'; export * from './service'; export * from './util'; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index f7904ca70f..615a095e50 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -187,7 +187,6 @@ export class AwsS3UrlReader implements UrlReader { const { Contents, IsTruncated, NextContinuationToken } = await this.s3 .listObjectsV2(params) .promise(); - const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3 From c6d019d00758009bd4bf04c9a21d7e836deb81d9 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:40:25 -0700 Subject: [PATCH 32/61] Add readTree() to AwsS3UrlReader and AwsS3ReadTreeProcessor Signed-off-by: Sean Tan --- .changeset/shiny-emus-perform.md | 7 +++++++ app-config.yaml | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/shiny-emus-perform.md diff --git a/.changeset/shiny-emus-perform.md b/.changeset/shiny-emus-perform.md new file mode 100644 index 0000000000..76a40504c8 --- /dev/null +++ b/.changeset/shiny-emus-perform.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +'@backstage/plugin-catalog-backend': patch +--- + +Add readTree() to AwsS3UrlReader and aws-read-tree Catalog Processor diff --git a/app-config.yaml b/app-config.yaml index fcfd92a66f..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,8 +259,6 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml - - type: aws-read-tree - target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created From 8a6154110aa4fe4d65390b5bb8f92a8c26800463 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:46:48 -0700 Subject: [PATCH 33/61] Remove local changes made to catalog.ts Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 9c94b3851e..75709eefc7 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,7 +24,6 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); - builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, From d4346f3ff9cae85f4a509f6b8f73e81d5614d1ef Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:49:11 -0700 Subject: [PATCH 34/61] Fix one more error in catalog.ts Signed-off-by: Sean Tan Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 75709eefc7..055595dcb5 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { CatalogBuilder, createRouter, From 6b45ebac7f9fdacac4897fcf8e9a97c09439548d Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 15:51:31 -0700 Subject: [PATCH 35/61] Regenerate api reports Signed-off-by: Sean Tan --- packages/backend-common/api-report.md | 52 +++++++++++++++++++++++++++ plugins/catalog-backend/api-report.md | 14 ++++++++ 2 files changed, 66 insertions(+) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index de8c1d4101..1c1b66042b 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -6,6 +6,7 @@ /// /// +import { AwsS3Integration } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BitbucketIntegration } from '@backstage/integration'; import { Config } from '@backstage/config'; @@ -28,10 +29,40 @@ import { Readable } from 'stream'; import { ReadCommitResult } from 'isomorphic-git'; import { RequestHandler } from 'express'; import { Router } from 'express'; +import { S3 } from 'aws-sdk'; import { Server } from 'http'; import * as winston from 'winston'; import { Writable } from 'stream'; +// Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3UrlReader implements UrlReader { + // Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts + constructor( + integration: AwsS3Integration, + s3: S3, + treeResponseFactory: ReadTreeResponseFactory, + ); + // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(url: string): Promise; + // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts + // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts + // + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // Warning: (ae-missing-release-tag) "AzureUrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -164,6 +195,27 @@ export class DatabaseManager { static fromConfig(config: Config): DatabaseManager; } +// Warning: (ae-missing-release-tag) "DefaultReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { + constructor(workDir: string); + // (undocumented) + static create(options: { config: Config }): DefaultReadTreeResponseFactory; + // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; + // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromTarArchive(options: FromArchiveOptions): Promise; + // (undocumented) + fromZipArchive(options: FromArchiveOptions): Promise; +} + // Warning: (ae-missing-release-tag) "DockerContainerRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a7a100db65..f2dbd80464 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -121,6 +121,20 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "AwsS3ReadTreeProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AwsS3ReadTreeProcessor implements CatalogProcessor { + constructor(reader: UrlReader); + // (undocumented) + readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise; +} + // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From 6575f2359cfd25b13b2a3a741453abcc04c10b61 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 16:30:05 -0700 Subject: [PATCH 36/61] First go Signed-off-by: Sean Tan --- packages/backend-common/src/reading/AwsS3UrlReader.ts | 4 ++++ packages/backend-common/src/reading/types.ts | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 615a095e50..6c2d55972b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -187,6 +187,10 @@ export class AwsS3UrlReader implements UrlReader { const { Contents, IsTruncated, NextContinuationToken } = await this.s3 .listObjectsV2(params) .promise(); +<<<<<<< HEAD +======= + +>>>>>>> First go const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3 diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 8d896408c3..448e678435 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -174,6 +174,13 @@ export type FromReadableArrayOptions = { etag: string; }; +export type FromReadableArrayOptions = { + // An array of readable streams + stream: Readable[]; + // etag of the file tree + etag: string; +}; + export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; From d88281b81b6d12a9cf82f242d3fbfef42d164432 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Fri, 13 Aug 2021 17:24:14 -0700 Subject: [PATCH 37/61] Merge work done on previous branch Signed-off-by: Sean Tan --- app-config.yaml | 2 ++ packages/backend/src/plugins/catalog.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index 9f365becef..fcfd92a66f 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,6 +259,8 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml + - type: aws-read-tree + target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 055595dcb5..9c94b3851e 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,6 +25,7 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); + builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, From 00c7f4e0c4c05f7c5c3128b308112bc8cb0b8bf9 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:28:27 -0700 Subject: [PATCH 38/61] Preparing for PR Signed-off-by: Sean Tan --- packages/backend-common/src/reading/AwsS3UrlReader.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 6c2d55972b..a512d7f64b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -188,9 +188,12 @@ export class AwsS3UrlReader implements UrlReader { .listObjectsV2(params) .promise(); <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> First go +======= +>>>>>>> Preparing for PR const responses = await Promise.all( (Contents || []).map(({ Key }) => { const s3Response = this.s3 From b1d9838e86da0c101f62aa405db79dd4539757fa Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:40:25 -0700 Subject: [PATCH 39/61] Add readTree() to AwsS3UrlReader and AwsS3ReadTreeProcessor Signed-off-by: Sean Tan --- app-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index fcfd92a66f..9f365becef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -259,8 +259,6 @@ catalog: # Backstage example groups and users - type: file target: ../catalog-model/examples/acme-corp.yaml - - type: aws-read-tree - target: https://seant-splunk.s3.us-east-2.amazonaws.com scaffolder: # Use to customize default commit author info used when new components are created From ba25f092daf82e05fac0107f5116503fe910d984 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:46:48 -0700 Subject: [PATCH 40/61] Remove local changes made to catalog.ts Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 9c94b3851e..75709eefc7 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AwsS3ReadTreeProcessor } from '/Users/seantan/backstage/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor'; import { CatalogBuilder, createRouter, @@ -25,7 +24,6 @@ export default async function createPlugin( env: PluginEnvironment, ): Promise { const builder = await CatalogBuilder.create(env); - builder.addProcessor(new AwsS3ReadTreeProcessor(env.reader)); const { entitiesCatalog, locationAnalyzer, From c25cdd9cf79236a11a05db003165cb6515d32b95 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 14:49:11 -0700 Subject: [PATCH 41/61] Fix one more error in catalog.ts Signed-off-by: Sean Tan Signed-off-by: Sean Tan --- packages/backend/src/plugins/catalog.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 75709eefc7..055595dcb5 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { CatalogBuilder, createRouter, From 3edc024f4927a54fd0ab1662937ae62231863e81 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 15:57:23 -0700 Subject: [PATCH 42/61] Duplicate entry Signed-off-by: Sean Tan --- packages/backend-common/src/reading/types.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index 448e678435..c0acc04701 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -173,14 +173,6 @@ export type FromReadableArrayOptions = { // etag of the file tree etag: string; }; - -export type FromReadableArrayOptions = { - // An array of readable streams - stream: Readable[]; - // etag of the file tree - etag: string; -}; - export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; From 670aa2f9895ed9aeb4fac94526af65d0f79f465e Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 18 Aug 2021 16:00:31 -0700 Subject: [PATCH 43/61] Remove random file change Signed-off-by: Sean Tan --- .../software-templates/migrating-from-v1alpha1-to-v1beta2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md index 28001cce0b..25ff2a2f3a 100644 --- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md +++ b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md @@ -325,7 +325,7 @@ spec: output: links: - url: '{{steps.publish.output.remoteUrl}}' - title: 'Go to Repo' + text: 'Go to Repo' ``` ## Questions? From 5ee96032fe970dc4949b7143fab8ca71a6c002d3 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 23 Aug 2021 12:57:42 -0700 Subject: [PATCH 44/61] Begin changes on PR Signed-off-by: Sean Tan --- docs/integrations/aws-s3/readtree.md | 4 +- packages/backend-common/src/index.ts | 1 - .../src/reading/AwsS3UrlReader.ts | 29 +++++++------- .../processors/AwsS3ReadTreeProcessor.test.ts | 39 +++++-------------- .../processors/AwsS3ReadTreeProcessor.ts | 2 +- 5 files changed, 26 insertions(+), 49 deletions(-) diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md index 7ae1fece63..47ce867137 100644 --- a/docs/integrations/aws-s3/readtree.md +++ b/docs/integrations/aws-s3/readtree.md @@ -21,8 +21,8 @@ configuration: ```yaml catalog: locations: - - type: aws-read-tree + - type: s3-bucket target: https://sample-bucket.s3.us-east-2.amazonaws.com/ ``` -Note the `aws-read-tree` type, as this is not a regular `url` processor. +Note the `s3-bucket` type, as this is not a regular `url` processor. diff --git a/packages/backend-common/src/index.ts b/packages/backend-common/src/index.ts index 5b37c2fb9f..f2f38d9aab 100644 --- a/packages/backend-common/src/index.ts +++ b/packages/backend-common/src/index.ts @@ -23,7 +23,6 @@ export * from './logging'; export * from './middleware'; export * from './paths'; export * from './reading'; -export * from './reading/tree'; export * from './scm'; export * from './service'; export * from './util'; diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index a512d7f64b..593a66cfcc 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -72,7 +72,10 @@ export class AwsS3UrlReader implements UrlReader { apiVersion: '2006-03-01', credentials: creds, }); - const reader = new AwsS3UrlReader(integration, s3, treeResponseFactory); + const reader = new AwsS3UrlReader(integration, { + s3, + treeResponseFactory, + }); const predicate = (url: URL) => url.host.endsWith(integration.config.host); return { reader, predicate }; @@ -81,8 +84,10 @@ export class AwsS3UrlReader implements UrlReader { constructor( private readonly integration: AwsS3Integration, - private readonly s3: S3, - private readonly treeResponseFactory: ReadTreeResponseFactory, + private readonly deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, ) {} /** @@ -148,7 +153,7 @@ export class AwsS3UrlReader implements UrlReader { }; } - const response = this.s3.getObject(params); + const response = this.deps.s3.getObject(params); const buffer = await getRawBody(response.createReadStream()); const etag = (await response.promise()).ETag; @@ -184,19 +189,11 @@ export class AwsS3UrlReader implements UrlReader { ContinuationToken: continuationToken, }; } - const { Contents, IsTruncated, NextContinuationToken } = await this.s3 - .listObjectsV2(params) - .promise(); -<<<<<<< HEAD -<<<<<<< HEAD -======= - ->>>>>>> First go -======= ->>>>>>> Preparing for PR + const { Contents, IsTruncated, NextContinuationToken } = + await this.deps.s3.listObjectsV2(params).promise(); const responses = await Promise.all( (Contents || []).map(({ Key }) => { - const s3Response = this.s3 + const s3Response = this.deps.s3 .getObject({ Bucket: bucket, Key: String(Key) }) .createReadStream(); Object.defineProperty(s3Response, 'path', { @@ -216,7 +213,7 @@ export class AwsS3UrlReader implements UrlReader { awsS3Readables = awsS3Readables.concat(responses); } - return await this.treeResponseFactory.fromReadableArray({ + return await this.deps.treeResponseFactory.fromReadableArray({ stream: awsS3Readables, etag: '', }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts index 6b99506e26..0b8ea6f814 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -13,16 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - AwsS3UrlReader, - DefaultReadTreeResponseFactory, -} from '@backstage/backend-common'; - +import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { - AwsS3Integration, - readAwsS3IntegrationConfig, -} from '@backstage/integration'; import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; import { defaultEntityDataParser } from './util/parse'; @@ -30,10 +22,6 @@ import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; -const treeResponseFactory = DefaultReadTreeResponseFactory.create({ - config: new ConfigReader({}), -}); - AWSMock.setSDKInstance(aws); const object: aws.S3.Types.Object = { Key: 'awsS3-mock-object.yaml', @@ -61,25 +49,18 @@ AWSMock.mock( ), ); -const s3 = new aws.S3(); -const reader = new AwsS3UrlReader( - new AwsS3Integration( - readAwsS3IntegrationConfig( - new ConfigReader({ - host: 'amazonaws.com', - accessKeyId: 'fake-access-key', - secretAccessKey: 'fake-secret-key', - }), - ), - ), - s3, - treeResponseFactory, -); +const logger = getVoidLogger(); +const reader = UrlReaders.default({ + logger, + config: new ConfigReader({ + backend: { reading: { allow: [{ host: 'localhost' }] } }, + }), +}); describe('readLocation', () => { const processor = new AwsS3ReadTreeProcessor(reader); const spec = { - type: 'aws-read-tree', + type: 's3-bucket', target: 'https://testbucket.s3.us-east-2.amazonaws.com', }; @@ -90,7 +71,7 @@ describe('readLocation', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ target: 'awsS3-mock-object.yaml', - type: 'aws-read-tree', + type: 's3-bucket', }); expect(generated.entity).toEqual({ site_name: 'Test' }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts index 8e65714a4c..9d8e1cdce5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts @@ -32,7 +32,7 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'aws-read-tree') { + if (location.type !== 's3-bucket') { return false; } From 85039c1e69140fff84058bd098f92b21c4077f0c Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 24 Aug 2021 17:29:21 -0700 Subject: [PATCH 45/61] Finish ReadableArrayResponse changes Signed-off-by: Sean Tan --- .../src/reading/AwsS3UrlReader.test.ts | 9 ++-- .../reading/tree/ReadTreeResponseFactory.ts | 6 ++- .../tree/ReadableArrayResponse.test.ts | 34 +++++++++--- .../src/reading/tree/ReadableArrayResponse.ts | 54 ++++++++++++++++--- 4 files changed, 83 insertions(+), 20 deletions(-) diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts index 9be6de7a30..736248b10b 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.test.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.test.ts @@ -156,8 +156,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -211,8 +210,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { @@ -273,8 +271,7 @@ describe('AwsS3UrlReader', () => { }), ), ), - s3, - treeResponseFactory, + { s3, treeResponseFactory }, ); it('returns contents of an object in a bucket', async () => { const response = await awsS3UrlReader.readTree( diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 19d7ed2fab..7d20385635 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -59,6 +59,10 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { async fromReadableArray( options: FromReadableArrayOptions, ): Promise { - return new ReadableArrayResponse(options.stream, options.etag); + return new ReadableArrayResponse( + options.stream, + this.workDir, + options.etag, + ); } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index c2e3c26269..bf457f02e4 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -14,11 +14,13 @@ * limitations under the License. */ import { ReadableArrayResponse } from './ReadableArrayResponse'; -import path from 'path'; +import path, { resolve as resolvePath } from 'path'; + import { Readable } from 'stream'; -import fs from 'fs'; +import fs from 'fs-extra'; const arr: Readable[] = []; +const arr2: Readable[] = []; const file1 = path.resolve( 'src', 'reading', @@ -33,14 +35,15 @@ const file2 = path.resolve( 'awsS3', 'awsS3-mock-object2.yaml', ); -const stream1 = fs.createReadStream(file1); -const stream2 = fs.createReadStream(file2); -arr.push(stream1); -arr.push(stream2); describe('ReadableArrayResponse', () => { it('should read files', async () => { - const res = new ReadableArrayResponse(arr, 'etag'); + const stream1 = fs.createReadStream(file1); + const stream2 = fs.createReadStream(file2); + arr.push(stream1); + arr.push(stream2); + + const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -59,4 +62,21 @@ describe('ReadableArrayResponse', () => { 'site_name: Test2', ]); }); + + it('should extract entire archive into directory', async () => { + const stream1 = fs.createReadStream(file1); + const stream2 = fs.createReadStream(file2); + + arr2.push(stream1); + arr2.push(stream2); + + const res = new ReadableArrayResponse(arr2, '/tmp', 'etag'); + const dir = await res.dir(); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'awsS3-mock-object2.yaml'), 'utf8'), + ).resolves.toBe('site_name: Test2\n'); + }); }); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index 8b5d3ff0b5..bf74a19603 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -14,9 +14,21 @@ * limitations under the License. */ +import concatStream from 'concat-stream'; +import platformPath, { basename } from 'path'; + import getRawBody from 'raw-body'; -import { Readable } from 'stream'; -import { ReadTreeResponse, ReadTreeResponseFile } from '../types'; +import fs from 'fs-extra'; +import { promisify } from 'util'; +import tar from 'tar'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import { + ReadTreeResponse, + ReadTreeResponseFile, + ReadTreeResponseDirOptions, +} from '../types'; + +const pipeline = promisify(pipelineCb); /** * Wraps a array of Readable objects into a tree response reader. @@ -26,6 +38,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { constructor( private readonly stream: Readable[], + private readonly workDir: string, public readonly etag: string, ) { this.etag = etag; @@ -56,11 +69,40 @@ export class ReadableArrayResponse implements ReadTreeResponse { return files; } - archive(): Promise { - throw new Error('Method not implemented.'); + async archive(): Promise { + const tmpDir = await this.dir(); + + try { + const data = await new Promise(async resolve => { + await pipeline( + tar.create({ cwd: tmpDir }, ['']), + concatStream(resolve), + ); + }); + return Readable.from(data); + } finally { + await fs.remove(tmpDir); + } } - dir(): Promise { - throw new Error('Method not implemented.'); + async dir(options?: ReadTreeResponseDirOptions): Promise { + this.onlyOnce(); + + const dir = + options?.targetDir ?? + (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); + + for (let i = 0; i < this.stream.length; i++) { + if (!(this.stream[i] as any).path.endsWith('/')) { + await pipeline( + this.stream[i], + fs.createWriteStream( + platformPath.join(dir, basename((this.stream[i] as any).path)), + ), + ); + } + } + + return dir; } } From 8b6e30e62266832405628ee18988aa28a1ed79e8 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 24 Aug 2021 18:39:08 -0700 Subject: [PATCH 46/61] Finish up changes Signed-off-by: Sean Tan --- packages/backend-common/api-report.md | 29 ++++--------------- .../src/reading/AwsS3UrlReader.ts | 2 +- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 1c1b66042b..65f5cc0410 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -38,11 +38,12 @@ import { Writable } from 'stream'; // // @public (undocumented) export class AwsS3UrlReader implements UrlReader { - // Warning: (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts constructor( integration: AwsS3Integration, - s3: S3, - treeResponseFactory: ReadTreeResponseFactory, + deps: { + s3: S3; + treeResponseFactory: ReadTreeResponseFactory; + }, ); // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts // @@ -195,27 +196,6 @@ export class DatabaseManager { static fromConfig(config: Config): DatabaseManager; } -// Warning: (ae-missing-release-tag) "DefaultReadTreeResponseFactory" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { - constructor(workDir: string); - // (undocumented) - static create(options: { config: Config }): DefaultReadTreeResponseFactory; - // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts - // - // (undocumented) - fromReadableArray( - options: FromReadableArrayOptions, - ): Promise; - // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts - // - // (undocumented) - fromTarArchive(options: FromArchiveOptions): Promise; - // (undocumented) - fromZipArchive(options: FromArchiveOptions): Promise; -} - // Warning: (ae-missing-release-tag) "DockerContainerRunner" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -699,6 +679,7 @@ export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; // // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here +// src/reading/AwsS3UrlReader.d.ts:11:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts // src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 593a66cfcc..c48d16cc69 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -169,7 +169,6 @@ export class AwsS3UrlReader implements UrlReader { async readTree(url: string): Promise { try { const { path, bucket, region } = parseURL(url); - aws.config.update({ region: region }); let moreKeys = true; let awsS3Readables: Readable[] = []; @@ -189,6 +188,7 @@ export class AwsS3UrlReader implements UrlReader { ContinuationToken: continuationToken, }; } + aws.config.update({ region: region }); const { Contents, IsTruncated, NextContinuationToken } = await this.deps.s3.listObjectsV2(params).promise(); const responses = await Promise.all( From adf71877778f6c381acad6d38e01ccd652b2f8cc Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 24 Aug 2021 19:34:48 -0700 Subject: [PATCH 47/61] Change FileReaderProcessor tests to reflect new test file 'aws-s3-mock-object.yaml' Signed-off-by: Sean Tan --- .../src/ingestion/processors/FileReaderProcessor.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts index 41c7e4f732..c1dfbb0af7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts @@ -77,7 +77,7 @@ describe('FileReaderProcessor', () => { emit, ); - expect(emit).toBeCalledTimes(2); + expect(emit).toBeCalledTimes(3); expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', From b129e0214abc622fb6bbd7858464e0f9d7b3b9b7 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 25 Aug 2021 10:48:27 -0700 Subject: [PATCH 48/61] Change AwsS3ReadTreeProcessor tests as to not interfere with FileReaderProcessor tests Signed-off-by: Sean Tan --- .../src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts | 6 +++--- .../src/ingestion/processors/FileReaderProcessor.test.ts | 2 +- .../awsS3/{awsS3-mock-object.yaml => awsS3-mock-object.txt} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/{awsS3-mock-object.yaml => awsS3-mock-object.txt} (100%) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts index 0b8ea6f814..ebc258a29e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts @@ -24,7 +24,7 @@ import path from 'path'; AWSMock.setSDKInstance(aws); const object: aws.S3.Types.Object = { - Key: 'awsS3-mock-object.yaml', + Key: 'awsS3-mock-object.txt', }; const objectList: aws.S3.ObjectList = [object]; const output: aws.S3.Types.ListObjectsV2Output = { @@ -43,7 +43,7 @@ AWSMock.mock( '__fixtures__', 'fileReaderProcessor', 'awsS3', - 'awsS3-mock-object.yaml', + 'awsS3-mock-object.txt', ), ), ), @@ -70,7 +70,7 @@ describe('readLocation', () => { )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ - target: 'awsS3-mock-object.yaml', + target: 'awsS3-mock-object.txt', type: 's3-bucket', }); expect(generated.entity).toEqual({ site_name: 'Test' }); diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts index c1dfbb0af7..41c7e4f732 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts @@ -77,7 +77,7 @@ describe('FileReaderProcessor', () => { emit, ); - expect(emit).toBeCalledTimes(3); + expect(emit).toBeCalledTimes(2); expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' }); expect(emit.mock.calls[0][0].location).toEqual({ type: 'file', diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.yaml rename to plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt From 0f485962267b32cb072ecb94d7a88a36a36bfa4e Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Thu, 26 Aug 2021 10:31:09 -0700 Subject: [PATCH 49/61] Resolve conflict with package.json Signed-off-by: Sean Tan --- plugins/catalog-backend/package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 15a7e65570..9be4333a35 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -68,9 +68,14 @@ "@backstage/cli": "^0.7.9", ======= "aws-sdk-mock": "^5.2.1", +<<<<<<< HEAD "@backstage/backend-test-utils": "^0.1.5", "@backstage/cli": "^0.7.8", >>>>>>> Finish AwsS3ReadTreeProcessor tests +======= + "@backstage/backend-test-utils": "^0.1.6", + "@backstage/cli": "^0.7.9", +>>>>>>> Resolve conflict with package.json "@backstage/test-utils": "^0.1.17", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", From d399742c152798fbec91cc2ad34ea74fdcb8582c Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Thu, 26 Aug 2021 10:38:09 -0700 Subject: [PATCH 50/61] Copy package.json from master and add 'aws-sdk-mock' to devDependencies Signed-off-by: Sean Tan --- plugins/catalog-backend/package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9be4333a35..07b0b09f0f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -40,6 +40,7 @@ "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "aws-sdk": "^2.840.0", + "aws-sdk-mock": "^5.2.1", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", @@ -63,6 +64,7 @@ "yup": "^0.29.3" }, "devDependencies": { +<<<<<<< HEAD <<<<<<< HEAD "@backstage/backend-test-utils": "^0.1.6", "@backstage/cli": "^0.7.9", @@ -73,6 +75,8 @@ "@backstage/cli": "^0.7.8", >>>>>>> Finish AwsS3ReadTreeProcessor tests ======= +======= +>>>>>>> Copy package.json from master and add 'aws-sdk-mock' to devDependencies "@backstage/backend-test-utils": "^0.1.6", "@backstage/cli": "^0.7.9", >>>>>>> Resolve conflict with package.json From 7177a9a8226a0ecd2b119d38c5f67b376d16a338 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 31 Aug 2021 12:53:29 -0700 Subject: [PATCH 51/61] Update PR to reflect change requests Signed-off-by: Sean Tan --- .changeset/shiny-emus-perform.md | 7 -- .../migrating-from-v1alpha1-to-v1beta2.md | 2 +- docs/integrations/aws-s3/discovery.md | 28 +++++++ docs/integrations/aws-s3/readtree.md | 28 ------- microsite/sidebars.json | 2 +- mkdocs.yml | 2 +- .../src/reading/AwsS3UrlReader.ts | 75 ++++++++----------- .../reading/tree/ReadTreeResponseFactory.ts | 6 +- .../tree/ReadableArrayResponse.test.ts | 31 ++++---- .../src/reading/tree/ReadableArrayResponse.ts | 15 ++-- packages/backend-common/src/reading/types.ts | 12 +-- plugins/catalog-backend/package.json | 2 +- ...est.ts => AwsS3DiscoveryProcessor.test.ts} | 8 +- ...rocessor.ts => AwsS3DiscoveryProcessor.ts} | 8 +- .../src/ingestion/processors/index.ts | 2 +- 15 files changed, 102 insertions(+), 126 deletions(-) delete mode 100644 .changeset/shiny-emus-perform.md create mode 100644 docs/integrations/aws-s3/discovery.md delete mode 100644 docs/integrations/aws-s3/readtree.md rename plugins/catalog-backend/src/ingestion/processors/{AwsS3ReadTreeProcessor.test.ts => AwsS3DiscoveryProcessor.test.ts} (92%) rename plugins/catalog-backend/src/ingestion/processors/{AwsS3ReadTreeProcessor.ts => AwsS3DiscoveryProcessor.ts} (89%) diff --git a/.changeset/shiny-emus-perform.md b/.changeset/shiny-emus-perform.md deleted file mode 100644 index 76a40504c8..0000000000 --- a/.changeset/shiny-emus-perform.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@backstage/backend-common': patch -'@backstage/integration': patch -'@backstage/plugin-catalog-backend': patch ---- - -Add readTree() to AwsS3UrlReader and aws-read-tree Catalog Processor diff --git a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md index 25ff2a2f3a..28001cce0b 100644 --- a/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md +++ b/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2.md @@ -325,7 +325,7 @@ spec: output: links: - url: '{{steps.publish.output.remoteUrl}}' - text: 'Go to Repo' + title: 'Go to Repo' ``` ## Questions? diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md new file mode 100644 index 0000000000..6e1827f395 --- /dev/null +++ b/docs/integrations/aws-s3/discovery.md @@ -0,0 +1,28 @@ +--- +id: discovery +title: AWS S3 Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from an AWS S3 Bucket +--- + +The AWS S3 integration has a special discovery processor for discovering catalog +entities located in an S3 Bucket. If you have a bucket that contains multiple +catalog-info files and want to automatically discover them, you can use this +processor. The processor will crawl your S3 bucket and register entities +matching the configured path. This can be useful as an alternative to static +locations or manually adding things to the catalog. + +To use the discovery processor, you'll need an AWS S3 integration +[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and +optionally a `roleArn`. Then you can add a location target to the catalog +configuration: + +```yaml +catalog: + locations: + - type: awsS3-discovery + target: https://sample-bucket.s3.us-east-2.amazonaws.com/ +``` + +Note the `awsS3-discovery` type, as this is not a regular `url` processor. diff --git a/docs/integrations/aws-s3/readtree.md b/docs/integrations/aws-s3/readtree.md deleted file mode 100644 index 47ce867137..0000000000 --- a/docs/integrations/aws-s3/readtree.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -id: read-tree -title: AWS S3 Read Tree -sidebar_label: Read Tree -# prettier-ignore -description: Discovering multiple catalog entities from an AWS S3 Bucket ---- - -The AWS S3 integration has a special processor for using AwsS3UrlReader's -readTree() method. The method readTree() allows a user to retrieve more than one -file hosted inside an S3 bucket. The processor passes a valid S3 URL and -credentials to AwsS3UrlReader's readTree(). A user can supply a URL which points -to root of a bucket or a subdirectory inside the bucket. This can be useful as -an alternative to static locations or manually adding things to the catalog. - -To use the discovery processor, you'll need an AWS S3 integration -[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and -optionally a `roleArn`. Then you can add a location target to the catalog -configuration: - -```yaml -catalog: - locations: - - type: s3-bucket - target: https://sample-bucket.s3.us-east-2.amazonaws.com/ -``` - -Note the `s3-bucket` type, as this is not a regular `url` processor. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 3517c8d70c..56df6b4b50 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -107,7 +107,7 @@ { "type": "subcategory", "label": "AWS S3", - "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/readtree"] + "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/discovery"] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 3b4dad765c..6840fee68a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,7 +81,7 @@ nav: - Overview: 'integrations/index.md' - AWS S3: - Locations: 'integrations/aws-s3/locations.md' - - Read Tree: 'integrations/aws-s3/readtree.md' + - Discovery: 'integrations/aws-s3/discovery.md' - Azure: - Locations: 'integrations/azure/locations.md' - Org Data: 'integrations/azure/org.md' diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index c48d16cc69..53bf99b307 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -27,7 +27,7 @@ import { } from './types'; import getRawBody from 'raw-body'; import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; -import { Readable } from 'stream'; +import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3'; const parseURL = ( url: string, @@ -169,54 +169,39 @@ export class AwsS3UrlReader implements UrlReader { async readTree(url: string): Promise { try { const { path, bucket, region } = parseURL(url); - - let moreKeys = true; - let awsS3Readables: Readable[] = []; - let continuationToken = ''; - - while (moreKeys) { - let params; - if (continuationToken === '') { - params = { - Bucket: bucket, - Prefix: path, - }; - } else { - params = { - Bucket: bucket, - Prefix: path, - ContinuationToken: continuationToken, - }; - } + const allObjects: ObjectList = []; + const responses = []; + let continuationToken: string | undefined; + let output: ListObjectsV2Output; + do { aws.config.update({ region: region }); - const { Contents, IsTruncated, NextContinuationToken } = - await this.deps.s3.listObjectsV2(params).promise(); - const responses = await Promise.all( - (Contents || []).map(({ Key }) => { - const s3Response = this.deps.s3 - .getObject({ Bucket: bucket, Key: String(Key) }) - .createReadStream(); - Object.defineProperty(s3Response, 'path', { - value: String(Key), - writable: false, - }); - return s3Response; - }), - ); - - if (IsTruncated) { - continuationToken = String(NextContinuationToken); - } else { - continuationToken = ''; - moreKeys = false; + output = await this.deps.s3 + .listObjectsV2({ + Bucket: bucket, + ContinuationToken: continuationToken, + Prefix: path, + }) + .promise(); + if (output.Contents) { + output.Contents.forEach(function (contents) { + allObjects.push(contents); + }); } - awsS3Readables = awsS3Readables.concat(responses); + continuationToken = output.NextContinuationToken; + } while (continuationToken); + + for (let i = 0; i < allObjects.length; i++) { + const object = this.deps.s3.getObject({ + Bucket: bucket, + Key: String(allObjects[i].Key), + }); + responses.push({ + data: object.createReadStream(), + path: String(allObjects[i].Key), + }); } - return await this.deps.treeResponseFactory.fromReadableArray({ - stream: awsS3Readables, - etag: '', - }); + return await this.deps.treeResponseFactory.fromReadableArray(responses); } catch (e) { throw new Error(`Could not retrieve file tree from S3: ${e.message}`); } diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts index 7d20385635..48523a1319 100644 --- a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -59,10 +59,6 @@ export class DefaultReadTreeResponseFactory implements ReadTreeResponseFactory { async fromReadableArray( options: FromReadableArrayOptions, ): Promise { - return new ReadableArrayResponse( - options.stream, - this.workDir, - options.etag, - ); + return new ReadableArrayResponse(options, this.workDir, ''); } } diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index bf457f02e4..736dc0195c 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -15,20 +15,19 @@ */ import { ReadableArrayResponse } from './ReadableArrayResponse'; import path, { resolve as resolvePath } from 'path'; - -import { Readable } from 'stream'; import fs from 'fs-extra'; +import { FromReadableArrayOptions } from '../types'; -const arr: Readable[] = []; -const arr2: Readable[] = []; -const file1 = path.resolve( +const arr: FromReadableArrayOptions = []; +const arr2: FromReadableArrayOptions = []; +const path1 = path.resolve( 'src', 'reading', '__fixtures__', 'awsS3', 'awsS3-mock-object.yaml', ); -const file2 = path.resolve( +const path2 = path.resolve( 'src', 'reading', '__fixtures__', @@ -38,21 +37,21 @@ const file2 = path.resolve( describe('ReadableArrayResponse', () => { it('should read files', async () => { - const stream1 = fs.createReadStream(file1); - const stream2 = fs.createReadStream(file2); - arr.push(stream1); - arr.push(stream2); + const stream1 = fs.createReadStream(path1); + const stream2 = fs.createReadStream(path2); + arr.push({ data: stream1, path: path1 }); + arr.push({ data: stream2, path: path2 }); const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); const files = await res.files(); expect(files).toEqual([ { - path: file1, + path: path1, content: expect.any(Function), }, { - path: file2, + path: path2, content: expect.any(Function), }, ]); @@ -64,11 +63,11 @@ describe('ReadableArrayResponse', () => { }); it('should extract entire archive into directory', async () => { - const stream1 = fs.createReadStream(file1); - const stream2 = fs.createReadStream(file2); + const stream1 = fs.createReadStream(path1); + const stream2 = fs.createReadStream(path2); - arr2.push(stream1); - arr2.push(stream2); + arr2.push({ data: stream1, path: path1 }); + arr2.push({ data: stream2, path: path2 }); const res = new ReadableArrayResponse(arr2, '/tmp', 'etag'); const dir = await res.dir(); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index bf74a19603..eabaa3bc56 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -26,6 +26,7 @@ import { ReadTreeResponse, ReadTreeResponseFile, ReadTreeResponseDirOptions, + FromReadableArrayOptions, } from '../types'; const pipeline = promisify(pipelineCb); @@ -37,7 +38,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { private read = false; constructor( - private readonly stream: Readable[], + private readonly stream: FromReadableArrayOptions, private readonly workDir: string, public readonly etag: string, ) { @@ -58,10 +59,10 @@ export class ReadableArrayResponse implements ReadTreeResponse { const files = Array(); for (let i = 0; i < this.stream.length; i++) { - if (!(this.stream[i] as any).path.endsWith('/')) { + if (!this.stream[i].path.endsWith('/')) { files.push({ - path: (this.stream[i] as any).path, - content: () => getRawBody(this.stream[i]), + path: this.stream[i].path, + content: () => getRawBody(this.stream[i].data), }); } } @@ -93,11 +94,11 @@ export class ReadableArrayResponse implements ReadTreeResponse { (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); for (let i = 0; i < this.stream.length; i++) { - if (!(this.stream[i] as any).path.endsWith('/')) { + if (!this.stream[i].path.endsWith('/')) { await pipeline( - this.stream[i], + this.stream[i].data, fs.createWriteStream( - platformPath.join(dir, basename((this.stream[i] as any).path)), + platformPath.join(dir, basename(this.stream[i].path)), ), ); } diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index c0acc04701..5149f5fc94 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -167,12 +167,12 @@ export type FromArchiveOptions = { filter?: (path: string, info?: { size: number }) => boolean; }; -export type FromReadableArrayOptions = { - // An array of readable streams - stream: Readable[]; - // etag of the file tree - etag: string; -}; +export type FromReadableArrayOptions = Array<{ + // Data in the form of a readable + data: Readable; + // A string containing the filepath of the data + path: string; +}>; export interface ReadTreeResponseFactory { fromTarArchive(options: FromArchiveOptions): Promise; fromZipArchive(options: FromArchiveOptions): Promise; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 79613a9319..e5341e8830 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.13.5", + "version": "0.13.2", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts rename to plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts index ebc258a29e..3f542c8459 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts @@ -15,7 +15,7 @@ */ import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; +import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; import { defaultEntityDataParser } from './util/parse'; import AWSMock from 'aws-sdk-mock'; @@ -58,9 +58,9 @@ const reader = UrlReaders.default({ }); describe('readLocation', () => { - const processor = new AwsS3ReadTreeProcessor(reader); + const processor = new AwsS3DiscoveryProcessor(reader); const spec = { - type: 's3-bucket', + type: 'awsS3-discovery', target: 'https://testbucket.s3.us-east-2.amazonaws.com', }; @@ -71,7 +71,7 @@ describe('readLocation', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ target: 'awsS3-mock-object.txt', - type: 's3-bucket', + type: 'awsS3-discovery', }); expect(generated.entity).toEqual({ site_name: 'Test' }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts similarity index 89% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts rename to plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts index 9d8e1cdce5..3024591339 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3ReadTreeProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts @@ -16,6 +16,7 @@ import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; +import limiterFactory from 'p-limit'; import * as result from './results'; import { CatalogProcessor, @@ -23,7 +24,7 @@ import { CatalogProcessorParser, } from './types'; -export class AwsS3ReadTreeProcessor implements CatalogProcessor { +export class AwsS3DiscoveryProcessor implements CatalogProcessor { constructor(private readonly reader: UrlReader) {} async readLocation( @@ -32,7 +33,7 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 's3-bucket') { + if (location.type !== 'awsS3-discovery') { return false; } @@ -63,11 +64,12 @@ export class AwsS3ReadTreeProcessor implements CatalogProcessor { private async doRead( location: string, ): Promise<{ data: Buffer; url: string }[]> { + const limiter = limiterFactory(5); const response = await this.reader.readTree(location); const responseFiles = await response.files(); const output = responseFiles.map(async file => ({ url: file.path, - data: await file.content(), + data: await limiter(file.content), })); return Promise.all(output); } diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 1cfb9980f2..756dae178d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -19,7 +19,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; -export { AwsS3ReadTreeProcessor } from './AwsS3ReadTreeProcessor'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; From 4e6c15cc669b87be2a78fc112058bb6bbab140e4 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 31 Aug 2021 18:01:15 -0700 Subject: [PATCH 52/61] Fix lint warning Signed-off-by: Sean Tan --- packages/backend-common/api-report.md | 47 +++---------------- .../src/reading/AwsS3UrlReader.ts | 2 +- plugins/catalog-backend/api-report.md | 4 +- 3 files changed, 9 insertions(+), 44 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 4f97cd77e8..a05e539969 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -35,39 +35,6 @@ import * as winston from 'winston'; import { Writable } from 'stream'; // Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -<<<<<<< HEAD -======= -// -// @public (undocumented) -export class AwsS3UrlReader implements UrlReader { - constructor( - integration: AwsS3Integration, - deps: { - s3: S3; - treeResponseFactory: ReadTreeResponseFactory; - }, - ); - // Warning: (ae-forgotten-export) The symbol "ReaderFactory" needs to be exported by the entry point index.d.ts - // - // (undocumented) - static factory: ReaderFactory; - // (undocumented) - read(url: string): Promise; - // (undocumented) - readTree(url: string): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts - // - // (undocumented) - readUrl(url: string, options?: ReadUrlOptions): Promise; - // (undocumented) - search(): Promise; - // (undocumented) - toString(): string; -} - -// Warning: (ae-missing-release-tag) "AzureUrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) ->>>>>>> bd119f21cc075b9d09eb0470566eee996364fbb0 // // @public (undocumented) export class AwsS3UrlReader implements UrlReader { @@ -83,14 +50,7 @@ export class AwsS3UrlReader implements UrlReader { // (undocumented) read(url: string): Promise; // (undocumented) -<<<<<<< HEAD readTree(url: string): Promise; - // Warning: (ae-forgotten-export) The symbol "ReadUrlOptions" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "ReadUrlResponse" needs to be exported by the entry point index.d.ts - // -======= - readTree(url: string, options?: ReadTreeOptions): Promise; ->>>>>>> bd119f21cc075b9d09eb0470566eee996364fbb0 // (undocumented) readUrl(url: string, options?: ReadUrlOptions): Promise; // (undocumented) @@ -509,6 +469,12 @@ export type ReadTreeResponse = { // // @public (undocumented) export interface ReadTreeResponseFactory { + // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + fromReadableArray( + options: FromReadableArrayOptions, + ): Promise; // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts // // (undocumented) @@ -714,7 +680,6 @@ export function useHotMemoize(_module: NodeModule, valueFactory: () => T): T; // // src/cache/types.d.ts:34:5 - (ae-forgotten-export) The symbol "ClientOptions" needs to be exported by the entry point index.d.ts // src/middleware/errorHandler.d.ts:17:26 - (tsdoc-malformed-html-name) Invalid HTML element: A space is not allowed here -// src/reading/AwsS3UrlReader.d.ts:11:9 - (ae-forgotten-export) The symbol "ReadTreeResponseFactory" needs to be exported by the entry point index.d.ts // src/reading/types.d.ts:108:5 - (ae-forgotten-export) The symbol "ReadTreeResponseDirOptions" needs to be exported by the entry point index.d.ts // src/service/types.d.ts:12:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen // src/service/types.d.ts:22:8 - (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen diff --git a/packages/backend-common/src/reading/AwsS3UrlReader.ts b/packages/backend-common/src/reading/AwsS3UrlReader.ts index 53bf99b307..cb03ec4f71 100644 --- a/packages/backend-common/src/reading/AwsS3UrlReader.ts +++ b/packages/backend-common/src/reading/AwsS3UrlReader.ts @@ -183,7 +183,7 @@ export class AwsS3UrlReader implements UrlReader { }) .promise(); if (output.Contents) { - output.Contents.forEach(function (contents) { + output.Contents.forEach(contents => { allObjects.push(contents); }); } diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index f2dbd80464..b2384cfa19 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -121,10 +121,10 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { ): Promise; } -// Warning: (ae-missing-release-tag) "AwsS3ReadTreeProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class AwsS3ReadTreeProcessor implements CatalogProcessor { +export class AwsS3DiscoveryProcessor implements CatalogProcessor { constructor(reader: UrlReader); // (undocumented) readLocation( From ba6498f92e8f89b7ee021aaac2226b1c29dfbb13 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Tue, 31 Aug 2021 18:20:44 -0700 Subject: [PATCH 53/61] Fix prettier warn in sidebars.json Signed-off-by: Sean Tan --- microsite/sidebars.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 56df6b4b50..97dfa61951 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -36,6 +36,7 @@ "label": "Software Catalog", "ids": [ "features/software-catalog/software-catalog-overview", + "features/software-catalog/life-of-an-entity", "features/software-catalog/configuration", "features/software-catalog/system-model", "features/software-catalog/descriptor-format", @@ -273,4 +274,4 @@ ], "FAQ": ["FAQ"] } -} +} \ No newline at end of file From bba12b0e7bd65dc0ced67860dac2d1dfcd3d6f23 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Thu, 2 Sep 2021 08:23:54 -0700 Subject: [PATCH 54/61] Fix borked package.json Signed-off-by: Sean Tan --- microsite/sidebars.json | 7 +- plugins/catalog-backend/package.json | 21 +- yarn.lock | 337 ++++++++++++++++++++++----- 3 files changed, 292 insertions(+), 73 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 97dfa61951..6569f41c20 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -108,7 +108,10 @@ { "type": "subcategory", "label": "AWS S3", - "ids": ["integrations/aws-s3/locations", "integrations/aws-s3/discovery"] + "ids": [ + "integrations/aws-s3/locations", + "integrations/aws-s3/discovery" + ] }, { "type": "subcategory", @@ -274,4 +277,4 @@ ], "FAQ": ["FAQ"] } -} \ No newline at end of file +} diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index e5341e8830..d853e1eb6b 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend", - "version": "0.13.2", + "version": "0.13.6", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -29,14 +29,14 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.9.0", + "@backstage/backend-common": "^0.9.1", "@backstage/catalog-client": "^0.3.18", - "@backstage/catalog-model": "^0.9.0", + "@backstage/catalog-model": "^0.9.1", "@backstage/config": "^0.1.8", "@backstage/errors": "^0.1.1", - "@backstage/integration": "^0.6.2", - "@backstage/plugin-search-backend-node": "^0.4.0", - "@backstage/search-common": "^0.1.2", + "@backstage/integration": "^0.6.3", + "@backstage/plugin-search-backend-node": "^0.4.2", + "@backstage/search-common": "^0.2.0", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "aws-sdk": "^2.840.0", @@ -65,14 +65,7 @@ }, "devDependencies": { "@backstage/backend-test-utils": "^0.1.6", - "@backstage/cli": "^0.7.9", - "aws-sdk-mock": "^5.2.1", - "@backstage/backend-test-utils": "^0.1.5", - "@backstage/cli": "^0.7.8", - "@backstage/backend-test-utils": "^0.1.6", - "@backstage/cli": "^0.7.9", - "@backstage/backend-test-utils": "^0.1.6", - "@backstage/cli": "^0.7.9", + "@backstage/cli": "^0.7.10", "@backstage/test-utils": "^0.1.17", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", diff --git a/yarn.lock b/yarn.lock index e643ac9750..1ccd383439 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2294,6 +2294,161 @@ "@babel/helper-validator-identifier" "^7.14.9" to-fast-properties "^2.0.0" +"@backstage/backend-common@^0.9.1": + version "0.9.1" + resolved "https://registry.npmjs.org/@backstage/backend-common/-/backend-common-0.9.1.tgz#f7753922b5264b6e0eb6286369fd9a211c3357f2" + integrity sha512-hXyEu/jZMZ/DJ/4mKrCNWg8yrttmSGXCVJ5Yiiv8XqXgaYOgnJqU+siRLVlrp6oC0HqGVoxtXm6rO9M6cTJ1Ag== + dependencies: + "@backstage/cli-common" "^0.1.2" + "@backstage/config" "^0.1.8" + "@backstage/config-loader" "^0.6.7" + "@backstage/errors" "^0.1.1" + "@backstage/integration" "^0.6.3" + "@google-cloud/storage" "^5.8.0" + "@octokit/rest" "^18.5.3" + "@types/cors" "^2.8.6" + "@types/dockerode" "^3.2.1" + "@types/express" "^4.17.6" + archiver "^5.0.2" + aws-sdk "^2.840.0" + compression "^1.7.4" + concat-stream "^2.0.0" + cors "^2.8.5" + cross-fetch "^3.0.6" + dockerode "^3.2.1" + express "^4.17.1" + express-promise-router "^4.1.0" + fs-extra "9.1.0" + git-url-parse "~11.4.4" + helmet "^4.0.0" + isomorphic-git "^1.8.0" + keyv "^4.0.3" + keyv-memcache "^1.2.5" + knex "^0.95.1" + lodash "^4.17.15" + logform "^2.1.1" + minimatch "^3.0.4" + minimist "^1.2.5" + morgan "^1.10.0" + raw-body "^2.4.1" + selfsigned "^1.10.7" + stoppable "^1.1.0" + tar "^6.1.2" + unzipper "^0.10.11" + winston "^3.2.1" + yn "^4.0.0" + +"@backstage/catalog-model@^0.9.1": + version "0.9.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.9.1.tgz#dc3a497e26894a768c9ceaf4c5af94d35782a048" + integrity sha512-Q/N0p7ATNBwZUSzbSP3lFSUZ5UsDGdTmOUfjR2cupBe4tnJfK8KbE+TQCKlIsznBENRq4vtBGJqiAe+7oxpZWQ== + dependencies: + "@backstage/config" "^0.1.5" + "@backstage/errors" "^0.1.1" + "@types/json-schema" "^7.0.5" + "@types/yup" "^0.29.8" + ajv "^7.0.3" + json-schema "^0.3.0" + lodash "^4.17.15" + uuid "^8.0.0" + yup "^0.29.3" + +"@backstage/cli@^0.7.10": + version "0.7.10" + resolved "https://registry.npmjs.org/@backstage/cli/-/cli-0.7.10.tgz#3d460b4ed558ffc3bd58c1a2b48745167fe9829e" + integrity sha512-+9F+j9elhBX4XujZSG4qgrE2PeJW8/5+TqntpKRJZllSTSCkqNh5O93m34+MlH9PQQ++w3sLFt7bmNF8zaYEdA== + dependencies: + "@babel/core" "^7.4.4" + "@babel/plugin-transform-modules-commonjs" "^7.4.4" + "@backstage/cli-common" "^0.1.2" + "@backstage/config" "^0.1.8" + "@backstage/config-loader" "^0.6.7" + "@hot-loader/react-dom" "^16.13.0" + "@lerna/package-graph" "^4.0.0" + "@lerna/project" "^4.0.0" + "@octokit/request" "^5.4.12" + "@rollup/plugin-commonjs" "^17.1.0" + "@rollup/plugin-json" "^4.0.2" + "@rollup/plugin-node-resolve" "^13.0.0" + "@rollup/plugin-yaml" "^3.0.0" + "@spotify/eslint-config-base" "^9.0.0" + "@spotify/eslint-config-react" "^10.0.0" + "@spotify/eslint-config-typescript" "^10.0.0" + "@sucrase/jest-plugin" "^2.1.0" + "@sucrase/webpack-loader" "^2.0.0" + "@svgr/plugin-jsx" "5.5.x" + "@svgr/plugin-svgo" "5.4.x" + "@svgr/rollup" "5.5.x" + "@svgr/webpack" "5.5.x" + "@types/webpack-env" "^1.15.2" + "@typescript-eslint/eslint-plugin" "^v4.30.0" + "@typescript-eslint/parser" "^v4.28.3" + "@yarnpkg/lockfile" "^1.1.0" + babel-plugin-dynamic-import-node "^2.3.3" + bfj "^7.0.2" + buffer "^6.0.3" + chalk "^4.0.0" + chokidar "^3.3.1" + commander "^6.1.0" + css-loader "^5.2.6" + dashify "^2.0.0" + diff "^5.0.0" + esbuild "^0.8.56" + eslint "^7.30.0" + eslint-config-prettier "^8.3.0" + eslint-formatter-friendly "^7.0.0" + eslint-plugin-import "^2.20.2" + eslint-plugin-jest "^24.1.0" + eslint-plugin-jsx-a11y "^6.2.1" + eslint-plugin-monorepo "^0.3.2" + eslint-plugin-react "^7.12.4" + eslint-plugin-react-hooks "^4.0.0" + express "^4.17.1" + file-loader "^6.2.0" + fork-ts-checker-webpack-plugin "^4.0.5" + fs-extra "9.1.0" + handlebars "^4.7.3" + html-webpack-plugin "^5.3.1" + inquirer "^7.0.4" + jest "^26.0.1" + jest-css-modules "^2.1.0" + json-schema "^0.3.0" + lodash "^4.17.19" + mini-css-extract-plugin "^1.4.1" + node-libs-browser "^2.2.1" + ora "^5.3.0" + postcss "^8.1.0" + process "^0.11.10" + raw-loader "^4.0.1" + react "^16.0.0" + react-dev-utils "^11.0.4" + react-hot-loader "^4.12.21" + recursive-readdir "^2.2.2" + replace-in-file "^6.0.0" + rollup "2.44.x" + rollup-plugin-dts "^3.0.1" + rollup-plugin-esbuild "2.6.x" + rollup-plugin-peer-deps-external "^2.2.2" + rollup-plugin-postcss "^4.0.0" + rollup-pluginutils "^2.8.2" + run-script-webpack-plugin "^0.0.11" + semver "^7.3.2" + style-loader "^1.2.1" + sucrase "^3.18.2" + tar "^6.1.2" + terser-webpack-plugin "^5.1.3" + ts-loader "^8.0.17" + typescript "^4.0.3" + url-loader "^4.1.0" + util "^0.12.3" + webpack "^5.48.0" + webpack-dev-server "4.0.0-rc.0" + webpack-node-externals "^3.0.0" + yaml "^1.10.0" + yaml-jest "^1.0.5" + yml-loader "^2.1.0" + yn "^4.0.0" + "@backstage/core-api@^0.2.23": version "0.2.23" resolved "https://registry.npmjs.org/@backstage/core-api/-/core-api-0.2.23.tgz#c0ec13407ff7c78d376eb18e9d8e1490d54d995b" @@ -2358,6 +2513,35 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" +"@backstage/integration@^0.6.3": + version "0.6.3" + resolved "https://registry.npmjs.org/@backstage/integration/-/integration-0.6.3.tgz#e94f47d82e1325a5542dc883347192f949dd8adb" + integrity sha512-Dg+k0l0hU7r+MlZHFNQbUXG/hIO75KKUiqinQQ7V3MYs4zRKDhzaTXcxuWU258WgFaA8oxOr7pQdqqKdN7qZlw== + dependencies: + "@backstage/config" "^0.1.8" + "@octokit/auth-app" "^3.4.0" + "@octokit/rest" "^18.5.3" + cross-fetch "^3.0.6" + git-url-parse "~11.4.4" + luxon "^2.0.2" + +"@backstage/plugin-search-backend-node@^0.4.2": + version "0.4.2" + resolved "https://registry.npmjs.org/@backstage/plugin-search-backend-node/-/plugin-search-backend-node-0.4.2.tgz#1ba6a9a811e79877e26ce553f8dc4f72fdec190c" + integrity sha512-QmHTtsjWNzxNPynbAMW+WaBzoWuMZfr1Mf+sWWxTrn4GHUiukFt2og1uF/LEGT1smfwOnWZAy782ZHVQerogxQ== + dependencies: + "@backstage/search-common" "^0.2.0" + "@types/lunr" "^2.3.3" + lunr "^2.3.9" + winston "^3.2.1" + +"@backstage/search-common@^0.2.0": + version "0.2.0" + resolved "https://registry.npmjs.org/@backstage/search-common/-/search-common-0.2.0.tgz#91971536b0b9a7d8eb308b79d744d4a8dfdeea4d" + integrity sha512-eJzY+UF+edhr8EpMwAPWrETzCuR4HMJ7V1Z590gveEsAB7/OATd3MHh5hyvlICbERrejAsvTlFkta6kR+aFIzg== + dependencies: + "@backstage/config" "^0.1.6" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -4749,9 +4933,9 @@ universal-user-agent "^5.0.0" "@octokit/graphql@^4.5.8": - version "4.6.4" - resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.6.4.tgz#0c3f5bed440822182e972317122acb65d311a5ed" - integrity sha512-SWTdXsVheRmlotWNjKzPOb6Js6tjSqA2a8z9+glDJng0Aqjzti8MEWOtuT8ZSu6wHnci7LZNuarE87+WJBG4vg== + version "4.7.0" + resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.7.0.tgz#cbe12edc2bc61e9eaa5f9e5d092644c92b6fcb74" + integrity sha512-diY0qMPyQjfu4rDu3kDhJ9qIZadIm4IISO3RJSv9ajYUWJUCO0AykbgzLcg1xclxtXgzY583u3gAv66M6zz5SA== dependencies: "@octokit/request" "^5.6.0" "@octokit/types" "^6.0.3" @@ -4974,9 +5158,9 @@ react-lifecycles-compat "^3.0.4" "@rjsf/core@^3.0.0": - version "3.0.0" - resolved "https://registry.npmjs.org/@rjsf/core/-/core-3.0.0.tgz#7f79b0635c969e3fbfe0aaa4773d844fbee68a63" - integrity sha512-QSvyVMDiwd7neVnIMOte4tZUvtDWgLDYKOsq1tgCsIFh+RXskd3AEBf6IsK8w6Wg36Om7YalDqqrtXbk9z+rLw== + version "3.1.0" + resolved "https://registry.npmjs.org/@rjsf/core/-/core-3.1.0.tgz#7667364a4532e25c164abbc2adabe39157ffec4c" + integrity sha512-EwM2juiQxEdXzFy9rIIsDr6/e+FYeR1cKx0/no8ASEuXZ1p+/nf/CxKGobzD6UhpRip7JK9PhhuHFEFBIjHFJA== dependencies: "@types/json-schema" "^7.0.7" ajv "^6.7.0" @@ -4989,9 +5173,9 @@ react-is "^16.9.0" "@rjsf/material-ui@^3.0.0": - version "3.0.0" - resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-3.0.0.tgz#69ece2cb549f0e860b5f89898db90edcc95b15ba" - integrity sha512-T2B8QnrDQphbFNxDz7baAa0zTd5TXJmO9soHBPTKKdniRbMEOQ19AJBbZkA3ED2XZa/xrUY/6XjERQLpNACddw== + version "3.1.0" + resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-3.1.0.tgz#ed15fc75de1594f87cd8336b7f2ebc492a432d8b" + integrity sha512-kWz37spT5SOXkb8Axq4g4BzQjXRylQr6B7eFAg1NPhSK7KrJYwSMSsFJ9Ze2vEBxwEiR0Z0n/4puaamGuGFRBw== "@roadiehq/backstage-plugin-buildkite@^1.0.8": version "1.0.8" @@ -6299,9 +6483,9 @@ "@types/express" "*" "@types/concat-stream@^1.6.0": - version "1.6.0" - resolved "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz#394dbe0bb5fee46b38d896735e8b68ef2390d00d" - integrity sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0= + version "1.6.1" + resolved "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.1.tgz#24bcfc101ecf68e886aaedce60dfd74b632a1b74" + integrity sha512-eHE4cQPoj6ngxBZMvVf6Hw7Mh4jMW4U9lpGmS5GBPB9RYxlFg+CHaVN7ErNY4W9XfLIEn20b4VDYaIrbq0q4uA== dependencies: "@types/node" "*" @@ -7546,28 +7730,28 @@ resolved "https://registry.npmjs.org/@types/zen-observable/-/zen-observable-0.8.2.tgz#808c9fa7e4517274ed555fa158f2de4b4f468e71" integrity sha512-HrCIVMLjE1MOozVoD86622S7aunluLb2PJdPfb3nYiEtohm8mIB/vyv0Fd37AdeMFrTUQXEunw78YloMA3Qilg== -"@typescript-eslint/eslint-plugin@^v4.28.3": - version "4.28.3" - resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.28.3.tgz#36cdcd9ca6f9e5cb49b9f61b970b1976708d084b" - integrity sha512-jW8sEFu1ZeaV8xzwsfi6Vgtty2jf7/lJmQmDkDruBjYAbx5DA8JtbcMnP0rNPUG+oH5GoQBTSp+9613BzuIpYg== +"@typescript-eslint/eslint-plugin@^v4.28.3", "@typescript-eslint/eslint-plugin@^v4.30.0": + version "4.30.0" + resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.30.0.tgz#4a0c1ae96b953f4e67435e20248d812bfa55e4fb" + integrity sha512-NgAnqk55RQ/SD+tZFD9aPwNSeHmDHHe5rtUyhIq0ZeCWZEvo4DK9rYz7v9HDuQZFvn320Ot+AikaCKMFKLlD0g== dependencies: - "@typescript-eslint/experimental-utils" "4.28.3" - "@typescript-eslint/scope-manager" "4.28.3" + "@typescript-eslint/experimental-utils" "4.30.0" + "@typescript-eslint/scope-manager" "4.30.0" debug "^4.3.1" functional-red-black-tree "^1.0.1" regexpp "^3.1.0" semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/experimental-utils@4.28.3", "@typescript-eslint/experimental-utils@^4.0.1": - version "4.28.3" - resolved "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.28.3.tgz#976f8c1191b37105fd06658ed57ddfee4be361ca" - integrity sha512-zZYl9TnrxwEPi3FbyeX0ZnE8Hp7j3OCR+ELoUfbwGHGxWnHg9+OqSmkw2MoCVpZksPCZYpQzC559Ee9pJNHTQw== +"@typescript-eslint/experimental-utils@4.30.0", "@typescript-eslint/experimental-utils@^4.0.1": + version "4.30.0" + resolved "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.30.0.tgz#9e49704fef568432ae16fc0d6685c13d67db0fd5" + integrity sha512-K8RNIX9GnBsv5v4TjtwkKtqMSzYpjqAQg/oSphtxf3xxdt6T0owqnpojztjjTcatSteH3hLj3t/kklKx87NPqw== dependencies: "@types/json-schema" "^7.0.7" - "@typescript-eslint/scope-manager" "4.28.3" - "@typescript-eslint/types" "4.28.3" - "@typescript-eslint/typescript-estree" "4.28.3" + "@typescript-eslint/scope-manager" "4.30.0" + "@typescript-eslint/types" "4.30.0" + "@typescript-eslint/typescript-estree" "4.30.0" eslint-scope "^5.1.1" eslint-utils "^3.0.0" @@ -7589,11 +7773,24 @@ "@typescript-eslint/types" "4.28.3" "@typescript-eslint/visitor-keys" "4.28.3" +"@typescript-eslint/scope-manager@4.30.0": + version "4.30.0" + resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.30.0.tgz#1a3ffbb385b1a06be85cd5165a22324f069a85ee" + integrity sha512-VJ/jAXovxNh7rIXCQbYhkyV2Y3Ac/0cVHP/FruTJSAUUm4Oacmn/nkN5zfWmWFEanN4ggP0vJSHOeajtHq3f8A== + dependencies: + "@typescript-eslint/types" "4.30.0" + "@typescript-eslint/visitor-keys" "4.30.0" + "@typescript-eslint/types@4.28.3": version "4.28.3" resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.28.3.tgz#8fffd436a3bada422c2c1da56060a0566a9506c7" integrity sha512-kQFaEsQBQVtA9VGVyciyTbIg7S3WoKHNuOp/UF5RG40900KtGqfoiETWD/v0lzRXc+euVE9NXmfer9dLkUJrkA== +"@typescript-eslint/types@4.30.0": + version "4.30.0" + resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.30.0.tgz#fb9d9b0358426f18687fba82eb0b0f869780204f" + integrity sha512-YKldqbNU9K4WpTNwBqtAerQKLLW/X2A/j4yw92e3ZJYLx+BpKLeheyzoPfzIXHfM8BXfoleTdiYwpsvVPvHrDw== + "@typescript-eslint/typescript-estree@4.28.3": version "4.28.3" resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.28.3.tgz#253d7088100b2a38aefe3c8dd7bd1f8232ec46fb" @@ -7607,6 +7804,19 @@ semver "^7.3.5" tsutils "^3.21.0" +"@typescript-eslint/typescript-estree@4.30.0": + version "4.30.0" + resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.30.0.tgz#ae57833da72a753f4846cd3053758c771670c2ac" + integrity sha512-6WN7UFYvykr/U0Qgy4kz48iGPWILvYL34xXJxvDQeiRE018B7POspNRVtAZscWntEPZpFCx4hcz/XBT+erenfg== + dependencies: + "@typescript-eslint/types" "4.30.0" + "@typescript-eslint/visitor-keys" "4.30.0" + debug "^4.3.1" + globby "^11.0.3" + is-glob "^4.0.1" + semver "^7.3.5" + tsutils "^3.21.0" + "@typescript-eslint/visitor-keys@4.28.3": version "4.28.3" resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.28.3.tgz#26ac91e84b23529968361045829da80a4e5251c4" @@ -7615,6 +7825,14 @@ "@typescript-eslint/types" "4.28.3" eslint-visitor-keys "^2.0.0" +"@typescript-eslint/visitor-keys@4.30.0": + version "4.30.0" + resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.30.0.tgz#a47c6272fc71b0c627d1691f68eaecf4ad71445e" + integrity sha512-pNaaxDt/Ol/+JZwzP7MqWc8PJQTUhZwoee/PVlQ+iYoYhagccvoHnC9e4l+C/krQYYkENxznhVSDwClIbZVxRw== + dependencies: + "@typescript-eslint/types" "4.30.0" + eslint-visitor-keys "^2.0.0" + "@webassemblyjs/ast@1.11.1": version "1.11.1" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" @@ -10107,7 +10325,7 @@ chokidar@^3.2.2, chokidar@^3.3.1, chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3. optionalDependencies: fsevents "~2.3.2" -chownr@^1.1.1: +chownr@^1.1.1, chownr@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== @@ -10933,9 +11151,9 @@ core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: integrity sha512-GUbtPllXMYRzIgHNZ4dTYTcUemls2cni83Q4Q/TrFONHfhcg9oEGOtaGHfb0cpzec60P96UKPvMkjX1jET8rUw== core-js@^3.6.0: - version "3.16.2" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.16.2.tgz#3f485822889c7fc48ef463e35be5cc2a4a01a1f4" - integrity sha512-P0KPukO6OjMpjBtHSceAZEWlDD1M2Cpzpg6dBbrjFqFhBHe/BwhxaP820xKOjRn/lZRQirrCusIpLS/n2sgXLQ== + version "3.17.1" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.17.1.tgz#b39e086f413789cf2ca4680c4ecd1b36a50ba277" + integrity sha512-C8i/FNpVN2Ti89QIJcFn9ZQmnM+HaAQr2OpE+ja3TRM9Q34FigsGlAVuwPGkIgydSVClo/1l1D1grP8LVt9IYA== core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -14062,7 +14280,7 @@ fs-extra@^8.1.0: jsonfile "^4.0.0" universalify "^0.1.0" -fs-minipass@^1.2.5: +fs-minipass@^1.2.7: version "1.2.7" resolved "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz#ccff8570841e7fe4265693da88936c55aed7f7c7" integrity sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA== @@ -19213,7 +19431,7 @@ minipass-sized@^1.0.3: dependencies: minipass "^3.0.0" -minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0: +minipass@^2.6.0, minipass@^2.9.0: version "2.9.0" resolved "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6" integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg== @@ -19228,7 +19446,7 @@ minipass@^3.0.0, minipass@^3.1.0, minipass@^3.1.1, minipass@^3.1.3: dependencies: yallist "^4.0.0" -minizlib@^1.2.1: +minizlib@^1.3.3: version "1.3.3" resolved "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d" integrity sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q== @@ -20076,9 +20294,9 @@ object-keys@^1.0.12, object-keys@^1.1.1: integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== object-path@^0.11.4: - version "0.11.5" - resolved "https://registry.npmjs.org/object-path/-/object-path-0.11.5.tgz#d4e3cf19601a5140a55a16ad712019a9c50b577a" - integrity sha512-jgSbThcoR/s+XumvGMTMf81QVBmah+/Q7K7YduKeKVWL7N111unR2d6pZZarSk6kY/caeNxUDyxOvMWyzoU2eg== + version "0.11.7" + resolved "https://registry.npmjs.org/object-path/-/object-path-0.11.7.tgz#5f211161f34bb395e4b13a5f565b79d933b6f65d" + integrity sha512-T4evaK9VfGGQskXBDILcn6F90ZD+WO3OwRFFQ2rmZdUH4vQeDBpiolTpVlPY2yj5xSepyILTjDyM6UvbbdHMZw== object-visit@^1.0.0: version "1.0.1" @@ -22372,9 +22590,9 @@ react-helmet@6.1.0: react-side-effect "^2.1.0" react-hook-form@^6.15.4: - version "6.15.4" - resolved "https://registry.npmjs.org/react-hook-form/-/react-hook-form-6.15.4.tgz#328003e1ccc096cd158899ffe7e3b33735a9b024" - integrity sha512-K+Sw33DtTMengs8OdqFJI3glzNl1wBzSefD/ksQw/hJf9CnOHQAU6qy82eOrh0IRNt2G53sjr7qnnw1JDjvx1w== + version "6.15.8" + resolved "https://registry.npmjs.org/react-hook-form/-/react-hook-form-6.15.8.tgz#725c139d308c431c4611e4b9d85a49f01cfc0e7a" + integrity sha512-prq82ofMbnRyj5wqDe8hsTRcdR25jQ+B8KtCS7BLCzjFHAwNuCjRwzPuP4eYLsEBjEIeYd6try+pdLdw0kPkpg== react-hook-form@^7.12.2: version "7.12.2" @@ -23726,7 +23944,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -25404,22 +25622,22 @@ tar@^2.0.0: inherits "2" tar@^4, tar@^4.4.12, tar@^4.4.2: - version "4.4.13" - resolved "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" - integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA== + version "4.4.19" + resolved "https://registry.npmjs.org/tar/-/tar-4.4.19.tgz#2e4d7263df26f2b914dee10c825ab132123742f3" + integrity sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA== dependencies: - chownr "^1.1.1" - fs-minipass "^1.2.5" - minipass "^2.8.6" - minizlib "^1.2.1" - mkdirp "^0.5.0" - safe-buffer "^5.1.2" - yallist "^3.0.3" + chownr "^1.1.4" + fs-minipass "^1.2.7" + minipass "^2.9.0" + minizlib "^1.3.3" + mkdirp "^0.5.5" + safe-buffer "^5.2.1" + yallist "^3.1.1" tar@^6.0.2, tar@^6.1.0, tar@^6.1.2: - version "6.1.2" - resolved "https://registry.npmjs.org/tar/-/tar-6.1.2.tgz#1f045a90a6eb23557a603595f41a16c57d47adc6" - integrity sha512-EwKEgqJ7nJoS+s8QfLYVGMDmAsj+StbI2AM/RTHeUSsOw6Z8bwNBRv5z3CY0m7laC5qUAqruLX5AhMuc5deY3Q== + version "6.1.11" + resolved "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" + integrity sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA== dependencies: chownr "^2.0.0" fs-minipass "^2.0.0" @@ -27481,10 +27699,10 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" -ws@7.4.5, ws@^7.4.6, ws@^7.5.3: - version "7.5.3" - resolved "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" - integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== +ws@7.4.5, ws@^7.4.6: + version "7.5.4" + resolved "https://registry.npmjs.org/ws/-/ws-7.5.4.tgz#56bfa20b167427e138a7795de68d134fe92e21f9" + integrity sha512-zP9z6GXm6zC27YtspwH99T3qTG7bBFv2VIkeHstMLrLlDJuzA7tQ5ls3OJ1hOGGCzTQPniNJoHXIAOS0Jljohg== ws@^5.2.0: version "5.2.3" @@ -27510,6 +27728,11 @@ ws@^7.2.3, ws@^7.3.1: resolved "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8" integrity sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA== +ws@^7.5.3: + version "7.5.3" + resolved "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" + integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== + xcase@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/xcase/-/xcase-2.0.1.tgz#c7fa72caa0f440db78fd5673432038ac984450b9" @@ -27644,7 +27867,7 @@ yallist@^2.1.2: resolved "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= -yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: +yallist@^3.0.0, yallist@^3.0.2, yallist@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== From 02a9a2c50cf9bd21b6997960466f637e8c6fec54 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Thu, 2 Sep 2021 08:58:39 -0700 Subject: [PATCH 55/61] Change yarn.lock Signed-off-by: Sean Tan --- yarn.lock | 188 ------------------------------------------------------ 1 file changed, 188 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7cfb1e618f..b0d233c5d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2294,161 +2294,6 @@ "@babel/helper-validator-identifier" "^7.14.9" to-fast-properties "^2.0.0" -"@backstage/backend-common@^0.9.1": - version "0.9.1" - resolved "https://registry.npmjs.org/@backstage/backend-common/-/backend-common-0.9.1.tgz#f7753922b5264b6e0eb6286369fd9a211c3357f2" - integrity sha512-hXyEu/jZMZ/DJ/4mKrCNWg8yrttmSGXCVJ5Yiiv8XqXgaYOgnJqU+siRLVlrp6oC0HqGVoxtXm6rO9M6cTJ1Ag== - dependencies: - "@backstage/cli-common" "^0.1.2" - "@backstage/config" "^0.1.8" - "@backstage/config-loader" "^0.6.7" - "@backstage/errors" "^0.1.1" - "@backstage/integration" "^0.6.3" - "@google-cloud/storage" "^5.8.0" - "@octokit/rest" "^18.5.3" - "@types/cors" "^2.8.6" - "@types/dockerode" "^3.2.1" - "@types/express" "^4.17.6" - archiver "^5.0.2" - aws-sdk "^2.840.0" - compression "^1.7.4" - concat-stream "^2.0.0" - cors "^2.8.5" - cross-fetch "^3.0.6" - dockerode "^3.2.1" - express "^4.17.1" - express-promise-router "^4.1.0" - fs-extra "9.1.0" - git-url-parse "~11.4.4" - helmet "^4.0.0" - isomorphic-git "^1.8.0" - keyv "^4.0.3" - keyv-memcache "^1.2.5" - knex "^0.95.1" - lodash "^4.17.15" - logform "^2.1.1" - minimatch "^3.0.4" - minimist "^1.2.5" - morgan "^1.10.0" - raw-body "^2.4.1" - selfsigned "^1.10.7" - stoppable "^1.1.0" - tar "^6.1.2" - unzipper "^0.10.11" - winston "^3.2.1" - yn "^4.0.0" - -"@backstage/catalog-model@^0.9.1": - version "0.9.1" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.9.1.tgz#dc3a497e26894a768c9ceaf4c5af94d35782a048" - integrity sha512-Q/N0p7ATNBwZUSzbSP3lFSUZ5UsDGdTmOUfjR2cupBe4tnJfK8KbE+TQCKlIsznBENRq4vtBGJqiAe+7oxpZWQ== - dependencies: - "@backstage/config" "^0.1.5" - "@backstage/errors" "^0.1.1" - "@types/json-schema" "^7.0.5" - "@types/yup" "^0.29.8" - ajv "^7.0.3" - json-schema "^0.3.0" - lodash "^4.17.15" - uuid "^8.0.0" - yup "^0.29.3" - -"@backstage/cli@^0.7.10": - version "0.7.10" - resolved "https://registry.npmjs.org/@backstage/cli/-/cli-0.7.10.tgz#3d460b4ed558ffc3bd58c1a2b48745167fe9829e" - integrity sha512-+9F+j9elhBX4XujZSG4qgrE2PeJW8/5+TqntpKRJZllSTSCkqNh5O93m34+MlH9PQQ++w3sLFt7bmNF8zaYEdA== - dependencies: - "@babel/core" "^7.4.4" - "@babel/plugin-transform-modules-commonjs" "^7.4.4" - "@backstage/cli-common" "^0.1.2" - "@backstage/config" "^0.1.8" - "@backstage/config-loader" "^0.6.7" - "@hot-loader/react-dom" "^16.13.0" - "@lerna/package-graph" "^4.0.0" - "@lerna/project" "^4.0.0" - "@octokit/request" "^5.4.12" - "@rollup/plugin-commonjs" "^17.1.0" - "@rollup/plugin-json" "^4.0.2" - "@rollup/plugin-node-resolve" "^13.0.0" - "@rollup/plugin-yaml" "^3.0.0" - "@spotify/eslint-config-base" "^9.0.0" - "@spotify/eslint-config-react" "^10.0.0" - "@spotify/eslint-config-typescript" "^10.0.0" - "@sucrase/jest-plugin" "^2.1.0" - "@sucrase/webpack-loader" "^2.0.0" - "@svgr/plugin-jsx" "5.5.x" - "@svgr/plugin-svgo" "5.4.x" - "@svgr/rollup" "5.5.x" - "@svgr/webpack" "5.5.x" - "@types/webpack-env" "^1.15.2" - "@typescript-eslint/eslint-plugin" "^v4.30.0" - "@typescript-eslint/parser" "^v4.28.3" - "@yarnpkg/lockfile" "^1.1.0" - babel-plugin-dynamic-import-node "^2.3.3" - bfj "^7.0.2" - buffer "^6.0.3" - chalk "^4.0.0" - chokidar "^3.3.1" - commander "^6.1.0" - css-loader "^5.2.6" - dashify "^2.0.0" - diff "^5.0.0" - esbuild "^0.8.56" - eslint "^7.30.0" - eslint-config-prettier "^8.3.0" - eslint-formatter-friendly "^7.0.0" - eslint-plugin-import "^2.20.2" - eslint-plugin-jest "^24.1.0" - eslint-plugin-jsx-a11y "^6.2.1" - eslint-plugin-monorepo "^0.3.2" - eslint-plugin-react "^7.12.4" - eslint-plugin-react-hooks "^4.0.0" - express "^4.17.1" - file-loader "^6.2.0" - fork-ts-checker-webpack-plugin "^4.0.5" - fs-extra "9.1.0" - handlebars "^4.7.3" - html-webpack-plugin "^5.3.1" - inquirer "^7.0.4" - jest "^26.0.1" - jest-css-modules "^2.1.0" - json-schema "^0.3.0" - lodash "^4.17.19" - mini-css-extract-plugin "^1.4.1" - node-libs-browser "^2.2.1" - ora "^5.3.0" - postcss "^8.1.0" - process "^0.11.10" - raw-loader "^4.0.1" - react "^16.0.0" - react-dev-utils "^11.0.4" - react-hot-loader "^4.12.21" - recursive-readdir "^2.2.2" - replace-in-file "^6.0.0" - rollup "2.44.x" - rollup-plugin-dts "^3.0.1" - rollup-plugin-esbuild "2.6.x" - rollup-plugin-peer-deps-external "^2.2.2" - rollup-plugin-postcss "^4.0.0" - rollup-pluginutils "^2.8.2" - run-script-webpack-plugin "^0.0.11" - semver "^7.3.2" - style-loader "^1.2.1" - sucrase "^3.18.2" - tar "^6.1.2" - terser-webpack-plugin "^5.1.3" - ts-loader "^8.0.17" - typescript "^4.0.3" - url-loader "^4.1.0" - util "^0.12.3" - webpack "^5.48.0" - webpack-dev-server "4.0.0-rc.0" - webpack-node-externals "^3.0.0" - yaml "^1.10.0" - yaml-jest "^1.0.5" - yml-loader "^2.1.0" - yn "^4.0.0" - "@backstage/core-api@^0.2.23": version "0.2.23" resolved "https://registry.npmjs.org/@backstage/core-api/-/core-api-0.2.23.tgz#c0ec13407ff7c78d376eb18e9d8e1490d54d995b" @@ -2558,35 +2403,6 @@ remark-gfm "^1.0.0" zen-observable "^0.8.15" -"@backstage/integration@^0.6.3": - version "0.6.3" - resolved "https://registry.npmjs.org/@backstage/integration/-/integration-0.6.3.tgz#e94f47d82e1325a5542dc883347192f949dd8adb" - integrity sha512-Dg+k0l0hU7r+MlZHFNQbUXG/hIO75KKUiqinQQ7V3MYs4zRKDhzaTXcxuWU258WgFaA8oxOr7pQdqqKdN7qZlw== - dependencies: - "@backstage/config" "^0.1.8" - "@octokit/auth-app" "^3.4.0" - "@octokit/rest" "^18.5.3" - cross-fetch "^3.0.6" - git-url-parse "~11.4.4" - luxon "^2.0.2" - -"@backstage/plugin-search-backend-node@^0.4.2": - version "0.4.2" - resolved "https://registry.npmjs.org/@backstage/plugin-search-backend-node/-/plugin-search-backend-node-0.4.2.tgz#1ba6a9a811e79877e26ce553f8dc4f72fdec190c" - integrity sha512-QmHTtsjWNzxNPynbAMW+WaBzoWuMZfr1Mf+sWWxTrn4GHUiukFt2og1uF/LEGT1smfwOnWZAy782ZHVQerogxQ== - dependencies: - "@backstage/search-common" "^0.2.0" - "@types/lunr" "^2.3.3" - lunr "^2.3.9" - winston "^3.2.1" - -"@backstage/search-common@^0.2.0": - version "0.2.0" - resolved "https://registry.npmjs.org/@backstage/search-common/-/search-common-0.2.0.tgz#91971536b0b9a7d8eb308b79d744d4a8dfdeea4d" - integrity sha512-eJzY+UF+edhr8EpMwAPWrETzCuR4HMJ7V1Z590gveEsAB7/OATd3MHh5hyvlICbERrejAsvTlFkta6kR+aFIzg== - dependencies: - "@backstage/config" "^0.1.6" - "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -7775,11 +7591,7 @@ resolved "https://registry.npmjs.org/@types/zen-observable/-/zen-observable-0.8.2.tgz#808c9fa7e4517274ed555fa158f2de4b4f468e71" integrity sha512-HrCIVMLjE1MOozVoD86622S7aunluLb2PJdPfb3nYiEtohm8mIB/vyv0Fd37AdeMFrTUQXEunw78YloMA3Qilg== -<<<<<<< HEAD -"@typescript-eslint/eslint-plugin@^v4.28.3", "@typescript-eslint/eslint-plugin@^v4.30.0": -======= "@typescript-eslint/eslint-plugin@^v4.30.0": ->>>>>>> 0d7f928b416938842ccd0e2248d6139cc2bfba67 version "4.30.0" resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.30.0.tgz#4a0c1ae96b953f4e67435e20248d812bfa55e4fb" integrity sha512-NgAnqk55RQ/SD+tZFD9aPwNSeHmDHHe5rtUyhIq0ZeCWZEvo4DK9rYz7v9HDuQZFvn320Ot+AikaCKMFKLlD0g== From ae79b3e909dc081bf6a4c1f8c4577e80b3fe7e7b Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 6 Sep 2021 11:59:05 -0700 Subject: [PATCH 56/61] Regenerate api-report in backend-common Signed-off-by: Sean Tan --- packages/backend-common/api-report.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 8d53aa6cf5..49104faa56 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -32,7 +32,6 @@ import { Server } from 'http'; import * as winston from 'winston'; import { Writable } from 'stream'; -<<<<<<< HEAD // Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -58,10 +57,6 @@ export class AwsS3UrlReader implements UrlReader { toString(): string; } -// Warning: (ae-missing-release-tag) "AzureUrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -======= ->>>>>>> 3d0fb9a68527d4d93cbc114fca1dfaba7136f46c // @public (undocumented) export class AzureUrlReader implements UrlReader { constructor( @@ -431,17 +426,12 @@ export type ReadTreeResponseDirOptions = { // @public (undocumented) export interface ReadTreeResponseFactory { -<<<<<<< HEAD // Warning: (ae-forgotten-export) The symbol "FromReadableArrayOptions" needs to be exported by the entry point index.d.ts // // (undocumented) fromReadableArray( options: FromReadableArrayOptions, ): Promise; - // Warning: (ae-forgotten-export) The symbol "FromArchiveOptions" needs to be exported by the entry point index.d.ts - // -======= ->>>>>>> 3d0fb9a68527d4d93cbc114fca1dfaba7136f46c // (undocumented) fromTarArchive( options: ReadTreeResponseFactoryOptions, From 60f1010582fa375860ce946040236dd5f56a394a Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 13 Sep 2021 13:19:08 -0700 Subject: [PATCH 57/61] Remove AwsS3DiscoveryProcessor export Signed-off-by: Sean Tan --- plugins/catalog-backend/src/ingestion/processors/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 756dae178d..fe4d374829 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -19,7 +19,6 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; -export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; From 9dcc208c30846d48d14da21da5bfdb925d2c76e9 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 13 Sep 2021 13:21:06 -0700 Subject: [PATCH 58/61] Add back AwsS3Discovery processor export Signed-off-by: Sean Tan --- plugins/catalog-backend/src/ingestion/processors/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index fbdadba667..4f3c502a8e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -20,6 +20,7 @@ export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcess export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; export type { AwsOrganizationProviderConfig } from './awsOrganization/config'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; From a42ea40655b0ca1ae628b1453b0f5b4aacdd8f47 Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Mon, 13 Sep 2021 13:34:52 -0700 Subject: [PATCH 59/61] Regenerate api-report Signed-off-by: Sean Tan --- plugins/catalog-backend/api-report.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index c0beed6b45..30e1887825 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -151,7 +151,13 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { ): Promise; } -<<<<<<< HEAD +// Warning: (ae-missing-release-tag) "AwsOrganizationProviderConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type AwsOrganizationProviderConfig = { + roleArn?: string; +}; + // Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -165,14 +171,6 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { parser: CatalogProcessorParser, ): Promise; } -======= -// Warning: (ae-missing-release-tag) "AwsOrganizationProviderConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export type AwsOrganizationProviderConfig = { - roleArn?: string; -}; ->>>>>>> 11da7e59e940bb91e61c3b0d2db6ec7d38ce0126 // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // From 72289da72cd23c74f473c7b183f6776f273e6b5d Mon Sep 17 00:00:00 2001 From: Sean Tan <86497360+seant-splunk@users.noreply.github.com> Date: Mon, 13 Sep 2021 14:48:38 -0700 Subject: [PATCH 60/61] Update types.ts Signed-off-by: Sean Tan --- plugins/catalog-graphql/src/graphql/types.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-graphql/src/graphql/types.ts b/plugins/catalog-graphql/src/graphql/types.ts index 71e40181d4..5c53a6d1e3 100644 --- a/plugins/catalog-graphql/src/graphql/types.ts +++ b/plugins/catalog-graphql/src/graphql/types.ts @@ -27,8 +27,7 @@ export type Exact = { export type Omit = Pick>; export type RequireFields = { [X in Exclude]?: T[X]; -} & - { [P in K]-?: NonNullable }; +} & { [P in K]-?: NonNullable }; /** All built-in and custom scalars, mapped to their actual values */ export type Scalars = { ID: string; From fab79adde1af5043753e3163e70b68af5f59585a Mon Sep 17 00:00:00 2001 From: Sean Tan Date: Wed, 15 Sep 2021 13:50:48 -0700 Subject: [PATCH 61/61] Add changeset. Change awsS3-discovery to s3-discovery. Change dependencies Signed-off-by: Sean Tan --- .changeset/curvy-books-help.md | 6 ++++++ docs/integrations/aws-s3/discovery.md | 4 ++-- plugins/catalog-backend/package.json | 2 +- .../ingestion/processors/AwsS3DiscoveryProcessor.test.ts | 4 ++-- .../src/ingestion/processors/AwsS3DiscoveryProcessor.ts | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .changeset/curvy-books-help.md diff --git a/.changeset/curvy-books-help.md b/.changeset/curvy-books-help.md new file mode 100644 index 0000000000..de450b12f0 --- /dev/null +++ b/.changeset/curvy-books-help.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-catalog-backend': patch +--- + +Add AWS S3 Discovery Processor. Add readTree() to AwsS3UrlReader. Add ReadableArrayResponse type that implements ReadTreeResponse to use in AwsS3UrlReader's readTree() diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md index 6e1827f395..d50aceacb8 100644 --- a/docs/integrations/aws-s3/discovery.md +++ b/docs/integrations/aws-s3/discovery.md @@ -21,8 +21,8 @@ configuration: ```yaml catalog: locations: - - type: awsS3-discovery + - type: s3-discovery target: https://sample-bucket.s3.us-east-2.amazonaws.com/ ``` -Note the `awsS3-discovery` type, as this is not a regular `url` processor. +Note the `s3-discovery` type, as this is not a regular `url` processor. diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 478038eebd..934457982d 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -40,7 +40,6 @@ "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "aws-sdk": "^2.840.0", - "aws-sdk-mock": "^5.2.1", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", @@ -73,6 +72,7 @@ "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@types/yup": "^0.29.8", + "aws-sdk-mock": "^5.2.1", "msw": "^0.29.0", "sqlite3": "^5.0.1", "supertest": "^6.1.3", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts index 3f542c8459..435ad368c7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts @@ -60,7 +60,7 @@ const reader = UrlReaders.default({ describe('readLocation', () => { const processor = new AwsS3DiscoveryProcessor(reader); const spec = { - type: 'awsS3-discovery', + type: 's3-discovery', target: 'https://testbucket.s3.us-east-2.amazonaws.com', }; @@ -71,7 +71,7 @@ describe('readLocation', () => { expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ target: 'awsS3-mock-object.txt', - type: 'awsS3-discovery', + type: 's3-discovery', }); expect(generated.entity).toEqual({ site_name: 'Test' }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts index 3024591339..c394e9b047 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts @@ -33,7 +33,7 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (location.type !== 'awsS3-discovery') { + if (location.type !== 's3-discovery') { return false; }