From f115a7f8fd9bc30c9905b851050c59dfe1190bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sat, 5 Mar 2022 15:16:22 +0100 Subject: [PATCH] moved AwsS3DiscoveryProcessor to where it should live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/fair-ants-look.md | 5 +++++ .changeset/twenty-planes-dress.md | 5 +++++ docs/integrations/aws-s3/discovery.md | 2 +- .../catalog-backend-module-aws/api-report.md | 16 ++++++++++++++++ plugins/catalog-backend-module-aws/package.json | 5 ++++- .../processors}/AwsS3DiscoveryProcessor.test.ts | 17 ++++++++++++++--- .../src/processors}/AwsS3DiscoveryProcessor.ts | 12 +++++++++--- .../__fixtures__/awsS3-mock-object.txt | 0 .../src/processors/index.ts | 1 + plugins/catalog-backend/api-report.md | 14 -------------- plugins/catalog-backend/package.json | 2 -- .../catalog-backend/src/modules/aws/index.ts | 17 ----------------- plugins/catalog-backend/src/modules/index.ts | 1 - 13 files changed, 55 insertions(+), 42 deletions(-) create mode 100644 .changeset/fair-ants-look.md create mode 100644 .changeset/twenty-planes-dress.md rename plugins/{catalog-backend/src/modules/aws => catalog-backend-module-aws/src/processors}/AwsS3DiscoveryProcessor.test.ts (85%) rename plugins/{catalog-backend/src/modules/aws => catalog-backend-module-aws/src/processors}/AwsS3DiscoveryProcessor.ts (89%) rename plugins/{catalog-backend/src/modules/aws => catalog-backend-module-aws/src/processors}/__fixtures__/awsS3-mock-object.txt (100%) delete mode 100644 plugins/catalog-backend/src/modules/aws/index.ts diff --git a/.changeset/fair-ants-look.md b/.changeset/fair-ants-look.md new file mode 100644 index 0000000000..4e70a042f8 --- /dev/null +++ b/.changeset/fair-ants-look.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +**BREAKING**: Removed `AwsS3DiscoveryProcessor`, which now instead should be imported from `@backstage/plugin-catalog-backend-module-aws`. diff --git a/.changeset/twenty-planes-dress.md b/.changeset/twenty-planes-dress.md new file mode 100644 index 0000000000..4d1a5dc1d1 --- /dev/null +++ b/.changeset/twenty-planes-dress.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +--- + +Added `AwsS3DiscoveryProcessor`, which was moved here from `@backstage/plugin-catalog-backend` where it previously resided. diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md index 545a578e63..4e72c80b95 100644 --- a/docs/integrations/aws-s3/discovery.md +++ b/docs/integrations/aws-s3/discovery.md @@ -33,7 +33,7 @@ the below to `packages/backend/src/plugins/catalog.ts`: ```ts /* packages/backend/src/plugins/catalog.ts */ -import { AwsS3DiscoveryProcessor } from '@backstage/plugin-catalog-backend'; +import { AwsS3DiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-aws'; const builder = await CatalogBuilder.create(env); /** ... other processors ... */ diff --git a/plugins/catalog-backend-module-aws/api-report.md b/plugins/catalog-backend-module-aws/api-report.md index 1cab6555fe..ef58210e11 100644 --- a/plugins/catalog-backend-module-aws/api-report.md +++ b/plugins/catalog-backend-module-aws/api-report.md @@ -5,9 +5,11 @@ ```ts import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; +import { CatalogProcessorParser } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger as Logger_2 } from 'winston'; +import { UrlReader } from '@backstage/backend-common'; // @public export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { @@ -27,4 +29,18 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, ): Promise; } + +// @public +export class AwsS3DiscoveryProcessor implements CatalogProcessor { + constructor(reader: UrlReader); + // (undocumented) + getProcessorName(): string; + // (undocumented) + readLocation( + location: LocationSpec, + optional: boolean, + emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, + ): Promise; +} ``` diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index 2ea57652cc..3f8149a6f7 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -33,6 +33,7 @@ "start": "backstage-cli package start" }, "dependencies": { + "@backstage/backend-common": "^0.12.0", "@backstage/catalog-model": "^0.12.0", "@backstage/config": "^0.1.15", "@backstage/errors": "^0.2.2", @@ -40,12 +41,14 @@ "@backstage/types": "^0.1.3", "aws-sdk": "^2.840.0", "lodash": "^4.17.21", + "p-limit": "^3.0.2", "winston": "^3.2.1" }, "devDependencies": { "@backstage/cli": "^0.15.0", "@types/lodash": "^4.14.151", - "aws-sdk-mock": "^5.2.1" + "aws-sdk-mock": "^5.2.1", + "yaml": "^1.9.2" }, "files": [ "dist", diff --git a/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts similarity index 85% rename from plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts rename to plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts index 249de71591..6aab7b2bec 100644 --- a/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts @@ -20,11 +20,12 @@ import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; import { CatalogProcessorEntityResult, CatalogProcessorResult, -} from '../../api'; -import { defaultEntityDataParser } from '../util/parse'; + processingResult, +} from '@backstage/plugin-catalog-backend'; import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; +import YAML from 'yaml'; AWSMock.setSDKInstance(aws); const object: aws.S3.Types.Object = { @@ -62,7 +63,17 @@ describe('readLocation', () => { it('should load from url', async () => { const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit, defaultEntityDataParser), + processor.readLocation( + spec, + false, + emit, + async function* r({ data, location }) { + yield processingResult.entity( + location, + YAML.parse(data.toString('utf8')) as any, + ); + }, + ), )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); expect(generated.location).toEqual({ diff --git a/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts b/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.ts similarity index 89% rename from plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts rename to plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.ts index 737f8f7c2f..0390defb2a 100644 --- a/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.ts @@ -16,16 +16,22 @@ import { UrlReader } from '@backstage/backend-common'; import { isError } from '@backstage/errors'; -import limiterFactory from 'p-limit'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, processingResult, -} from '../../api'; +} from '@backstage/plugin-catalog-backend'; +import limiterFactory from 'p-limit'; -/** @public */ +/** + * A processor for automatic discovery of entities from S3 buckets. Handles the + * `s3-discovery` location type, and target bucket URLs e.g. on the form + * `https://testbucket.s3.us-east-2.amazonaws.com`. + * + * @public + */ export class AwsS3DiscoveryProcessor implements CatalogProcessor { constructor(private readonly reader: UrlReader) {} diff --git a/plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt b/plugins/catalog-backend-module-aws/src/processors/__fixtures__/awsS3-mock-object.txt similarity index 100% rename from plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt rename to plugins/catalog-backend-module-aws/src/processors/__fixtures__/awsS3-mock-object.txt diff --git a/plugins/catalog-backend-module-aws/src/processors/index.ts b/plugins/catalog-backend-module-aws/src/processors/index.ts index fa858aa744..8718e77a6f 100644 --- a/plugins/catalog-backend-module-aws/src/processors/index.ts +++ b/plugins/catalog-backend-module-aws/src/processors/index.ts @@ -15,3 +15,4 @@ */ export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 64ab27ba25..922e1d2741 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -97,20 +97,6 @@ export class AnnotateScmSlugEntityProcessor implements CatalogProcessor { preProcessEntity(entity: Entity, location: LocationSpec): Promise; } -// @public (undocumented) -export class AwsS3DiscoveryProcessor implements CatalogProcessor { - constructor(reader: UrlReader); - // (undocumented) - getProcessorName(): string; - // (undocumented) - readLocation( - location: LocationSpec, - optional: boolean, - emit: CatalogProcessorEmit, - parser: CatalogProcessorParser, - ): Promise; -} - // @public export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { constructor(options: { diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 9b8e6347a5..abc7f85427 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -48,7 +48,6 @@ "@backstage/types": "^0.1.3", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", - "aws-sdk": "^2.840.0", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "express": "^4.17.1", @@ -81,7 +80,6 @@ "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", "@vscode/sqlite3": "^5.0.7", - "aws-sdk-mock": "^5.2.1", "msw": "^0.35.0", "supertest": "^6.1.3", "wait-for-expect": "^3.0.2", diff --git a/plugins/catalog-backend/src/modules/aws/index.ts b/plugins/catalog-backend/src/modules/aws/index.ts deleted file mode 100644 index 9477d4104d..0000000000 --- a/plugins/catalog-backend/src/modules/aws/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/modules/index.ts b/plugins/catalog-backend/src/modules/index.ts index 6acf057ca3..1cb7e3007b 100644 --- a/plugins/catalog-backend/src/modules/index.ts +++ b/plugins/catalog-backend/src/modules/index.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -export * from './aws'; export * from './azure'; export * from './bitbucket'; export * from './codeowners';