moved AwsS3DiscoveryProcessor to where it should live

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

.changeset/fair-ants-look.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+**BREAKING**: Removed `AwsS3DiscoveryProcessor`, which now instead should be imported from `@backstage/plugin-catalog-backend-module-aws`.

.changeset/twenty-planes-dress.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-aws': patch
+---
+
+Added `AwsS3DiscoveryProcessor`, which was moved here from `@backstage/plugin-catalog-backend` where it previously resided.

docs/integrations/aws-s3/discovery.md

@@ -33,7 +33,7 @@ the below to `packages/backend/src/plugins/catalog.ts`:
 ```ts
 /* packages/backend/src/plugins/catalog.ts */
 
-import { AwsS3DiscoveryProcessor } from '@backstage/plugin-catalog-backend';
+import { AwsS3DiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-aws';
 
 const builder = await CatalogBuilder.create(env);
 /** ... other processors ... */

plugins/catalog-backend-module-aws/api-report.md

@@ -5,9 +5,11 @@
 ```ts
 import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
+import { CatalogProcessorParser } from '@backstage/plugin-catalog-backend';
 import { Config } from '@backstage/config';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger as Logger_2 } from 'winston';
+import { UrlReader } from '@backstage/backend-common';
 
 // @public
 export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
@@ -27,4 +29,18 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
     emit: CatalogProcessorEmit,
   ): Promise<boolean>;
 }
+
+// @public
+export class AwsS3DiscoveryProcessor implements CatalogProcessor {
+  constructor(reader: UrlReader);
+  // (undocumented)
+  getProcessorName(): string;
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean>;
+}
 ```

plugins/catalog-backend-module-aws/package.json

@@ -33,6 +33,7 @@
     "start": "backstage-cli package start"
   },
   "dependencies": {
+    "@backstage/backend-common": "^0.12.0",
     "@backstage/catalog-model": "^0.12.0",
     "@backstage/config": "^0.1.15",
     "@backstage/errors": "^0.2.2",
@@ -40,12 +41,14 @@
     "@backstage/types": "^0.1.3",
     "aws-sdk": "^2.840.0",
     "lodash": "^4.17.21",
+    "p-limit": "^3.0.2",
     "winston": "^3.2.1"
   },
   "devDependencies": {
     "@backstage/cli": "^0.15.0",
     "@types/lodash": "^4.14.151",
-    "aws-sdk-mock": "^5.2.1"
+    "aws-sdk-mock": "^5.2.1",
+    "yaml": "^1.9.2"
   },
   "files": [
     "dist",

plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.test.ts

@@ -20,11 +20,12 @@ import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
 import {
   CatalogProcessorEntityResult,
   CatalogProcessorResult,
-} from '../../api';
-import { defaultEntityDataParser } from '../util/parse';
+  processingResult,
+} from '@backstage/plugin-catalog-backend';
 import AWSMock from 'aws-sdk-mock';
 import aws from 'aws-sdk';
 import path from 'path';
+import YAML from 'yaml';
 
 AWSMock.setSDKInstance(aws);
 const object: aws.S3.Types.Object = {
@@ -62,7 +63,17 @@ describe('readLocation', () => {
 
   it('should load from url', async () => {
     const generated = (await new Promise<CatalogProcessorResult>(emit =>
-      processor.readLocation(spec, false, emit, defaultEntityDataParser),
+      processor.readLocation(
+        spec,
+        false,
+        emit,
+        async function* r({ data, location }) {
+          yield processingResult.entity(
+            location,
+            YAML.parse(data.toString('utf8')) as any,
+          );
+        },
+      ),
     )) as CatalogProcessorEntityResult;
     expect(generated.type).toBe('entity');
     expect(generated.location).toEqual({

plugins/catalog-backend-module-aws/src/processors/AwsS3DiscoveryProcessor.ts

@@ -16,16 +16,22 @@
 
 import { UrlReader } from '@backstage/backend-common';
 import { isError } from '@backstage/errors';
-import limiterFactory from 'p-limit';
 import {
   CatalogProcessor,
   CatalogProcessorEmit,
   CatalogProcessorParser,
   LocationSpec,
   processingResult,
-} from '../../api';
+} from '@backstage/plugin-catalog-backend';
+import limiterFactory from 'p-limit';
 
-/** @public */
+/**
+ * A processor for automatic discovery of entities from S3 buckets. Handles the
+ * `s3-discovery` location type, and target bucket URLs e.g. on the form
+ * `https://testbucket.s3.us-east-2.amazonaws.com`.
+ *
+ * @public
+ */
 export class AwsS3DiscoveryProcessor implements CatalogProcessor {
   constructor(private readonly reader: UrlReader) {}
 

plugins/catalog-backend-module-aws/src/processors/index.ts

@@ -15,3 +15,4 @@
  */
 
 export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
+export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';

plugins/catalog-backend/api-report.md

@@ -97,20 +97,6 @@ export class AnnotateScmSlugEntityProcessor implements CatalogProcessor {
   preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
 }
 
-// @public (undocumented)
-export class AwsS3DiscoveryProcessor implements CatalogProcessor {
-  constructor(reader: UrlReader);
-  // (undocumented)
-  getProcessorName(): string;
-  // (undocumented)
-  readLocation(
-    location: LocationSpec,
-    optional: boolean,
-    emit: CatalogProcessorEmit,
-    parser: CatalogProcessorParser,
-  ): Promise<boolean>;
-}
-
 // @public
 export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
   constructor(options: {

plugins/catalog-backend/package.json

@@ -48,7 +48,6 @@
     "@backstage/types": "^0.1.3",
     "@octokit/graphql": "^4.5.8",
     "@types/express": "^4.17.6",
-    "aws-sdk": "^2.840.0",
     "codeowners-utils": "^1.0.2",
     "core-js": "^3.6.5",
     "express": "^4.17.1",
@@ -81,7 +80,6 @@
     "@types/supertest": "^2.0.8",
     "@types/uuid": "^8.0.0",
     "@vscode/sqlite3": "^5.0.7",
-    "aws-sdk-mock": "^5.2.1",
     "msw": "^0.35.0",
     "supertest": "^6.1.3",
     "wait-for-expect": "^3.0.2",

plugins/catalog-backend/src/modules/aws/index.ts

@@ -1,17 +0,0 @@
-/*
- * Copyright 2022 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';

plugins/catalog-backend/src/modules/index.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-export * from './aws';
 export * from './azure';
 export * from './bitbucket';
 export * from './codeowners';