From 4f0a68bd7c2672357ca1245e73de778fb8d55025 Mon Sep 17 00:00:00 2001
From: Brian Fletcher <brian@roadie.io>
Date: Mon, 26 Feb 2024 10:00:54 +0000
Subject: [PATCH] add an event when errors occur in the catalog

Signed-off-by: Brian Fletcher <brian@roadie.io>
---
 .changeset/selfish-rivers-fetch.md            | 32 +++++++++++++++++++
 packages/backend/src/plugins/catalog.ts       |  7 +++-
 plugins/catalog-backend/src/constants.ts      |  1 +
 .../DefaultCatalogProcessingEngine.ts         | 17 +++++++---
 .../src/service/CatalogBuilder.ts             |  1 +
 5 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/.changeset/selfish-rivers-fetch.md b/.changeset/selfish-rivers-fetch.md
index 8b84766662..04fda7f3ef 100644
--- a/.changeset/selfish-rivers-fetch.md
+++ b/.changeset/selfish-rivers-fetch.md
@@ -3,3 +3,35 @@
 ---
 
 Make entity collection errors a little quieter in the logs.
+
+Instead of logging a warning line when an entity has an error
+during processing, it will now instead emit an event on the event
+broker.
+
+This only removes a single log line, however it is possible to
+add the log line back if it is required by subscribing to the
+`CATALOG_ERRORS_TOPIC` as shown below.
+
+```typescript
+env.eventBroker.subscribe({
+  supportsEventTopics(): string[] {
+    return [CATALOG_ERRORS_TOPIC];
+  },
+
+  async onEvent(
+    params: EventParams<{
+      entity: string;
+      location?: string;
+      errors: Array<Error>;
+    }>,
+  ): Promise<void> {
+    const { entity, location, errors } = params.eventPayload;
+    for (const error of errors) {
+      env.logger.warn(error.message, {
+        entity,
+        location,
+      });
+    }
+  },
+});
+```
diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts
index 223acab818..255226d5d3 100644
--- a/packages/backend/src/plugins/catalog.ts
+++ b/packages/backend/src/plugins/catalog.ts
@@ -14,12 +14,16 @@
  * limitations under the License.
  */
 
-import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+import {
+  CATALOG_ERRORS_TOPIC,
+  CatalogBuilder,
+} from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-catalog-backend-module-scaffolder-entity-model';
 import { UnprocessedEntitiesModule } from '@backstage/plugin-catalog-backend-module-unprocessed';
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
 import { DemoEventBasedEntityProvider } from './DemoEventBasedEntityProvider';
+import { EventParams } from '@backstage/plugin-events-node';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -33,6 +37,7 @@ export default async function createPlugin(
     eventBroker: env.eventBroker,
   });
   builder.addEntityProvider(demoProvider);
+  builder.setEventBroker(env.eventBroker);
 
   const { processingEngine, router } = await builder.build();
 
diff --git a/plugins/catalog-backend/src/constants.ts b/plugins/catalog-backend/src/constants.ts
index 00ae2673e2..20b54cbad2 100644
--- a/plugins/catalog-backend/src/constants.ts
+++ b/plugins/catalog-backend/src/constants.ts
@@ -16,3 +16,4 @@
 
 /** @public */
 export const CATALOG_CONFLICTS_TOPIC = 'experimental.catalog.conflict';
+export const CATALOG_ERRORS_TOPIC = 'experimental.catalog.errors';
diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts
index b17482641c..81c31cc1c7 100644
--- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts
+++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts
@@ -38,6 +38,8 @@ import {
   withActiveSpan,
 } from '../util/opentelemetry';
 import { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';
+import { EventBroker } from '@backstage/plugin-events-node';
+import { CATALOG_ERRORS_TOPIC } from '../constants';
 
 const CACHE_TTL = 5;
 
@@ -67,6 +69,7 @@ export class DefaultCatalogProcessingEngine {
     errors: Error[];
   }) => Promise<void> | void;
   private readonly tracker: ProgressTracker;
+  private readonly eventBroker?: EventBroker;
 
   private stopFunc?: () => void;
 
@@ -86,6 +89,7 @@ export class DefaultCatalogProcessingEngine {
       errors: Error[];
     }) => Promise<void> | void;
     tracker?: ProgressTracker;
+    eventBroker?: EventBroker;
   }) {
     this.config = options.config;
     this.scheduler = options.scheduler;
@@ -99,6 +103,7 @@ export class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
+    this.eventBroker = options.eventBroker;
 
     this.stopFunc = undefined;
   }
@@ -194,10 +199,14 @@ export class DefaultCatalogProcessingEngine {
 
             const location =
               unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];
-            for (const error of result.errors) {
-              this.logger.debug(error.message, {
-                entity: entityRef,
-                location,
+            if (result.errors.length) {
+              this.eventBroker?.publish({
+                topic: CATALOG_ERRORS_TOPIC,
+                eventPayload: {
+                  entity: entityRef,
+                  location,
+                  errors: result.errors,
+                },
               });
             }
             const errorsString = JSON.stringify(
diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts
index 0abaf2ec0a..61a5dc8c21 100644
--- a/plugins/catalog-backend/src/service/CatalogBuilder.ts
+++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts
@@ -549,6 +549,7 @@ export class CatalogBuilder {
       onProcessingError: event => {
         this.onProcessingError?.(event);
       },
+      eventBroker: this.eventBroker,
     });
 
     const locationAnalyzer =