add an event when errors occur in the catalog

Signed-off-by: Brian Fletcher <brian@roadie.io>

.changeset/selfish-rivers-fetch.md

@@ -3,3 +3,35 @@
 ---
 
 Make entity collection errors a little quieter in the logs.
+
+Instead of logging a warning line when an entity has an error
+during processing, it will now instead emit an event on the event
+broker.
+
+This only removes a single log line, however it is possible to
+add the log line back if it is required by subscribing to the
+`CATALOG_ERRORS_TOPIC` as shown below.
+
+```typescript
+env.eventBroker.subscribe({
+  supportsEventTopics(): string[] {
+    return [CATALOG_ERRORS_TOPIC];
+  },
+
+  async onEvent(
+    params: EventParams<{
+      entity: string;
+      location?: string;
+      errors: Array<Error>;
+    }>,
+  ): Promise<void> {
+    const { entity, location, errors } = params.eventPayload;
+    for (const error of errors) {
+      env.logger.warn(error.message, {
+        entity,
+        location,
+      });
+    }
+  },
+});
+```

packages/backend/src/plugins/catalog.ts

@@ -14,12 +14,16 @@
  * limitations under the License.
  */
 
-import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+import {
+  CATALOG_ERRORS_TOPIC,
+  CatalogBuilder,
+} from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-catalog-backend-module-scaffolder-entity-model';
 import { UnprocessedEntitiesModule } from '@backstage/plugin-catalog-backend-module-unprocessed';
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
 import { DemoEventBasedEntityProvider } from './DemoEventBasedEntityProvider';
+import { EventParams } from '@backstage/plugin-events-node';
 
 export default async function createPlugin(
   env: PluginEnvironment,
@@ -33,6 +37,7 @@ export default async function createPlugin(
     eventBroker: env.eventBroker,
   });
   builder.addEntityProvider(demoProvider);
+  builder.setEventBroker(env.eventBroker);
 
   const { processingEngine, router } = await builder.build();
 

plugins/catalog-backend/src/constants.ts

@@ -16,3 +16,4 @@
 
 /** @public */
 export const CATALOG_CONFLICTS_TOPIC = 'experimental.catalog.conflict';
+export const CATALOG_ERRORS_TOPIC = 'experimental.catalog.errors';

plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts

@@ -38,6 +38,8 @@ import {
   withActiveSpan,
 } from '../util/opentelemetry';
 import { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';
+import { EventBroker } from '@backstage/plugin-events-node';
+import { CATALOG_ERRORS_TOPIC } from '../constants';
 
 const CACHE_TTL = 5;
 
@@ -67,6 +69,7 @@ export class DefaultCatalogProcessingEngine {
     errors: Error[];
   }) => Promise<void> | void;
   private readonly tracker: ProgressTracker;
+  private readonly eventBroker?: EventBroker;
 
   private stopFunc?: () => void;
 
@@ -86,6 +89,7 @@ export class DefaultCatalogProcessingEngine {
       errors: Error[];
     }) => Promise<void> | void;
     tracker?: ProgressTracker;
+    eventBroker?: EventBroker;
   }) {
     this.config = options.config;
     this.scheduler = options.scheduler;
@@ -99,6 +103,7 @@ export class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
+    this.eventBroker = options.eventBroker;
 
     this.stopFunc = undefined;
   }
@@ -194,10 +199,14 @@ export class DefaultCatalogProcessingEngine {
 
             const location =
               unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];
-            for (const error of result.errors) {
-              this.logger.debug(error.message, {
-                entity: entityRef,
-                location,
+            if (result.errors.length) {
+              this.eventBroker?.publish({
+                topic: CATALOG_ERRORS_TOPIC,
+                eventPayload: {
+                  entity: entityRef,
+                  location,
+                  errors: result.errors,
+                },
               });
             }
             const errorsString = JSON.stringify(

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -549,6 +549,7 @@ export class CatalogBuilder {
       onProcessingError: event => {
         this.onProcessingError?.(event);
       },
+      eventBroker: this.eventBroker,
     });
 
     const locationAnalyzer =