Merge pull request #23022 from RoadieHQ/quiter-logs

Make catalog logging a little quieter where entities are erroring

.changeset/selfish-rivers-fetch.md

@@ -0,0 +1,37 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Make entity collection errors a little quieter in the logs.
+
+Instead of logging a warning line when an entity has an error
+during processing, it will now instead emit an event on the event
+broker.
+
+This only removes a single log line, however it is possible to
+add the log line back if it is required by subscribing to the
+`CATALOG_ERRORS_TOPIC` as shown below.
+
+```typescript
+env.eventBroker.subscribe({
+  supportsEventTopics(): string[] {
+    return [CATALOG_ERRORS_TOPIC];
+  },
+
+  async onEvent(
+    params: EventParams<{
+      entity: string;
+      location?: string;
+      errors: Array<Error>;
+    }>,
+  ): Promise<void> {
+    const { entity, location, errors } = params.eventPayload;
+    for (const error of errors) {
+      env.logger.warn(error.message, {
+        entity,
+        location,
+      });
+    }
+  },
+});
+```

plugins/catalog-backend/api-report.md

@@ -132,6 +132,9 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor_2 {
 // @public (undocumented)
 export const CATALOG_CONFLICTS_TOPIC = 'experimental.catalog.conflict';
 
+// @public (undocumented)
+export const CATALOG_ERRORS_TOPIC = 'experimental.catalog.errors';
+
 // @public
 export class CatalogBuilder {
   addEntityPolicy(

plugins/catalog-backend/src/constants.ts

@@ -16,3 +16,5 @@
 
 /** @public */
 export const CATALOG_CONFLICTS_TOPIC = 'experimental.catalog.conflict';
+/** @public */
+export const CATALOG_ERRORS_TOPIC = 'experimental.catalog.errors';

plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts

@@ -38,6 +38,8 @@ import {
   withActiveSpan,
 } from '../util/opentelemetry';
 import { deleteOrphanedEntities } from '../database/operations/util/deleteOrphanedEntities';
+import { EventBroker } from '@backstage/plugin-events-node';
+import { CATALOG_ERRORS_TOPIC } from '../constants';
 
 const CACHE_TTL = 5;
 
@@ -67,6 +69,7 @@ export class DefaultCatalogProcessingEngine {
     errors: Error[];
   }) => Promise<void> | void;
   private readonly tracker: ProgressTracker;
+  private readonly eventBroker?: EventBroker;
 
   private stopFunc?: () => void;
 
@@ -86,6 +89,7 @@ export class DefaultCatalogProcessingEngine {
       errors: Error[];
     }) => Promise<void> | void;
     tracker?: ProgressTracker;
+    eventBroker?: EventBroker;
   }) {
     this.config = options.config;
     this.scheduler = options.scheduler;
@@ -99,6 +103,7 @@ export class DefaultCatalogProcessingEngine {
     this.orphanCleanupIntervalMs = options.orphanCleanupIntervalMs ?? 30_000;
     this.onProcessingError = options.onProcessingError;
     this.tracker = options.tracker ?? progressTracker();
+    this.eventBroker = options.eventBroker;
 
     this.stopFunc = undefined;
   }
@@ -194,10 +199,14 @@ export class DefaultCatalogProcessingEngine {
 
             const location =
               unprocessedEntity?.metadata?.annotations?.[ANNOTATION_LOCATION];
-            for (const error of result.errors) {
-              this.logger.warn(error.message, {
-                entity: entityRef,
-                location,
+            if (result.errors.length) {
+              this.eventBroker?.publish({
+                topic: CATALOG_ERRORS_TOPIC,
+                eventPayload: {
+                  entity: entityRef,
+                  location,
+                  errors: result.errors,
+                },
               });
             }
             const errorsString = JSON.stringify(

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -591,6 +591,7 @@ export class CatalogBuilder {
       onProcessingError: event => {
         this.onProcessingError?.(event);
       },
+      eventBroker: this.eventBroker,
     });
 
     const locationAnalyzer =