Merge pull request #19586 from awanlin/topic/logging-techdocs-collating-errors

Added indexerBatchSize option and debug logging

plugins/search-backend-module-pg/config.d.ts

@@ -57,6 +57,10 @@ export interface Config {
          */
         fragmentDelimiter?: string;
       };
+      /**
+       * Batch size to use when indexing
+       */
+      indexerBatchSize?: number;
     };
   };
 }

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts

@@ -86,6 +86,7 @@ export type PgSearchHighlightOptions = {
 export class PgSearchEngine implements SearchEngine {
   private readonly logger?: Logger;
   private readonly highlightOptions: PgSearchHighlightOptions;
+  private readonly indexerBatchSize: number;
 
   /**
    * @deprecated This will be marked as private in a future release, please us fromConfig instead
@@ -114,6 +115,8 @@ export class PgSearchEngine implements SearchEngine {
         highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ',
     };
     this.highlightOptions = highlightOptions;
+    this.indexerBatchSize =
+      config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000;
     this.logger = logger;
   }
 
@@ -178,7 +181,7 @@ export class PgSearchEngine implements SearchEngine {
 
   async getIndexer(type: string) {
     return new PgSearchEngineIndexer({
-      batchSize: 1000,
+      batchSize: this.indexerBatchSize,
       type,
       databaseStore: this.databaseStore,
       logger: this.logger?.child({ documentType: type }),

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts

@@ -59,6 +59,11 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer {
   async index(documents: IndexableDocument[]): Promise<void> {
     this.numRecords += documents.length;
 
+    const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))];
+    this.logger.debug(
+      `Attempting to index the following entities: ${refs.toString()}`,
+    );
+
     try {
       await this.store.insertDocuments(this.tx!, this.type, documents);
     } catch (e) {