Added indexerBatchSize option and debug logging

Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com>

.changeset/brave-carrots-occur.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-pg': patch
+---
+
+Added `indexerBatchSize` option to be able to control the size of the batches being indexed. Also added a debug log entry to list out all the entities in the batch

packages/backend/package.json

@@ -65,8 +65,11 @@
     "@backstage/plugin-scaffolder-backend-module-confluence-to-markdown": "workspace:^",
     "@backstage/plugin-scaffolder-backend-module-rails": "workspace:^",
     "@backstage/plugin-search-backend": "workspace:^",
+    "@backstage/plugin-search-backend-module-catalog": "workspace:^",
     "@backstage/plugin-search-backend-module-elasticsearch": "workspace:^",
+    "@backstage/plugin-search-backend-module-explore": "workspace:^",
     "@backstage/plugin-search-backend-module-pg": "workspace:^",
+    "@backstage/plugin-search-backend-module-techdocs": "workspace:^",
     "@backstage/plugin-search-backend-node": "workspace:^",
     "@backstage/plugin-search-common": "workspace:^",
     "@backstage/plugin-tech-insights-backend": "workspace:^",

packages/backend/src/plugins/search.ts

@@ -16,8 +16,8 @@
 
 import { useHotCleanup } from '@backstage/backend-common';
 import { DefaultAdrCollatorFactory } from '@backstage/plugin-adr-backend';
-import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend';
-import { ToolDocumentCollatorFactory } from '@backstage/plugin-explore-backend';
+import { DefaultCatalogCollatorFactory } from '@backstage/plugin-search-backend-module-catalog';
+import { ToolDocumentCollatorFactory } from '@backstage/plugin-search-backend-module-explore';
 import { createRouter } from '@backstage/plugin-search-backend';
 import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
 import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg';
@@ -26,7 +26,7 @@ import {
   LunrSearchEngine,
 } from '@backstage/plugin-search-backend-node';
 import { SearchEngine } from '@backstage/plugin-search-common';
-import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend';
+import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-search-backend-module-techdocs';
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
 
@@ -43,6 +43,7 @@ async function createSearchEngine(
   if (await PgSearchEngine.supported(env.database)) {
     return await PgSearchEngine.fromConfig(env.config, {
       database: env.database,
+      logger: env.logger,
     });
   }
 

plugins/search-backend-module-pg/config.d.ts

@@ -57,6 +57,10 @@ export interface Config {
          */
         fragmentDelimiter?: string;
       };
+      /**
+       * Batch size to use when indexing
+       */
+      indexerBatchSize?: number;
     };
   };
 }

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts

@@ -86,6 +86,7 @@ export type PgSearchHighlightOptions = {
 export class PgSearchEngine implements SearchEngine {
   private readonly logger?: Logger;
   private readonly highlightOptions: PgSearchHighlightOptions;
+  private readonly indexerBatchSize: number;
 
   /**
    * @deprecated This will be marked as private in a future release, please us fromConfig instead
@@ -114,6 +115,8 @@ export class PgSearchEngine implements SearchEngine {
         highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ',
     };
     this.highlightOptions = highlightOptions;
+    this.indexerBatchSize =
+      config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000;
     this.logger = logger;
   }
 
@@ -178,7 +181,7 @@ export class PgSearchEngine implements SearchEngine {
 
   async getIndexer(type: string) {
     return new PgSearchEngineIndexer({
-      batchSize: 1000,
+      batchSize: this.indexerBatchSize,
       type,
       databaseStore: this.databaseStore,
       logger: this.logger?.child({ documentType: type }),

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts

@@ -59,6 +59,11 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer {
   async index(documents: IndexableDocument[]): Promise<void> {
     this.numRecords += documents.length;
 
+    const refs = [...new Set(documents.map(d => d.authorization?.resourceRef))];
+    this.logger.debug(
+      `Attempting to index the following entities: ${refs.toString()}`,
+    );
+
     try {
       await this.store.insertDocuments(this.tx!, this.type, documents);
     } catch (e) {

yarn.lock

@@ -25256,8 +25256,11 @@ __metadata:
     "@backstage/plugin-scaffolder-backend-module-confluence-to-markdown": "workspace:^"
     "@backstage/plugin-scaffolder-backend-module-rails": "workspace:^"
     "@backstage/plugin-search-backend": "workspace:^"
+    "@backstage/plugin-search-backend-module-catalog": "workspace:^"
     "@backstage/plugin-search-backend-module-elasticsearch": "workspace:^"
+    "@backstage/plugin-search-backend-module-explore": "workspace:^"
     "@backstage/plugin-search-backend-module-pg": "workspace:^"
+    "@backstage/plugin-search-backend-module-techdocs": "workspace:^"
     "@backstage/plugin-search-backend-node": "workspace:^"
     "@backstage/plugin-search-common": "workspace:^"
     "@backstage/plugin-tech-insights-backend": "workspace:^"