Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000

Signed-off-by: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com>

.changeset/gentle-cobras-kick.md

@@ -0,0 +1,13 @@
+---
+'@backstage/plugin-search-backend-module-elasticsearch': patch
+---
+
+Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000
+
+Example usage:
+
+```yaml
+search:
+  elasticsearch:
+    batchSize: 100
+```

plugins/search-backend-module-elasticsearch/api-report.md

@@ -315,6 +315,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
     aliasPostfix: string,
     indexPrefix: string,
     logger: Logger,
+    batchSize: number,
     highlightOptions?: ElasticSearchHighlightOptions,
   );
   // (undocumented)
@@ -361,6 +362,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
   alias: string;
   logger: Logger;
   elasticSearchClientWrapper: ElasticSearchClientWrapper;
+  batchSize: number;
 };
 
 // @public (undocumented)

plugins/search-backend-module-elasticsearch/config.d.ts

@@ -21,6 +21,10 @@ export interface Config {
      * Options for ElasticSearch
      */
     elasticsearch?: {
+      /**
+       * Batch size for elastic search indexing tasks. Defaults to 1000.
+       */
+      batchSize?: number;
       /**
        * Options for configuring highlight settings
        * See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/highlighting.html

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts

@@ -79,12 +79,14 @@ describe('ElasticSearchSearchEngine', () => {
       'search',
       '',
       getVoidLogger(),
+      1000,
     );
     inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests(
       options,
       'search',
       '',
       getVoidLogger(),
+      1000,
     );
     // eslint-disable-next-line dot-notation
     clientWrapper = testSearchEngine['elasticSearchClientWrapper'];

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts

@@ -107,6 +107,8 @@ function isBlank(str: string) {
   return (isEmpty(str) && !isNumber(str)) || nan(str);
 }
 
+const DEFAULT_INDEXER_BATCH_SIZE = 1000;
+
 /**
  * @public
  */
@@ -119,6 +121,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
     private readonly aliasPostfix: string,
     private readonly indexPrefix: string,
     private readonly logger: Logger,
+    private readonly batchSize: number,
     highlightOptions?: ElasticSearchHighlightOptions,
   ) {
     this.elasticSearchClientWrapper =
@@ -156,6 +159,8 @@ export class ElasticSearchSearchEngine implements SearchEngine {
       aliasPostfix,
       indexPrefix,
       logger,
+      config.getOptionalNumber('search.elasticsearch.batchSize') ??
+        DEFAULT_INDEXER_BATCH_SIZE,
       config.getOptional<ElasticSearchHighlightOptions>(
         'search.elasticsearch.highlightOptions',
       ),
@@ -255,6 +260,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
       alias,
       elasticSearchClientWrapper: this.elasticSearchClientWrapper,
       logger: this.logger,
+      batchSize: this.batchSize,
     });
 
     // Attempt cleanup upon failure.

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts

@@ -44,6 +44,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
       alias: 'some-type-index__search',
       logger: getVoidLogger(),
       elasticSearchClientWrapper: clientWrapper,
+      batchSize: 1000,
     });
 
     // Set up all requisite Elastic mocks.

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts

@@ -31,6 +31,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
   alias: string;
   logger: Logger;
   elasticSearchClientWrapper: ElasticSearchClientWrapper;
+  batchSize: number;
 };
 
 function duration(startTimestamp: [number, number]): string {
@@ -61,7 +62,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
   private bulkResult: Promise<any>;
 
   constructor(options: ElasticSearchSearchEngineIndexerOptions) {
-    super({ batchSize: 1000 });
+    super({ batchSize: options.batchSize });
     this.logger = options.logger;
     this.startTimestamp = process.hrtime();
     this.type = options.type;