Merge pull request #12534 from nodify-at/master

.changeset/search-gentle-cobras-kick.md

@@ -0,0 +1,13 @@
+---
+'@backstage/plugin-search-backend-module-elasticsearch': patch
+---
+
+Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1000
+
+Example usage:
+
+```yaml
+search:
+  elasticsearch:
+    batchSize: 100
+```

docs/features/search/search-engines.md

@@ -232,3 +232,23 @@ search:
     auth:
       apiKey: base64EncodedKey
 ```
+
+### Elastic search batch size
+
+Default batch size of the elastic search engine is set to 1000. If you are using a lower spec computing resources (like AWS small instance),
+you may get an error caused by limited `thread_pool` configuration. ( `429 Too Many Requests /_bulk` )
+
+In this case you need to decrease the batch size to index the resources to prevent this kind of error. You can easily decrease
+or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for elasticsearch configuration.
+
+#### Configuration example
+
+**Set batch size to 100**
+
+```yaml
+search:
+  elasticsearch:
+    batchSize: 100
+```
+
+> You can also increase the batch size if you are using a large ES instance.

plugins/search-backend-module-elasticsearch/api-report.md

@@ -331,6 +331,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
     aliasPostfix: string,
     indexPrefix: string,
     logger: Logger,
+    batchSize: number,
     highlightOptions?: ElasticSearchHighlightOptions,
   );
   // (undocumented)
@@ -377,6 +378,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
   alias: string;
   logger: Logger;
   elasticSearchClientWrapper: ElasticSearchClientWrapper;
+  batchSize: number;
 };
 
 // @public (undocumented)

plugins/search-backend-module-elasticsearch/config.d.ts

@@ -21,6 +21,10 @@ export interface Config {
      * Options for ElasticSearch
      */
     elasticsearch?: {
+      /**
+       * Batch size for elastic search indexing tasks. Defaults to 1000.
+       */
+      batchSize?: number;
       /**
        * Options for configuring highlight settings
        * See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/highlighting.html

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts

@@ -79,12 +79,14 @@ describe('ElasticSearchSearchEngine', () => {
       'search',
       '',
       getVoidLogger(),
+      1000,
     );
     inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests(
       options,
       'search',
       '',
       getVoidLogger(),
+      1000,
     );
     // eslint-disable-next-line dot-notation
     clientWrapper = testSearchEngine['elasticSearchClientWrapper'];

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts

@@ -107,6 +107,8 @@ function isBlank(str: string) {
   return (isEmpty(str) && !isNumber(str)) || nan(str);
 }
 
+const DEFAULT_INDEXER_BATCH_SIZE = 1000;
+
 /**
  * @public
  */
@@ -119,6 +121,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
     private readonly aliasPostfix: string,
     private readonly indexPrefix: string,
     private readonly logger: Logger,
+    private readonly batchSize: number,
     highlightOptions?: ElasticSearchHighlightOptions,
   ) {
     this.elasticSearchClientWrapper =
@@ -156,6 +159,8 @@ export class ElasticSearchSearchEngine implements SearchEngine {
       aliasPostfix,
       indexPrefix,
       logger,
+      config.getOptionalNumber('search.elasticsearch.batchSize') ??
+        DEFAULT_INDEXER_BATCH_SIZE,
       config.getOptional<ElasticSearchHighlightOptions>(
         'search.elasticsearch.highlightOptions',
       ),
@@ -255,6 +260,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
       alias,
       elasticSearchClientWrapper: this.elasticSearchClientWrapper,
       logger: this.logger,
+      batchSize: this.batchSize,
     });
 
     // Attempt cleanup upon failure.

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts

@@ -44,6 +44,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
       alias: 'some-type-index__search',
       logger: getVoidLogger(),
       elasticSearchClientWrapper: clientWrapper,
+      batchSize: 1000,
     });
 
     // Set up all requisite Elastic mocks.

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts

@@ -31,6 +31,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
   alias: string;
   logger: Logger;
   elasticSearchClientWrapper: ElasticSearchClientWrapper;
+  batchSize: number;
 };
 
 function duration(startTimestamp: [number, number]): string {
@@ -61,7 +62,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
   private bulkResult: Promise<any>;
 
   constructor(options: ElasticSearchSearchEngineIndexerOptions) {
-    super({ batchSize: 1000 });
+    super({ batchSize: options.batchSize });
     this.logger = options.logger;
     this.startTimestamp = process.hrtime();
     this.type = options.type;