diff --git a/.changeset/search-gentle-cobras-kick.md b/.changeset/search-gentle-cobras-kick.md new file mode 100644 index 0000000000..3ddd4df67c --- /dev/null +++ b/.changeset/search-gentle-cobras-kick.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1000 + +Example usage: + +```yaml +search: + elasticsearch: + batchSize: 100 +``` diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index ecd8a85109..515c1268d7 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -232,3 +232,23 @@ search: auth: apiKey: base64EncodedKey ``` + +### Elastic search batch size + +Default batch size of the elastic search engine is set to 1000. If you are using a lower spec computing resources (like AWS small instance), +you may get an error caused by limited `thread_pool` configuration. ( `429 Too Many Requests /_bulk` ) + +In this case you need to decrease the batch size to index the resources to prevent this kind of error. You can easily decrease +or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for elasticsearch configuration. + +#### Configuration example + +**Set batch size to 100** + +```yaml +search: + elasticsearch: + batchSize: 100 +``` + +> You can also increase the batch size if you are using a large ES instance. diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index ff474fc9bf..5a951948c2 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -331,6 +331,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix: string, indexPrefix: string, logger: Logger, + batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ); // (undocumented) @@ -377,6 +378,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { alias: string; logger: Logger; elasticSearchClientWrapper: ElasticSearchClientWrapper; + batchSize: number; }; // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 8f63e66365..95e527b236 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -21,6 +21,10 @@ export interface Config { * Options for ElasticSearch */ elasticsearch?: { + /** + * Batch size for elastic search indexing tasks. Defaults to 1000. + */ + batchSize?: number; /** * Options for configuring highlight settings * See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/highlighting.html diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index b59c67c69c..64725e446d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -79,12 +79,14 @@ describe('ElasticSearchSearchEngine', () => { 'search', '', getVoidLogger(), + 1000, ); inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests( options, 'search', '', getVoidLogger(), + 1000, ); // eslint-disable-next-line dot-notation clientWrapper = testSearchEngine['elasticSearchClientWrapper']; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index cf0b4f441c..f3107df1fd 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -107,6 +107,8 @@ function isBlank(str: string) { return (isEmpty(str) && !isNumber(str)) || nan(str); } +const DEFAULT_INDEXER_BATCH_SIZE = 1000; + /** * @public */ @@ -119,6 +121,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly aliasPostfix: string, private readonly indexPrefix: string, private readonly logger: Logger, + private readonly batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ) { this.elasticSearchClientWrapper = @@ -156,6 +159,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix, indexPrefix, logger, + config.getOptionalNumber('search.elasticsearch.batchSize') ?? + DEFAULT_INDEXER_BATCH_SIZE, config.getOptional( 'search.elasticsearch.highlightOptions', ), @@ -255,6 +260,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { alias, elasticSearchClientWrapper: this.elasticSearchClientWrapper, logger: this.logger, + batchSize: this.batchSize, }); // Attempt cleanup upon failure. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index eff056c3f4..a19687a03b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -44,6 +44,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { alias: 'some-type-index__search', logger: getVoidLogger(), elasticSearchClientWrapper: clientWrapper, + batchSize: 1000, }); // Set up all requisite Elastic mocks. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 87a4472a15..c81950ea41 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -31,6 +31,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { alias: string; logger: Logger; elasticSearchClientWrapper: ElasticSearchClientWrapper; + batchSize: number; }; function duration(startTimestamp: [number, number]): string { @@ -61,7 +62,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private bulkResult: Promise; constructor(options: ElasticSearchSearchEngineIndexerOptions) { - super({ batchSize: 1000 }); + super({ batchSize: options.batchSize }); this.logger = options.logger; this.startTimestamp = process.hrtime(); this.type = options.type;