From ab338422e1ff2127db19f67199f114c93e35f1ed Mon Sep 17 00:00:00 2001 From: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:39:22 +0200 Subject: [PATCH 1/3] Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000 Signed-off-by: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com> --- .changeset/gentle-cobras-kick.md | 13 +++++++++++++ .../api-report.md | 2 ++ .../search-backend-module-elasticsearch/config.d.ts | 4 ++++ .../src/engines/ElasticSearchSearchEngine.test.ts | 2 ++ .../src/engines/ElasticSearchSearchEngine.ts | 6 ++++++ .../ElasticSearchSearchEngineIndexer.test.ts | 1 + .../src/engines/ElasticSearchSearchEngineIndexer.ts | 3 ++- 7 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .changeset/gentle-cobras-kick.md diff --git a/.changeset/gentle-cobras-kick.md b/.changeset/gentle-cobras-kick.md new file mode 100644 index 0000000000..7f971a6ed7 --- /dev/null +++ b/.changeset/gentle-cobras-kick.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000 + +Example usage: + +```yaml +search: + elasticsearch: + batchSize: 100 +``` diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index eed2b1b10a..6ed6705b0a 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -315,6 +315,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix: string, indexPrefix: string, logger: Logger, + batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ); // (undocumented) @@ -361,6 +362,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { alias: string; logger: Logger; elasticSearchClientWrapper: ElasticSearchClientWrapper; + batchSize: number; }; // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 8f63e66365..95e527b236 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -21,6 +21,10 @@ export interface Config { * Options for ElasticSearch */ elasticsearch?: { + /** + * Batch size for elastic search indexing tasks. Defaults to 1000. + */ + batchSize?: number; /** * Options for configuring highlight settings * See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/highlighting.html diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index b59c67c69c..64725e446d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -79,12 +79,14 @@ describe('ElasticSearchSearchEngine', () => { 'search', '', getVoidLogger(), + 1000, ); inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests( options, 'search', '', getVoidLogger(), + 1000, ); // eslint-disable-next-line dot-notation clientWrapper = testSearchEngine['elasticSearchClientWrapper']; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 42c99dfe61..c4c9ec4d37 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -107,6 +107,8 @@ function isBlank(str: string) { return (isEmpty(str) && !isNumber(str)) || nan(str); } +const DEFAULT_INDEXER_BATCH_SIZE = 1000; + /** * @public */ @@ -119,6 +121,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly aliasPostfix: string, private readonly indexPrefix: string, private readonly logger: Logger, + private readonly batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ) { this.elasticSearchClientWrapper = @@ -156,6 +159,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix, indexPrefix, logger, + config.getOptionalNumber('search.elasticsearch.batchSize') ?? + DEFAULT_INDEXER_BATCH_SIZE, config.getOptional( 'search.elasticsearch.highlightOptions', ), @@ -255,6 +260,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { alias, elasticSearchClientWrapper: this.elasticSearchClientWrapper, logger: this.logger, + batchSize: this.batchSize, }); // Attempt cleanup upon failure. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index eff056c3f4..a19687a03b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -44,6 +44,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { alias: 'some-type-index__search', logger: getVoidLogger(), elasticSearchClientWrapper: clientWrapper, + batchSize: 1000, }); // Set up all requisite Elastic mocks. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 87a4472a15..c81950ea41 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -31,6 +31,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { alias: string; logger: Logger; elasticSearchClientWrapper: ElasticSearchClientWrapper; + batchSize: number; }; function duration(startTimestamp: [number, number]): string { @@ -61,7 +62,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private bulkResult: Promise; constructor(options: ElasticSearchSearchEngineIndexerOptions) { - super({ batchSize: 1000 }); + super({ batchSize: options.batchSize }); this.logger = options.logger; this.startTimestamp = process.hrtime(); this.type = options.type; From b0b8213056e8421436334f67c6a82ab82e52c028 Mon Sep 17 00:00:00 2001 From: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com> Date: Sat, 9 Jul 2022 14:53:17 +0200 Subject: [PATCH 2/3] feature: update the search engine documentation to describe the batch size and rename change set file. Signed-off-by: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com> --- ...s-kick.md => search-gentle-cobras-kick.md} | 2 +- docs/features/search/search-engines.md | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) rename .changeset/{gentle-cobras-kick.md => search-gentle-cobras-kick.md} (78%) diff --git a/.changeset/gentle-cobras-kick.md b/.changeset/search-gentle-cobras-kick.md similarity index 78% rename from .changeset/gentle-cobras-kick.md rename to .changeset/search-gentle-cobras-kick.md index 7f971a6ed7..3ddd4df67c 100644 --- a/.changeset/gentle-cobras-kick.md +++ b/.changeset/search-gentle-cobras-kick.md @@ -2,7 +2,7 @@ '@backstage/plugin-search-backend-module-elasticsearch': patch --- -Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000 +Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1000 Example usage: diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index ecd8a85109..5b7b6f4384 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -232,3 +232,23 @@ search: auth: apiKey: base64EncodedKey ``` + +### Elastic search batch size + +Default batch size of the elastic search engine is set to 1000. If you are using a lower spec computing resources (like AWS small instance), +you may get an error caused by limited thread_pool configuration. ( `429 Too Many Requests /_bulk` ) + +In this case you need to decrease the batch size to index the resources to prevent this kind of error. You can easily decrease +or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for elasticsearch configuration. + +#### Configuration example + +**Set batch size to 100** + +```yaml +search: + elasticsearch: + batchSize: 100 +``` + +> You can also increase the batch size if you are using a large ES instance. From 75765ca72c2ca14071db1acb8256b7b406a50649 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 18 Jul 2022 10:56:38 +0200 Subject: [PATCH 3/3] Vale fix Signed-off-by: Eric Peterson --- docs/features/search/search-engines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 5b7b6f4384..515c1268d7 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -236,7 +236,7 @@ search: ### Elastic search batch size Default batch size of the elastic search engine is set to 1000. If you are using a lower spec computing resources (like AWS small instance), -you may get an error caused by limited thread_pool configuration. ( `429 Too Many Requests /_bulk` ) +you may get an error caused by limited `thread_pool` configuration. ( `429 Too Many Requests /_bulk` ) In this case you need to decrease the batch size to index the resources to prevent this kind of error. You can easily decrease or increase the batch size in your `app-config.yaml` using the `batchSize` option provided for elasticsearch configuration.