From ab338422e1ff2127db19f67199f114c93e35f1ed Mon Sep 17 00:00:00 2001 From: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com> Date: Fri, 8 Jul 2022 12:39:22 +0200 Subject: [PATCH] Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000 Signed-off-by: Hasan Oezdemir <21654050+nodify-at@users.noreply.github.com> --- .changeset/gentle-cobras-kick.md | 13 +++++++++++++ .../api-report.md | 2 ++ .../search-backend-module-elasticsearch/config.d.ts | 4 ++++ .../src/engines/ElasticSearchSearchEngine.test.ts | 2 ++ .../src/engines/ElasticSearchSearchEngine.ts | 6 ++++++ .../ElasticSearchSearchEngineIndexer.test.ts | 1 + .../src/engines/ElasticSearchSearchEngineIndexer.ts | 3 ++- 7 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .changeset/gentle-cobras-kick.md diff --git a/.changeset/gentle-cobras-kick.md b/.changeset/gentle-cobras-kick.md new file mode 100644 index 0000000000..7f971a6ed7 --- /dev/null +++ b/.changeset/gentle-cobras-kick.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Feature: add a new option to set the batch size for elastic search engine, if not given the default batch size is 1.000 + +Example usage: + +```yaml +search: + elasticsearch: + batchSize: 100 +``` diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index eed2b1b10a..6ed6705b0a 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -315,6 +315,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix: string, indexPrefix: string, logger: Logger, + batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ); // (undocumented) @@ -361,6 +362,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { alias: string; logger: Logger; elasticSearchClientWrapper: ElasticSearchClientWrapper; + batchSize: number; }; // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 8f63e66365..95e527b236 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -21,6 +21,10 @@ export interface Config { * Options for ElasticSearch */ elasticsearch?: { + /** + * Batch size for elastic search indexing tasks. Defaults to 1000. + */ + batchSize?: number; /** * Options for configuring highlight settings * See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/highlighting.html diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index b59c67c69c..64725e446d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -79,12 +79,14 @@ describe('ElasticSearchSearchEngine', () => { 'search', '', getVoidLogger(), + 1000, ); inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests( options, 'search', '', getVoidLogger(), + 1000, ); // eslint-disable-next-line dot-notation clientWrapper = testSearchEngine['elasticSearchClientWrapper']; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 42c99dfe61..c4c9ec4d37 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -107,6 +107,8 @@ function isBlank(str: string) { return (isEmpty(str) && !isNumber(str)) || nan(str); } +const DEFAULT_INDEXER_BATCH_SIZE = 1000; + /** * @public */ @@ -119,6 +121,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly aliasPostfix: string, private readonly indexPrefix: string, private readonly logger: Logger, + private readonly batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ) { this.elasticSearchClientWrapper = @@ -156,6 +159,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { aliasPostfix, indexPrefix, logger, + config.getOptionalNumber('search.elasticsearch.batchSize') ?? + DEFAULT_INDEXER_BATCH_SIZE, config.getOptional( 'search.elasticsearch.highlightOptions', ), @@ -255,6 +260,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { alias, elasticSearchClientWrapper: this.elasticSearchClientWrapper, logger: this.logger, + batchSize: this.batchSize, }); // Attempt cleanup upon failure. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index eff056c3f4..a19687a03b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -44,6 +44,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { alias: 'some-type-index__search', logger: getVoidLogger(), elasticSearchClientWrapper: clientWrapper, + batchSize: 1000, }); // Set up all requisite Elastic mocks. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 87a4472a15..c81950ea41 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -31,6 +31,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { alias: string; logger: Logger; elasticSearchClientWrapper: ElasticSearchClientWrapper; + batchSize: number; }; function duration(startTimestamp: [number, number]): string { @@ -61,7 +62,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private bulkResult: Promise; constructor(options: ElasticSearchSearchEngineIndexerOptions) { - super({ batchSize: 1000 }); + super({ batchSize: options.batchSize }); this.logger = options.logger; this.startTimestamp = process.hrtime(); this.type = options.type;