From 71d34327103b5c4f1ef4bc57e23a3e7e0b133ed3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 29 Apr 2022 17:26:02 +0200 Subject: [PATCH] Set the default batchSize to 1000 for all search engines. Signed-off-by: Eric Peterson --- .changeset/search-the-worst-of-you.md | 7 +++++++ .../src/engines/ElasticSearchSearchEngineIndexer.ts | 2 +- .../src/PgSearchEngine/PgSearchEngine.test.ts | 2 +- .../src/PgSearchEngine/PgSearchEngine.ts | 2 +- .../src/engines/LunrSearchEngineIndexer.ts | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 .changeset/search-the-worst-of-you.md diff --git a/.changeset/search-the-worst-of-you.md b/.changeset/search-the-worst-of-you.md new file mode 100644 index 0000000000..29d9661f24 --- /dev/null +++ b/.changeset/search-the-worst-of-you.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-node': patch +'@backstage/plugin-search-backend-module-elasticsearch': patch +'@backstage/plugin-search-backend-module-pg': patch +--- + +Search Engines will now index documents in batches of 1000 instead of 100 (under the hood). This may result in your Backstage backend consuming slightly more memory during index runs, but should dramatically improve indexing performance for large document sets. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index bf07b8a36d..2a738e1506 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -53,7 +53,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private bulkResult: Promise; constructor(options: ElasticSearchSearchEngineIndexerOptions) { - super({ batchSize: 100 }); + super({ batchSize: 1000 }); this.logger = options.logger; this.startTimestamp = process.hrtime(); this.type = options.type; diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 8618bed4c8..cc89f28c80 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -136,7 +136,7 @@ describe('PgSearchEngine', () => { // Indexer instantiated with expected args. expect(PgSearchEngineIndexer).toHaveBeenCalledWith( expect.objectContaining({ - batchSize: 100, + batchSize: 1000, type: 'my-type', databaseStore: database, }), diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index ea363d8ea1..445d1bb5c1 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -78,7 +78,7 @@ export class PgSearchEngine implements SearchEngine { async getIndexer(type: string) { return new PgSearchEngineIndexer({ - batchSize: 100, + batchSize: 1000, type, databaseStore: this.databaseStore, }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts index 49afafea64..a798f7c81f 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts @@ -27,7 +27,7 @@ export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { private docStore: Record = {}; constructor() { - super({ batchSize: 100 }); + super({ batchSize: 1000 }); this.builder = new lunr.Builder(); this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);