Merge pull request #31836 from SnowBlitzer/jordans/tokenizer

Changing tokenizer separator for search indexing

plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts

@@ -520,7 +520,7 @@ describe('LunrSearchEngine', () => {
               fields: {
                 title: `${highlightTags.pre}testTitle${highlightTags.post}`,
                 text: `${highlightTags.pre}testText${highlightTags.post}`,
-                location: `${highlightTags.pre}test/location${highlightTags.post}`,
+                location: `${highlightTags.pre}test${highlightTags.post}/location`,
               },
             },
             rank: 1,

plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts

@@ -106,4 +106,31 @@ describe('LunrSearchEngineIndexer', () => {
       ...[lunr.trimmer, lunr.stopWordFilter, lunr.stemmer],
     );
   });
+
+  it('should tokenize input on non-alphanumeric characters', () => {
+    const input =
+      "Tokenize_test string, entity-name. Doesn't break abc123def - also Unicode support also!三 stjärna عربي";
+    const expectedTokens = [
+      'tokenize',
+      'test',
+      'string',
+      'entity',
+      'name',
+      'doesn',
+      't',
+      'break',
+      'abc123def',
+      'also',
+      'unicode',
+      'support',
+      'also',
+      '三',
+      'stjärna',
+      'عربي',
+    ];
+
+    const tokens = lunr.tokenizer(input).map(token => token.toString());
+
+    expect(tokens).toEqual(expectedTokens);
+  });
 });

plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts

@@ -29,8 +29,8 @@ export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
 
   constructor() {
     super({ batchSize: 1000 });
-
     this.builder = new lunr.Builder();
+    this.builder.tokenizer.separator = /[^\p{L}\p{N}]+/u;
     this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);
     this.builder.searchPipeline.add(lunr.stemmer);
     this.builder.metadataWhitelist = ['position'];