Test fix and vocab addition

Signed-off-by: Jordan Snow <jordans@spotify.com>
This commit is contained in:
Jordan Snow
2025-11-21 10:40:19 -05:00
parent 4d3ddb99b6
commit 9353f6e3f9
3 changed files with 8 additions and 2 deletions
@@ -509,6 +509,7 @@ thumbsup
todo
todos
togglable
tokenizer
tolerations
Tolerations
toolchain
@@ -520,7 +520,7 @@ describe('LunrSearchEngine', () => {
fields: {
title: `${highlightTags.pre}testTitle${highlightTags.post}`,
text: `${highlightTags.pre}testText${highlightTags.post}`,
location: `${highlightTags.pre}test/location${highlightTags.post}`,
location: `${highlightTags.pre}test${highlightTags.post}/location`,
},
},
rank: 1,
@@ -109,13 +109,18 @@ describe('LunrSearchEngineIndexer', () => {
it('should tokenize input on non-alphanumeric characters', () => {
const input =
'Tokenize_test string, entity-name. Unicode support also!三 stjärna عربي';
"Tokenize_test string, entity-name. Doesn't break abc123def - also Unicode support also!三 stjärna عربي";
const expectedTokens = [
'tokenize',
'test',
'string',
'entity',
'name',
'doesn',
't',
'break',
'abc123def',
'also',
'unicode',
'support',
'also',