From 4d3ddb99b6dee1335be91f940bea14cb4bbfc18d Mon Sep 17 00:00:00 2001 From: Jordan Snow Date: Thu, 20 Nov 2025 16:23:32 -0500 Subject: [PATCH 1/2] Changing tokenizer seperator for seach indexing Signed-off-by: Jordan Snow --- .changeset/chilly-hotels-walk.md | 5 +++++ .../engines/LunrSearchEngineIndexer.test.ts | 22 +++++++++++++++++++ .../src/engines/LunrSearchEngineIndexer.ts | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 .changeset/chilly-hotels-walk.md diff --git a/.changeset/chilly-hotels-walk.md b/.changeset/chilly-hotels-walk.md new file mode 100644 index 0000000000..e9b1ff66ba --- /dev/null +++ b/.changeset/chilly-hotels-walk.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-node': minor +--- + +Improving method that search tokenizer breaks apart entity names diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts index ff76f0fa7c..4eb7ef9227 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts @@ -106,4 +106,26 @@ describe('LunrSearchEngineIndexer', () => { ...[lunr.trimmer, lunr.stopWordFilter, lunr.stemmer], ); }); + + it('should tokenize input on non-alphanumeric characters', () => { + const input = + 'Tokenize_test string, entity-name. Unicode support also!三 stjärna عربي'; + const expectedTokens = [ + 'tokenize', + 'test', + 'string', + 'entity', + 'name', + 'unicode', + 'support', + 'also', + '三', + 'stjärna', + 'عربي', + ]; + + const tokens = lunr.tokenizer(input).map(token => token.toString()); + + expect(tokens).toEqual(expectedTokens); + }); }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts index e723957077..ac7f7b1c0d 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts @@ -29,8 +29,8 @@ export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { constructor() { super({ batchSize: 1000 }); - this.builder = new lunr.Builder(); + this.builder.tokenizer.separator = /[^\p{L}\p{N}]+/u; this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); this.builder.searchPipeline.add(lunr.stemmer); this.builder.metadataWhitelist = ['position']; From 9353f6e3f9a7cb0a20d6d1a0c916796afac05def Mon Sep 17 00:00:00 2001 From: Jordan Snow Date: Fri, 21 Nov 2025 10:40:19 -0500 Subject: [PATCH 2/2] Test fix and vocab addition Signed-off-by: Jordan Snow --- .github/vale/config/vocabularies/Backstage/accept.txt | 1 + .../src/engines/LunrSearchEngine.test.ts | 2 +- .../src/engines/LunrSearchEngineIndexer.test.ts | 7 ++++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/vale/config/vocabularies/Backstage/accept.txt b/.github/vale/config/vocabularies/Backstage/accept.txt index 7c780c1079..6697fad858 100644 --- a/.github/vale/config/vocabularies/Backstage/accept.txt +++ b/.github/vale/config/vocabularies/Backstage/accept.txt @@ -509,6 +509,7 @@ thumbsup todo todos togglable +tokenizer tolerations Tolerations toolchain diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index 407fa779ab..acd18771c8 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -520,7 +520,7 @@ describe('LunrSearchEngine', () => { fields: { title: `${highlightTags.pre}testTitle${highlightTags.post}`, text: `${highlightTags.pre}testText${highlightTags.post}`, - location: `${highlightTags.pre}test/location${highlightTags.post}`, + location: `${highlightTags.pre}test${highlightTags.post}/location`, }, }, rank: 1, diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts index 4eb7ef9227..46f0317d33 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts @@ -109,13 +109,18 @@ describe('LunrSearchEngineIndexer', () => { it('should tokenize input on non-alphanumeric characters', () => { const input = - 'Tokenize_test string, entity-name. Unicode support also!三 stjärna عربي'; + "Tokenize_test string, entity-name. Doesn't break abc123def - also Unicode support also!三 stjärna عربي"; const expectedTokens = [ 'tokenize', 'test', 'string', 'entity', 'name', + 'doesn', + 't', + 'break', + 'abc123def', + 'also', 'unicode', 'support', 'also',