From 5e14cea98b9e90ad30827278b971719f49d1d3ed Mon Sep 17 00:00:00 2001
From: Seow Wen Han <106362727+swenhan@users.noreply.github.com>
Date: Tue, 23 Aug 2022 13:36:57 +0800
Subject: [PATCH] Fix in-doc search not working when stopword is in title (#1)

* Trying to reproduce stop-word error but couldn't

* Update and add appropriate test case for stop-word testing

* Implement fix for stopword in title, add unit test case

Signed-off-by: Seow Wen Han <wseow@seekasia.com>
---
 .../src/engines/LunrSearchEngine.test.ts      | 120 ++++++++++++++++++
 .../src/engines/LunrSearchEngine.ts           |  14 +-
 2 files changed, 130 insertions(+), 4 deletions(-)

diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts
index 1f69460448..521cb9ee81 100644
--- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts
+++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts
@@ -1092,3 +1092,123 @@ describe('parseHighlightFields', () => {
     });
   });
 });
+
+describe('stopword testing', () => {
+  let testLunrSearchEngine: SearchEngine;
+
+  beforeEach(() => {
+    testLunrSearchEngine = new LunrSearchEngine({ logger: getVoidLogger() });
+    jest.clearAllMocks();
+  });
+
+  it('test with stopword in title', async () => {
+    const mockDocuments = [
+      {
+        title: 'a home',
+        text: 'mary had a little lamb at home',
+        location: 'test/location',
+      },
+    ];
+
+    const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index');
+
+    await TestPipeline.withSubject(indexer)
+      .withDocuments(mockDocuments)
+      .execute();
+
+    const mockedSearchResult = await testLunrSearchEngine.query({
+      term: 'mary',
+      filters: {
+        title: 'a home',
+      },
+    });
+
+    expect(mockedSearchResult).toMatchObject({
+      results: [
+        {
+          document: {
+            title: 'a home',
+            text: 'mary had a little lamb at home',
+            location: 'test/location',
+          },
+          rank: 1,
+        },
+      ],
+      nextPageCursor: undefined,
+    });
+  });
+
+  it('test with stopword connected with dash', async () => {
+    const mockDocuments = [
+      {
+        title: 'a-home',
+        text: 'mary had a little lamb at home',
+        location: 'test/location',
+      },
+    ];
+
+    const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index');
+
+    await TestPipeline.withSubject(indexer)
+      .withDocuments(mockDocuments)
+      .execute();
+
+    const mockedSearchResult = await testLunrSearchEngine.query({
+      term: 'mary',
+      filters: {
+        title: 'a-home',
+      },
+    });
+
+    expect(mockedSearchResult).toMatchObject({
+      results: [
+        {
+          document: {
+            title: 'a-home',
+            text: 'mary had a little lamb at home',
+            location: 'test/location',
+          },
+          rank: 1,
+        },
+      ],
+      nextPageCursor: undefined,
+    });
+  });
+
+  it('test with only stop word in title', async () => {
+    const mockDocuments = [
+      {
+        title: 'a',
+        text: 'mary had a little lamb at home',
+        location: 'test/location',
+      },
+    ];
+
+    const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index');
+
+    await TestPipeline.withSubject(indexer)
+      .withDocuments(mockDocuments)
+      .execute();
+
+    const mockedSearchResult = await testLunrSearchEngine.query({
+      term: 'mary',
+      filters: {
+        title: 'a',
+      },
+    });
+
+    expect(mockedSearchResult).toMatchObject({
+      results: [
+        {
+          document: {
+            title: 'a',
+            text: 'mary had a little lamb at home',
+            location: 'test/location',
+          },
+          rank: 1,
+        },
+      ],
+      nextPageCursor: undefined,
+    });
+  });
+});
diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts
index ec8550d721..8780490dcc 100644
--- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts
+++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts
@@ -114,10 +114,16 @@ export class LunrSearchEngine implements SearchEngine {
 
             // Require that the given field has the given value
             if (['string', 'number', 'boolean'].includes(typeof value)) {
-              q.term(lunr.tokenizer(value?.toString()), {
-                presence: lunr.Query.presence.REQUIRED,
-                fields: [field],
-              });
+              q.term(
+                lunr
+                  .tokenizer(value?.toString())
+                  .map(lunr.stopWordFilter)
+                  .filter(element => element !== undefined),
+                {
+                  presence: lunr.Query.presence.REQUIRED,
+                  fields: [field],
+                },
+              );
             } else if (Array.isArray(value)) {
               // Illustrate how multi-value filters could work.
               // But warn that Lurn supports this poorly.