Implement offset and limit based paging in Lunr search engine

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>

plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts

@@ -48,14 +48,12 @@ describe('LunrSearchEngine', () => {
       await testLunrSearchEngine.query({
         term: 'testTerm',
         filters: {},
-        pageCursor: '',
       });
 
       // Then: the translator is invoked with expected args.
       expect(translatorSpy).toHaveBeenCalledWith({
         term: 'testTerm',
         filters: {},
-        pageCursor: '',
       });
     });
 
@@ -68,12 +66,101 @@ describe('LunrSearchEngine', () => {
       const actualTranslatedQuery = translatorUnderTest({
         term: 'testTerm',
         filters: {},
-        pageCursor: '',
+        offset: 0,
+        limit: 50,
       }) as ConcreteLunrQuery;
 
       expect(actualTranslatedQuery).toMatchObject({
         documentTypes: undefined,
         lunrQueryBuilder: expect.any(Function),
+        offset: 0,
+        limit: 50,
+      });
+
+      const query: jest.Mocked<lunr.Query> = {
+        allFields: [],
+        clauses: [],
+        term: jest.fn(),
+        clause: jest.fn(),
+      };
+
+      actualTranslatedQuery.lunrQueryBuilder.bind(query)(query);
+
+      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
+        boost: 100,
+        usePipeline: true,
+      });
+      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
+        boost: 10,
+        usePipeline: false,
+        wildcard: lunr.Query.wildcard.TRAILING,
+      });
+      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
+        boost: 1,
+        usePipeline: false,
+        editDistance: 2,
+      });
+    });
+
+    it('should have default offset and limit', async () => {
+      const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
+        logger: getVoidLogger(),
+      });
+      const translatorUnderTest = inspectableSearchEngine.getTranslator();
+
+      const actualTranslatedQuery = translatorUnderTest({
+        term: 'testTerm',
+      }) as ConcreteLunrQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        documentTypes: undefined,
+        lunrQueryBuilder: expect.any(Function),
+        offset: 0,
+        limit: 25,
+      });
+
+      const query: jest.Mocked<lunr.Query> = {
+        allFields: [],
+        clauses: [],
+        term: jest.fn(),
+        clause: jest.fn(),
+      };
+
+      actualTranslatedQuery.lunrQueryBuilder.bind(query)(query);
+
+      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
+        boost: 100,
+        usePipeline: true,
+      });
+      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
+        boost: 10,
+        usePipeline: false,
+        wildcard: lunr.Query.wildcard.TRAILING,
+      });
+      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
+        boost: 1,
+        usePipeline: false,
+        editDistance: 2,
+      });
+    });
+
+    it('should have maximum limit', async () => {
+      const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
+        logger: getVoidLogger(),
+      });
+      const translatorUnderTest = inspectableSearchEngine.getTranslator();
+
+      const actualTranslatedQuery = translatorUnderTest({
+        term: 'testTerm',
+        offset: 0,
+        limit: 1000,
+      }) as ConcreteLunrQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        documentTypes: undefined,
+        lunrQueryBuilder: expect.any(Function),
+        offset: 0,
+        limit: 100,
       });
 
       const query: jest.Mocked<lunr.Query> = {
@@ -110,7 +197,6 @@ describe('LunrSearchEngine', () => {
       const actualTranslatedQuery = translatorUnderTest({
         term: 'testTerm',
         filters: { kind: 'testKind' },
-        pageCursor: '',
       }) as ConcreteLunrQuery;
 
       expect(actualTranslatedQuery).toMatchObject({
@@ -156,7 +242,6 @@ describe('LunrSearchEngine', () => {
       const actualTranslatedQuery = translatorUnderTest({
         term: 'testTerm',
         filters: { kind: 'testKind', namespace: 'testNameSpace' },
-        pageCursor: '',
       }) as ConcreteLunrQuery;
 
       expect(actualTranslatedQuery).toMatchObject({
@@ -206,7 +291,6 @@ describe('LunrSearchEngine', () => {
       const actualTranslatedQuery = translatorUnderTest({
         term: 'testTerm',
         filters: { kind: 'testKind' },
-        pageCursor: '',
       }) as ConcreteLunrQuery;
 
       expect(actualTranslatedQuery).toMatchObject({
@@ -235,18 +319,16 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTerm',
         filters: {},
-        pageCursor: '',
       });
 
       expect(querySpy).toHaveBeenCalled();
       expect(querySpy).toHaveBeenCalledWith({
         term: 'testTerm',
         filters: {},
-        pageCursor: '',
       });
 
       // Should return 0 results as nothing is indexed here
-      expect(mockedSearchResult).toMatchObject({ results: [] });
+      expect(mockedSearchResult).toMatchObject({ results: [], totalCount: 0 });
     });
 
     it('should perform search query and return 0 results on no match', async () => {
@@ -265,11 +347,10 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'unknown',
         filters: {},
-        pageCursor: '',
       });
 
       // Should return 0 results as we are mocking the indexing of 1 document but with no match on the fields
-      expect(mockedSearchResult).toMatchObject({ results: [] });
+      expect(mockedSearchResult).toMatchObject({ results: [], totalCount: 0 });
     });
 
     it('should perform search query and return all results on empty term', async () => {
@@ -288,7 +369,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: '',
         filters: {},
-        pageCursor: '',
       });
 
       expect(mockedSearchResult).toMatchObject({
@@ -302,6 +382,7 @@ describe('LunrSearchEngine', () => {
             type: 'test-index',
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -321,7 +402,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitle',
         filters: {},
-        pageCursor: '',
       });
 
       expect(mockedSearchResult).toMatchObject({
@@ -334,6 +414,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -353,7 +434,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitle',
         filters: {},
-        pageCursor: '',
       });
 
       expect(mockedSearchResult).toMatchObject({
@@ -366,6 +446,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -385,7 +466,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitel', // Intentional typo
         filters: {},
-        pageCursor: '',
       });
 
       // Should return 1 result as we are mocking the indexing of 1 document with match on the title field
@@ -399,6 +479,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -418,7 +499,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'World',
         filters: {},
-        pageCursor: '',
       });
 
       // Should return 1 result as we are mocking the indexing of 1 document with match on the title field
@@ -432,6 +512,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -451,7 +532,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'Search',
         filters: {},
-        pageCursor: '',
       });
 
       // Should return 1 result as we are mocking the indexing of 1 document with match on the title field
@@ -465,6 +545,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -489,7 +570,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitle',
         filters: { location: 'test/location2' },
-        pageCursor: '',
       });
 
       // Should return 1 of 2 results as we are
@@ -505,6 +585,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -532,7 +613,6 @@ describe('LunrSearchEngine', () => {
       // Perform search query scoped to "test-index-2" with a filter on the field "extraField"
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitle',
-        pageCursor: '',
         filters: { extraField: 'testExtraField' },
       });
 
@@ -547,6 +627,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -571,7 +652,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitle',
         filters: { location: 'test:location2' },
-        pageCursor: '',
       });
 
       // Should return 1 of 2 results as we are
@@ -587,6 +667,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 1,
       });
     });
 
@@ -625,7 +706,6 @@ describe('LunrSearchEngine', () => {
       const mockedSearchResult = await testLunrSearchEngine.query({
         term: 'testTitle',
         types: ['test-index-2'],
-        pageCursor: '',
       });
 
       expect(mockedSearchResult).toMatchObject({
@@ -645,6 +725,7 @@ describe('LunrSearchEngine', () => {
             },
           },
         ],
+        totalCount: 2,
       });
     });
   });

plugins/search-backend-node/src/engines/LunrSearchEngine.ts

@@ -27,6 +27,8 @@ import { Logger } from 'winston';
 export type ConcreteLunrQuery = {
   lunrQueryBuilder: lunr.Index.QueryBuilder;
   documentTypes?: string[];
+  offset: number;
+  limit: number;
 };
 
 type LunrResultEnvelope = {
@@ -50,6 +52,8 @@ export class LunrSearchEngine implements SearchEngine {
     term,
     filters,
     types,
+    offset,
+    limit,
   }: SearchQuery): ConcreteLunrQuery => {
     return {
       lunrQueryBuilder: q => {
@@ -107,6 +111,8 @@ export class LunrSearchEngine implements SearchEngine {
         }
       },
       documentTypes: types,
+      offset: offset ?? 0,
+      limit: Math.min(limit ?? 25, 100),
     };
   };
 
@@ -141,7 +147,7 @@ export class LunrSearchEngine implements SearchEngine {
   }
 
   async query(query: SearchQuery): Promise<SearchResultSet> {
-    const { lunrQueryBuilder, documentTypes } = this.translator(
+    const { lunrQueryBuilder, documentTypes, offset, limit } = this.translator(
       query,
     ) as ConcreteLunrQuery;
 
@@ -179,9 +185,10 @@ export class LunrSearchEngine implements SearchEngine {
 
     // Translate results into SearchResultSet
     const realResultSet: SearchResultSet = {
-      results: results.map(d => {
+      results: results.slice(offset, offset + limit).map(d => {
         return { type: d.type, document: this.docStore[d.result.ref] };
       }),
+      totalCount: results.length,
     };
 
     return realResultSet;