Rework search paging based on cursors

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>
2021-08-19 17:57:11 +02:00
parent a13f21cdc9
commit 532b4cc656
30 changed files with 826 additions and 418 deletions
@@ -17,7 +17,12 @@
 import { getVoidLogger } from '@backstage/backend-common';
 import lunr from 'lunr';
 import { SearchEngine } from '@backstage/search-common';
-import { ConcreteLunrQuery, LunrSearchEngine } from './LunrSearchEngine';
+import {
+  ConcreteLunrQuery,
+  LunrSearchEngine,
+  decodePageCursor,
+  encodePageCursor,
+} from './LunrSearchEngine';

 /**
 * Just used to test the default translator shipped with LunrSearchEngine.
@@ -48,12 +53,14 @@ describe('LunrSearchEngine', () => {
      await testLunrSearchEngine.query({
        term: 'testTerm',
        filters: {},
+        pageCursor: 'MQ==',
      });

      // Then: the translator is invoked with expected args.
      expect(translatorSpy).toHaveBeenCalledWith({
        term: 'testTerm',
        filters: {},
+        pageCursor: 'MQ==',
      });
    });

@@ -66,15 +73,12 @@ describe('LunrSearchEngine', () => {
      const actualTranslatedQuery = translatorUnderTest({
        term: 'testTerm',
        filters: {},
-        offset: 0,
-        limit: 50,
      }) as ConcreteLunrQuery;

      expect(actualTranslatedQuery).toMatchObject({
        documentTypes: undefined,
        lunrQueryBuilder: expect.any(Function),
-        offset: 0,
-        limit: 50,
+        pageSize: 25,
      });

      const query: jest.Mocked<lunr.Query> = {
@@ -115,52 +119,7 @@ describe('LunrSearchEngine', () => {
      expect(actualTranslatedQuery).toMatchObject({
        documentTypes: undefined,
        lunrQueryBuilder: expect.any(Function),
-        offset: 0,
-        limit: 25,
-      });
-
-      const query: jest.Mocked<lunr.Query> = {
-        allFields: [],
-        clauses: [],
-        term: jest.fn(),
-        clause: jest.fn(),
-      };
-
-      actualTranslatedQuery.lunrQueryBuilder.bind(query)(query);
-
-      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
-        boost: 100,
-        usePipeline: true,
-      });
-      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
-        boost: 10,
-        usePipeline: false,
-        wildcard: lunr.Query.wildcard.TRAILING,
-      });
-      expect(query.term).toBeCalledWith(lunr.tokenizer('testTerm'), {
-        boost: 1,
-        usePipeline: false,
-        editDistance: 2,
-      });
-    });
-
-    it('should have maximum limit', async () => {
-      const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({
-        logger: getVoidLogger(),
-      });
-      const translatorUnderTest = inspectableSearchEngine.getTranslator();
-
-      const actualTranslatedQuery = translatorUnderTest({
-        term: 'testTerm',
-        offset: 0,
-        limit: 1000,
-      }) as ConcreteLunrQuery;
-
-      expect(actualTranslatedQuery).toMatchObject({
-        documentTypes: undefined,
-        lunrQueryBuilder: expect.any(Function),
-        offset: 0,
-        limit: 100,
+        pageSize: 25,
      });

      const query: jest.Mocked<lunr.Query> = {
@@ -328,7 +287,10 @@ describe('LunrSearchEngine', () => {
      });

      // Should return 0 results as nothing is indexed here
-      expect(mockedSearchResult).toMatchObject({ results: [], totalCount: 0 });
+      expect(mockedSearchResult).toMatchObject({
+        results: [],
+        nextPageCursor: undefined,
+      });
    });

    it('should perform search query and return 0 results on no match', async () => {
@@ -350,7 +312,10 @@ describe('LunrSearchEngine', () => {
      });

      // Should return 0 results as we are mocking the indexing of 1 document but with no match on the fields
-      expect(mockedSearchResult).toMatchObject({ results: [], totalCount: 0 });
+      expect(mockedSearchResult).toMatchObject({
+        results: [],
+        nextPageCursor: undefined,
+      });
    });

    it('should perform search query and return all results on empty term', async () => {
@@ -382,7 +347,7 @@ describe('LunrSearchEngine', () => {
            type: 'test-index',
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -414,7 +379,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -446,7 +411,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -479,7 +444,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -512,7 +477,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -545,7 +510,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -585,7 +550,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -627,7 +592,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -667,7 +632,7 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 1,
+        nextPageCursor: undefined,
      });
    });

@@ -725,9 +690,75 @@ describe('LunrSearchEngine', () => {
            },
          },
        ],
-        totalCount: 2,
+        nextPageCursor: undefined,
      });
    });
+
+    it('should return next page cursor if results exceed page size', async () => {
+      const mockDocuments = Array(30)
+        .fill(0)
+        .map((_, i) => ({
+          title: 'testTitle',
+          text: 'testText',
+          location: `test/location/${i}`,
+        }));
+
+      await testLunrSearchEngine.index('test-index', mockDocuments);
+
+      const mockedSearchResult = await testLunrSearchEngine.query({
+        term: 'testTitle',
+        types: ['test-index'],
+      });
+
+      expect(mockedSearchResult).toMatchObject({
+        results: Array(25)
+          .fill(0)
+          .map((_, i) => ({
+            document: {
+              title: 'testTitle',
+              text: 'testText',
+              location: `test/location/${i}`,
+            },
+            type: 'test-index',
+          })),
+        nextPageCursor: 'MQ==',
+        previousPageCursor: undefined,
+      });
+    });
+  });
+
+  it('should return previous page cursor if on another page', async () => {
+    const mockDocuments = Array(30)
+      .fill(0)
+      .map((_, i) => ({
+        title: 'testTitle',
+        text: 'testText',
+        location: `test/location/${i}`,
+      }));
+
+    await testLunrSearchEngine.index('test-index', mockDocuments);
+
+    const mockedSearchResult = await testLunrSearchEngine.query({
+      term: 'testTitle',
+      types: ['test-index'],
+      pageCursor: 'MQ==',
+    });
+
+    expect(mockedSearchResult).toMatchObject({
+      results: Array(30)
+        .fill(0)
+        .map((_, i) => ({
+          document: {
+            title: 'testTitle',
+            text: 'testText',
+            location: `test/location/${i}`,
+          },
+          type: 'test-index',
+        }))
+        .slice(25),
+      nextPageCursor: undefined,
+      previousPageCursor: 'MA==',
+    });
  });

  describe('index', () => {
@@ -750,3 +781,19 @@ describe('LunrSearchEngine', () => {
    });
  });
 });
+
+describe('decodePageCursor', () => {
+  test('should decode page', () => {
+    expect(decodePageCursor('MQ==')).toEqual({ page: 1 });
+  });
+
+  test('should fallback to first page if empty', () => {
+    expect(decodePageCursor()).toEqual({ page: 0 });
+  });
+});
+
+describe('encodePageCursor', () => {
+  test('should encode page', () => {
+    expect(encodePageCursor({ page: 1 })).toEqual('MQ==');
+  });
+});
@@ -27,8 +27,7 @@ import { Logger } from 'winston';
 export type ConcreteLunrQuery = {
  lunrQueryBuilder: lunr.Index.QueryBuilder;
  documentTypes?: string[];
-  offset: number;
-  limit: number;
+  pageSize: number;
 };

 type LunrResultEnvelope = {
@@ -52,9 +51,9 @@ export class LunrSearchEngine implements SearchEngine {
    term,
    filters,
    types,
-    offset,
-    limit,
  }: SearchQuery): ConcreteLunrQuery => {
+    const pageSize = 25;
+
    return {
      lunrQueryBuilder: q => {
        const termToken = lunr.tokenizer(term);
@@ -111,8 +110,7 @@ export class LunrSearchEngine implements SearchEngine {
        }
      },
      documentTypes: types,
-      offset: offset ?? 0,
-      limit: Math.min(limit ?? 25, 100),
+      pageSize,
    };
  };

@@ -147,7 +145,7 @@ export class LunrSearchEngine implements SearchEngine {
  }

  async query(query: SearchQuery): Promise<SearchResultSet> {
-    const { lunrQueryBuilder, documentTypes, offset, limit } = this.translator(
+    const { lunrQueryBuilder, documentTypes, pageSize } = this.translator(
      query,
    ) as ConcreteLunrQuery;

@@ -183,14 +181,41 @@ export class LunrSearchEngine implements SearchEngine {
      return doc2.result.score - doc1.result.score;
    });

+    // Perform paging
+    const { page } = decodePageCursor(query.pageCursor);
+    const offset = page * pageSize;
+    const hasPreviousPage = page > 0;
+    const hasNextPage = results.length > offset + pageSize;
+    const nextPageCursor = hasNextPage
+      ? encodePageCursor({ page: page + 1 })
+      : undefined;
+    const previousPageCursor = hasPreviousPage
+      ? encodePageCursor({ page: page - 1 })
+      : undefined;
+
    // Translate results into SearchResultSet
    const realResultSet: SearchResultSet = {
-      results: results.slice(offset, offset + limit).map(d => {
+      results: results.slice(offset, offset + pageSize).map(d => {
        return { type: d.type, document: this.docStore[d.result.ref] };
      }),
-      totalCount: results.length,
+      nextPageCursor,
+      previousPageCursor,
    };

    return realResultSet;
  }
 }
+
+export function decodePageCursor(pageCursor?: string): { page: number } {
+  if (!pageCursor) {
+    return { page: 0 };
+  }
+
+  return {
+    page: Number(Buffer.from(pageCursor, 'base64').toString('utf-8')),
+  };
+}
+
+export function encodePageCursor({ page }: { page: number }): string {
+  return Buffer.from(`${page}`, 'utf-8').toString('base64');
+}