Merge pull request #10981 from kuangp/feat/search/highlight

feat(search): highlight search result terms

plugins/search-backend-node/api-report.md

@@ -82,6 +82,10 @@ export class LunrSearchEngine implements SearchEngine {
   // (undocumented)
   getIndexer(type: string): Promise<LunrSearchEngineIndexer>;
   // (undocumented)
+  protected highlightPostTag: string;
+  // (undocumented)
+  protected highlightPreTag: string;
+  // (undocumented)
   protected logger: Logger;
   // (undocumented)
   protected lunrIndices: Record<string, lunr_2.Index>;

plugins/search-backend-node/package.json

@@ -30,6 +30,7 @@
     "lodash": "^4.17.21",
     "lunr": "^2.3.9",
     "node-abort-controller": "^3.0.1",
+    "uuid": "^8.3.2",
     "winston": "^3.2.1"
   },
   "devDependencies": {

plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts

@@ -25,6 +25,7 @@ import {
   LunrSearchEngine,
   decodePageCursor,
   encodePageCursor,
+  parseHighlightFields,
 } from './LunrSearchEngine';
 import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';
 import { TestPipeline } from '../test-utils';
@@ -33,6 +34,12 @@ import { TestPipeline } from '../test-utils';
  * Just used to test the default translator shipped with LunrSearchEngine.
  */
 class LunrSearchEngineForTests extends LunrSearchEngine {
+  getHighlightTags() {
+    return {
+      pre: this.highlightPreTag,
+      post: this.highlightPostTag,
+    };
+  }
   getDocStore() {
     return this.docStore;
   }
@@ -468,6 +475,58 @@ describe('LunrSearchEngine', () => {
       });
     });
 
+    it('should perform search query and return highlight metadata on match', async () => {
+      const inspectableSearchEngine = new LunrSearchEngineForTests({
+        logger: getVoidLogger(),
+      });
+
+      const mockDocuments = [
+        {
+          title: 'testTitle',
+          text: 'testText',
+          location: 'test/location',
+        },
+      ];
+
+      // Mock indexing of 1 document
+      const indexer = await getActualIndexer(
+        inspectableSearchEngine,
+        'test-index',
+      );
+      await TestPipeline.withSubject(indexer)
+        .withDocuments(mockDocuments)
+        .execute();
+
+      // Perform search query
+      const mockedSearchResult = await inspectableSearchEngine.query({
+        term: 'test',
+        filters: {},
+      });
+
+      const highlightTags = inspectableSearchEngine.getHighlightTags();
+      expect(mockedSearchResult).toMatchObject({
+        results: [
+          {
+            document: {
+              title: 'testTitle',
+              text: 'testText',
+              location: 'test/location',
+            },
+            highlight: {
+              preTag: highlightTags.pre,
+              postTag: highlightTags.post,
+              fields: {
+                title: `${highlightTags.pre}testTitle${highlightTags.post}`,
+                text: `${highlightTags.pre}testText${highlightTags.post}`,
+                location: `${highlightTags.pre}test/location${highlightTags.post}`,
+              },
+            },
+          },
+        ],
+        nextPageCursor: undefined,
+      });
+    });
+
     it('should perform search query and return search results on partial match', async () => {
       const mockDocuments = [
         {
@@ -976,3 +1035,33 @@ describe('encodePageCursor', () => {
     expect(encodePageCursor({ page: 1 })).toEqual('MQ==');
   });
 });
+
+describe('parseHighlightFields', () => {
+  it('should parse highlight metadata', () => {
+    expect(
+      parseHighlightFields({
+        preTag: '<>',
+        postTag: '</>',
+        doc: { foo: 'abc def', bar: 'ghi jkl' },
+        positionMetadata: {
+          test: {
+            foo: {
+              position: [[0, 3]],
+            },
+          },
+          anotherTest: {
+            foo: {
+              position: [[4, 3]],
+            },
+            bar: {
+              position: [[4, 3]],
+            },
+          },
+        },
+      }),
+    ).toEqual({
+      foo: '<>abc</> <>def</>',
+      bar: 'ghi <>jkl</>',
+    });
+  });
+});

plugins/search-backend-node/src/engines/LunrSearchEngine.ts

@@ -22,6 +22,7 @@ import {
   SearchEngine,
 } from '@backstage/plugin-search-common';
 import lunr from 'lunr';
+import { v4 as uuid } from 'uuid';
 import { Logger } from 'winston';
 import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer';
 
@@ -51,10 +52,15 @@ export class LunrSearchEngine implements SearchEngine {
   protected lunrIndices: Record<string, lunr.Index> = {};
   protected docStore: Record<string, IndexableDocument>;
   protected logger: Logger;
+  protected highlightPreTag: string;
+  protected highlightPostTag: string;
 
   constructor({ logger }: { logger: Logger }) {
     this.logger = logger;
     this.docStore = {};
+    const uuidTag = uuid();
+    this.highlightPreTag = `<${uuidTag}>`;
+    this.highlightPostTag = `</${uuidTag}>`;
   }
 
   protected translator: QueryTranslator = ({
@@ -198,9 +204,20 @@ export class LunrSearchEngine implements SearchEngine {
 
     // Translate results into IndexableResultSet
     const realResultSet: IndexableResultSet = {
-      results: results.slice(offset, offset + pageSize).map(d => {
-        return { type: d.type, document: this.docStore[d.result.ref] };
-      }),
+      results: results.slice(offset, offset + pageSize).map(d => ({
+        type: d.type,
+        document: this.docStore[d.result.ref],
+        highlight: {
+          preTag: this.highlightPreTag,
+          postTag: this.highlightPostTag,
+          fields: parseHighlightFields({
+            preTag: this.highlightPreTag,
+            postTag: this.highlightPostTag,
+            doc: this.docStore[d.result.ref],
+            positionMetadata: d.result.matchData.metadata as any,
+          }),
+        },
+      })),
       nextPageCursor,
       previousPageCursor,
     };
@@ -222,3 +239,52 @@ export function decodePageCursor(pageCursor?: string): { page: number } {
 export function encodePageCursor({ page }: { page: number }): string {
   return Buffer.from(`${page}`, 'utf-8').toString('base64');
 }
+
+type ParseHighlightFieldsProps = {
+  preTag: string;
+  postTag: string;
+  doc: any;
+  positionMetadata: {
+    [term: string]: {
+      [field: string]: {
+        position: number[][];
+      };
+    };
+  };
+};
+
+export function parseHighlightFields({
+  preTag,
+  postTag,
+  doc,
+  positionMetadata,
+}: ParseHighlightFieldsProps): { [field: string]: string } {
+  // Merge the field positions across all query terms
+  const highlightFieldPositions = Object.values(positionMetadata).reduce(
+    (fieldPositions, metadata) => {
+      Object.keys(metadata).map(fieldKey => {
+        fieldPositions[fieldKey] = fieldPositions[fieldKey] ?? [];
+        fieldPositions[fieldKey].push(...metadata[fieldKey].position);
+      });
+
+      return fieldPositions;
+    },
+    {} as { [field: string]: number[][] },
+  );
+
+  return Object.fromEntries(
+    Object.entries(highlightFieldPositions).map(([field, positions]) => {
+      positions.sort((a, b) => b[0] - a[0]);
+
+      const highlightedField = positions.reduce((content, pos) => {
+        return (
+          `${content.substring(0, pos[0])}${preTag}` +
+          `${content.substring(pos[0], pos[0] + pos[1])}` +
+          `${postTag}${content.substring(pos[0] + pos[1])}`
+        );
+      }, doc[field]);
+
+      return [field, highlightedField];
+    }),
+  );
+}

plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts

@@ -32,6 +32,7 @@ export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer {
     this.builder = new lunr.Builder();
     this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer);
     this.builder.searchPipeline.add(lunr.stemmer);
+    this.builder.metadataWhitelist = ['position'];
   }
 
   // No async initialization required.