Refactored to support config with options

Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com>

packages/backend/src/plugins/search.ts

@@ -39,7 +39,10 @@ async function createSearchEngine(
   }
 
   if (await PgSearchEngine.supported(env.database)) {
-    return await PgSearchEngine.from({ database: env.database });
+    return await PgSearchEngine.from({
+      database: env.database,
+      config: env.config,
+    });
   }
 
   return new LunrSearchEngine({ logger: env.logger });

plugins/search-backend-module-pg/config.d.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface Config {
+  /** Configuration options for the search plugin */
+  search?: {
+    /**
+     * Options for PG
+     */
+    pg?: {
+      /**
+       * Options for configuring highlight settings
+       * See https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE
+       */
+      highlightOptions?: {
+        /**
+         * Used to enable to disable the highlight feature. The default value is true
+         */
+        useHighlight?: boolean;
+        /**
+         * Used to set the longest headlines to output. The default value is 35.
+         */
+        maxWords?: number;
+        /**
+         * Used to set the shortest headlines to output. The default value is 15.
+         */
+        minWords?: number;
+        /**
+         * Words of this length or less will be dropped at the start and end of a headline, unless they are query terms.
+         * The default value of three (3) eliminates common English articles.
+         */
+        shortWord?: number;
+        /**
+         * If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false.
+         */
+        highlightAll?: boolean;
+        /**
+         * maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method.
+         * A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details).
+         */
+        maxFragments?: number;
+        /**
+         * Delimiter string used to concatenate fragments. Defaults to " ... ".
+         */
+        fragmentDelimiter?: string;
+      };
+    };
+  };
+}

plugins/search-backend-module-pg/package.json

@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@backstage/backend-common": "^0.14.1-next.1",
+    "@backstage/config": "^1.0.1",
     "@backstage/plugin-search-backend-node": "^0.6.3-next.1",
     "@backstage/plugin-search-common": "^0.3.6-next.0",
     "lodash": "^4.17.21",
@@ -36,6 +37,8 @@
   },
   "files": [
     "dist",
-    "migrations"
-  ]
+    "migrations",
+    "config.d.ts"
+  ],
+  "configSchema": "config.d.ts"
 }

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import { DatabaseStore } from '../database';
+import { PgSearchHighlightConfig } from '../types';
 import {
   ConcretePgSearchQuery,
   decodePageCursor,
@@ -22,6 +23,18 @@ import {
 } from './PgSearchEngine';
 import { PgSearchEngineIndexer } from './PgSearchEngineIndexer';
 
+const highlightOptions: PgSearchHighlightConfig = {
+  preTag: '<tag>',
+  postTag: '</tag>',
+  useHighlight: false,
+  maxWords: 35,
+  minWords: 15,
+  shortWord: 3,
+  highlightAll: false,
+  maxFragments: 0,
+  fragmentDelimiter: ' ... ',
+};
+
 jest.mock('./PgSearchEngineIndexer', () => ({
   PgSearchEngineIndexer: jest
     .fn()
@@ -65,10 +78,13 @@ describe('PgSearchEngine', () => {
     });
 
     it('should pass page cursor', async () => {
-      const actualTranslatedQuery = searchEngine.translator({
-        term: 'Hello',
-        pageCursor: 'MQ==',
-      });
+      const actualTranslatedQuery = searchEngine.translator(
+        {
+          term: 'Hello',
+          pageCursor: 'MQ==',
+        },
+        highlightOptions,
+      );
 
       expect(actualTranslatedQuery).toMatchObject({
         pgQuery: {
@@ -81,9 +97,12 @@ describe('PgSearchEngine', () => {
     });
 
     it('should return translated query term', async () => {
-      const actualTranslatedQuery = searchEngine.translator({
-        term: 'Hello World',
-      });
+      const actualTranslatedQuery = searchEngine.translator(
+        {
+          term: 'Hello World',
+        },
+        highlightOptions,
+      );
 
       expect(actualTranslatedQuery).toMatchObject({
         pgQuery: {
@@ -96,10 +115,13 @@ describe('PgSearchEngine', () => {
     });
 
     it('should sanitize query term', async () => {
-      const actualTranslatedQuery = searchEngine.translator({
-        term: 'H&e|l!l*o W\0o(r)l:d',
-        pageCursor: '',
-      }) as ConcretePgSearchQuery;
+      const actualTranslatedQuery = searchEngine.translator(
+        {
+          term: 'H&e|l!l*o W\0o(r)l:d',
+          pageCursor: '',
+        },
+        highlightOptions,
+      ) as ConcretePgSearchQuery;
 
       expect(actualTranslatedQuery).toMatchObject({
         pgQuery: {
@@ -110,11 +132,14 @@ describe('PgSearchEngine', () => {
     });
 
     it('should return translated query with filters', async () => {
-      const actualTranslatedQuery = searchEngine.translator({
-        term: 'testTerm',
-        filters: { kind: 'testKind' },
-        types: ['my-filter'],
-      });
+      const actualTranslatedQuery = searchEngine.translator(
+        {
+          term: 'testTerm',
+          filters: { kind: 'testKind' },
+          types: ['my-filter'],
+        },
+        highlightOptions,
+      );
 
       expect(actualTranslatedQuery).toMatchObject({
         pgQuery: {

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts

@@ -27,6 +27,8 @@ import {
   PgSearchQuery,
 } from '../database';
 import { v4 as uuid } from 'uuid';
+import { Config } from '@backstage/config';
+import { PgSearchHighlightConfig, PgSearchHighlightOptions } from '../types';
 
 export type ConcretePgSearchQuery = {
   pgQuery: PgSearchQuery;
@@ -34,13 +36,36 @@ export type ConcretePgSearchQuery = {
 };
 
 export class PgSearchEngine implements SearchEngine {
-  constructor(private readonly databaseStore: DatabaseStore) {}
+  private readonly highlightOptions: PgSearchHighlightConfig;
+
+  constructor(
+    private readonly databaseStore: DatabaseStore,
+    highlightOptions?: PgSearchHighlightOptions,
+  ) {
+    const uuidTag = uuid();
+    this.highlightOptions = {
+      preTag: `<${uuidTag}>`,
+      postTag: `</${uuidTag}>`,
+      useHighlight: false,
+      maxWords: 35,
+      minWords: 15,
+      shortWord: 3,
+      highlightAll: false,
+      maxFragments: 0,
+      fragmentDelimiter: ' ... ',
+      ...highlightOptions,
+    };
+  }
 
   static async from(options: {
     database: PluginDatabaseManager;
+    config: Config;
   }): Promise<PgSearchEngine> {
     return new PgSearchEngine(
       await DatabaseDocumentStore.create(await options.database.getClient()),
+      options.config.getOptional<PgSearchHighlightOptions>(
+        'search.pg.highlightOptions',
+      ),
     );
   }
 
@@ -48,13 +73,15 @@ export class PgSearchEngine implements SearchEngine {
     return await DatabaseDocumentStore.supported(await database.getClient());
   }
 
-  translator(query: SearchQuery): ConcretePgSearchQuery {
+  translator(
+    query: SearchQuery,
+    options: PgSearchHighlightConfig,
+  ): ConcretePgSearchQuery {
     const pageSize = 25;
     const { page } = decodePageCursor(query.pageCursor);
     const offset = page * pageSize;
     // We request more result to know whether there is another page
     const limit = pageSize + 1;
-    const uuidTag = uuid();
 
     return {
       pgQuery: {
@@ -68,8 +95,7 @@ export class PgSearchEngine implements SearchEngine {
         types: query.types,
         offset,
         limit,
-        preTag: `<${uuidTag}>`,
-        postTag: `</${uuidTag}>`,
+        options,
       },
       pageSize,
     };
@@ -90,7 +116,7 @@ export class PgSearchEngine implements SearchEngine {
   }
 
   async query(query: SearchQuery): Promise<IndexableResultSet> {
-    const { pgQuery, pageSize } = this.translator(query);
+    const { pgQuery, pageSize } = this.translator(query, this.highlightOptions);
 
     const rows = await this.databaseStore.transaction(async tx =>
       this.databaseStore.query(tx, pgQuery),
@@ -115,8 +141,8 @@ export class PgSearchEngine implements SearchEngine {
         document,
         rank: page * pageSize + index + 1,
         highlight: {
-          preTag: pgQuery.preTag,
-          postTag: pgQuery.postTag,
+          preTag: pgQuery.options.preTag,
+          postTag: pgQuery.options.postTag,
           fields: highlight
             ? {
                 text: highlight.text,

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts

@@ -15,8 +15,21 @@
  */
 import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
 import { IndexableDocument } from '@backstage/plugin-search-common';
+import { PgSearchHighlightConfig } from '../types';
 import { DatabaseDocumentStore } from './DatabaseDocumentStore';
 
+const highlightOptions: PgSearchHighlightConfig = {
+  preTag: '<tag>',
+  postTag: '</tag>',
+  useHighlight: false,
+  maxWords: 35,
+  minWords: 15,
+  shortWord: 3,
+  highlightAll: false,
+  maxFragments: 0,
+  fragmentDelimiter: ' ... ',
+};
+
 describe('DatabaseDocumentStore', () => {
   describe('unsupported', () => {
     const databases = TestDatabases.create({
@@ -224,8 +237,7 @@ describe('DatabaseDocumentStore', () => {
             pgTerm: 'Hello & World',
             offset: 1,
             limit: 1,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 
@@ -271,8 +283,7 @@ describe('DatabaseDocumentStore', () => {
             pgTerm: 'Hello & World',
             offset: 0,
             limit: 25,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 
@@ -334,8 +345,7 @@ describe('DatabaseDocumentStore', () => {
             types: ['my-type'],
             offset: 0,
             limit: 25,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 
@@ -389,8 +399,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: 'this' },
             offset: 0,
             limit: 25,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 
@@ -445,8 +454,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: ['this', 'that'] },
             offset: 0,
             limit: 25,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 
@@ -508,8 +516,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: 'this', otherField: 'another' },
             offset: 0,
             limit: 25,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 
@@ -559,8 +566,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: 'this' },
             offset: 0,
             limit: 25,
-            preTag: '<tag>',
-            postTag: '</tag>',
+            options: highlightOptions,
           }),
         );
 

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts

@@ -132,7 +132,7 @@ export class DatabaseDocumentStore implements DatabaseStore {
 
   async query(
     tx: Knex.Transaction,
-    { types, pgTerm, fields, offset, limit, preTag, postTag }: PgSearchQuery,
+    { types, pgTerm, fields, offset, limit, options }: PgSearchQuery,
   ): Promise<DocumentResultRow[]> {
     // Builds a query like:
     // SELECT ts_rank_cd(body, query) AS rank, type, document,
@@ -171,15 +171,20 @@ export class DatabaseDocumentStore implements DatabaseStore {
 
     query.select('type', 'document');
 
-    if (pgTerm) {
+    if (pgTerm && options.useHighlight) {
+      const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`;
       query
         .select(tx.raw('ts_rank_cd(body, query) AS "rank"'))
         .select(
           tx.raw(
-            `ts_headline(\'english\', document, query, 'StartSel=${preTag}, StopSel=${postTag}') as "highlight"`,
+            `ts_headline(\'english\', document, query, '${headlineOptions}') as "highlight"`,
           ),
         )
         .orderBy('rank', 'desc');
+    } else if (pgTerm && !options.useHighlight) {
+      query
+        .select(tx.raw('ts_rank_cd(body, query) AS "rank"'))
+        .orderBy('rank', 'desc');
     } else {
       query.select(tx.raw('1 as rank'));
     }

plugins/search-backend-module-pg/src/database/types.ts

@@ -15,6 +15,7 @@
  */
 import { IndexableDocument } from '@backstage/plugin-search-common';
 import { Knex } from 'knex';
+import { PgSearchHighlightConfig } from '../types';
 
 export interface PgSearchQuery {
   fields?: Record<string, string | string[]>;
@@ -22,8 +23,7 @@ export interface PgSearchQuery {
   pgTerm?: string;
   offset: number;
   limit: number;
-  preTag: string;
-  postTag: string;
+  options: PgSearchHighlightConfig;
 }
 
 export interface DatabaseStore {

plugins/search-backend-module-pg/src/types.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type PgSearchHighlightOptions = {
+  useHighlight?: boolean;
+  maxWords?: number;
+  minWords?: number;
+  shortWord?: number;
+  highlightAll?: boolean;
+  maxFragments?: number;
+  fragmentDelimiter?: string;
+};
+
+/**
+ * @public
+ */
+export type PgSearchHighlightConfig = {
+  useHighlight: boolean;
+  maxWords: number;
+  minWords: number;
+  shortWord: number;
+  highlightAll: boolean;
+  maxFragments: number;
+  fragmentDelimiter: string;
+  preTag: string;
+  postTag: string;
+};