Enable Normalization on Postgres Query (#28573)

* Enable Normalization on Postgres Query

Signed-off-by: patrick.pulfer <patrick.pulfer@digitecgalaxus.ch>

.changeset/famous-ladybugs-swim.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-pg': patch
+---
+
+Enable normalization in postgres query to change the behavior of the search.

plugins/search-backend-module-pg/README.md

@@ -17,7 +17,7 @@ for details on how to setup Postgres based search for your Backstage instance.
 
 ## Optional Configuration
 
-The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature:
+The following is an example of the optional configuration that can be applied when using Postgres as the search backend.
 
 ```yaml
 search:
@@ -30,6 +30,20 @@ search:
       highlightAll: false # If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false.
       maxFragments: 0 # Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details).
       fragmentDelimiter: ' ... ' # Delimiter string used to concatenate fragments. Defaults to " ... ".
+    normalization: 0 # Ranking functions use an integer bit mask to control document length impact on rank. The default value is 0
+```
+
+The Normalization option controls several behaviors. It is a bit mask. [Ranking Search Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING) has more details.
+
+Example with bit mask specifying more behaviours:
+
+- 2 divides the rank by the document length
+- 4 divides the rank by the mean harmonic distance between extents
+
+```yaml
+search:
+  pg:
+    normalization: 2 | 4
 ```
 
 **Note:** the highlight search term feature uses `ts_headline` which has been known to potentially impact performance. You only need this minimal config to disable it should you have issues:

plugins/search-backend-module-pg/config.d.ts

@@ -57,6 +57,10 @@ export interface Config {
          */
         fragmentDelimiter?: string;
       };
+      /**
+       * Ranking functions use an integer bit mask to control document length impact on rank. The default value is 0
+       */
+      normalization?: number | string;
       /**
        * Batch size to use when indexing
        */

plugins/search-backend-module-pg/report.api.md

@@ -164,6 +164,8 @@ export interface PgSearchQuery {
   // (undocumented)
   limit: number;
   // (undocumented)
+  normalization?: number;
+  // (undocumented)
   offset: number;
   // (undocumented)
   options: PgSearchHighlightOptions;
@@ -182,6 +184,7 @@ export type PgSearchQueryTranslator = (
 // @public
 export type PgSearchQueryTranslatorOptions = {
   highlightOptions: PgSearchHighlightOptions;
+  normalization?: number;
 };
 
 // @public (undocumented)

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts

@@ -86,7 +86,10 @@ describe('PgSearchEngine', () => {
           term: 'testTerm',
           filters: {},
         },
-        { highlightOptions },
+        {
+          highlightOptions,
+          normalization: 0,
+        },
       );
     });
 
@@ -237,6 +240,7 @@ describe('PgSearchEngine', () => {
         pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
         offset: 0,
         limit: 26,
+        normalization: 0,
         options: highlightOptions,
       });
     });
@@ -294,6 +298,7 @@ describe('PgSearchEngine', () => {
         pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
         offset: 0,
         limit: 26,
+        normalization: 0,
         options: highlightOptions,
       });
     });
@@ -354,6 +359,7 @@ describe('PgSearchEngine', () => {
         pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
         offset: 25,
         limit: 26,
+        normalization: 0,
         options: highlightOptions,
       });
     });

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts

@@ -45,6 +45,7 @@ export type ConcretePgSearchQuery = {
  */
 export type PgSearchQueryTranslatorOptions = {
   highlightOptions: PgSearchHighlightOptions;
+  normalization?: number;
 };
 
 /**
@@ -86,6 +87,7 @@ export class PgSearchEngine implements SearchEngine {
   private readonly logger?: LoggerService;
   private readonly highlightOptions: PgSearchHighlightOptions;
   private readonly indexerBatchSize: number;
+  private readonly normalization: number;
 
   /**
    * @deprecated This will be marked as private in a future release, please us fromConfig instead
@@ -116,9 +118,42 @@ export class PgSearchEngine implements SearchEngine {
     this.highlightOptions = highlightOptions;
     this.indexerBatchSize =
       config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000;
+
+    this.normalization = this.getNormalizationValue(config);
     this.logger = logger;
   }
 
+  private getNormalizationValue(config: Config) {
+    const normalizationConfig =
+      config.getOptional('search.pg.normalization') ?? 0;
+    if (typeof normalizationConfig === 'number') {
+      return normalizationConfig;
+    } else if (typeof normalizationConfig === 'string') {
+      return this.evaluateBitwiseOrExpression(normalizationConfig);
+    }
+    this.logger?.error(
+      `Unknown normalization configuration: ${normalizationConfig}`,
+    );
+
+    return 0;
+  }
+
+  private evaluateBitwiseOrExpression(expression: string) {
+    const tokens = expression.split('|').map(token => token.trim());
+
+    const numbers = tokens.map(token => {
+      const num = parseInt(token, 10);
+      if (isNaN(num)) {
+        this.logger?.error(
+          `Unknown expression for normalization: ${expression}`,
+        );
+        return 0;
+      }
+      return num;
+    });
+    return numbers.reduce((acc, num) => acc | num, 0);
+  }
+
   /**
    * @deprecated This will be removed in a future release, please use fromConfig instead
    */
@@ -155,6 +190,7 @@ export class PgSearchEngine implements SearchEngine {
     const offset = page * pageSize;
     // We request more result to know whether there is another page
     const limit = pageSize + 1;
+    const normalization = options.normalization || 0;
 
     return {
       pgQuery: {
@@ -168,6 +204,7 @@ export class PgSearchEngine implements SearchEngine {
         types: query.types,
         offset,
         limit,
+        normalization,
         options: options.highlightOptions,
       },
       pageSize,
@@ -190,6 +227,7 @@ export class PgSearchEngine implements SearchEngine {
   async query(query: SearchQuery): Promise<IndexableResultSet> {
     const { pgQuery, pageSize } = this.translator(query, {
       highlightOptions: this.highlightOptions,
+      normalization: this.normalization,
     });
 
     const rows = await this.databaseStore.transaction(async tx =>

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts

@@ -240,6 +240,7 @@ describe('DatabaseDocumentStore', () => {
             pgTerm: 'Hello & World',
             offset: 1,
             limit: 1,
+            normalization: 0,
             options: highlightOptions,
           }),
         );
@@ -285,6 +286,7 @@ describe('DatabaseDocumentStore', () => {
             pgTerm: 'Hello & World',
             offset: 0,
             limit: 25,
+            normalization: 0,
             options: highlightOptions,
           }),
         );
@@ -346,6 +348,7 @@ describe('DatabaseDocumentStore', () => {
             types: ['my-type'],
             offset: 0,
             limit: 25,
+            normalization: 0,
             options: highlightOptions,
           }),
         );
@@ -399,6 +402,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: 'this' },
             offset: 0,
             limit: 25,
+            normalization: 0,
             options: highlightOptions,
           }),
         );
@@ -465,6 +469,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: ['this', 'that'] },
             offset: 0,
             limit: 25,
+            normalization: 0,
             options: highlightOptions,
           }),
         );
@@ -546,6 +551,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: 'this', otherField: 'another' },
             offset: 0,
             limit: 25,
+            normalization: 0,
             options: highlightOptions,
           }),
         );
@@ -595,6 +601,7 @@ describe('DatabaseDocumentStore', () => {
             fields: { myField: 'this' },
             offset: 0,
             limit: 25,
+            normalization: 0,
             options: highlightOptions,
           }),
         );

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts

@@ -148,10 +148,18 @@ export class DatabaseDocumentStore implements DatabaseStore {
     tx: Knex.Transaction,
     searchQuery: PgSearchQuery,
   ): Promise<DocumentResultRow[]> {
-    const { types, pgTerm, fields, offset, limit, options } = searchQuery;
+    const {
+      types,
+      pgTerm,
+      fields,
+      offset,
+      limit,
+      normalization = 0,
+      options,
+    } = searchQuery;
     // TODO(awanlin): We should make the language a parameter so that we can support more then just english
     // Builds a query like:
-    // SELECT ts_rank_cd(body, query) AS rank, type, document,
+    // SELECT ts_rank_cd(body, query, 0) AS rank, type, document,
     // ts_headline('english', document, query) AS highlight
     // FROM documents, to_tsquery('english', 'consent') query
     // WHERE query @@ body AND (document @> '{"kind": "API"}')
@@ -194,7 +202,7 @@ export class DatabaseDocumentStore implements DatabaseStore {
     if (pgTerm && options.useHighlight) {
       const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`;
       query
-        .select(tx.raw('ts_rank_cd(body, query) AS "rank"'))
+        .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS "rank"`))
         .select(
           tx.raw(
             `ts_headline(\'english\', document, query, '${headlineOptions}') as "highlight"`,
@@ -203,7 +211,7 @@ export class DatabaseDocumentStore implements DatabaseStore {
         .orderBy('rank', 'desc');
     } else if (pgTerm && !options.useHighlight) {
       query
-        .select(tx.raw('ts_rank_cd(body, query) AS "rank"'))
+        .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS "rank"`))
         .orderBy('rank', 'desc');
     } else {
       query.select(tx.raw('1 as rank'));

plugins/search-backend-module-pg/src/database/types.ts

@@ -24,6 +24,7 @@ export interface PgSearchQuery {
   pgTerm?: string;
   offset: number;
   limit: number;
+  normalization?: number;
   options: PgSearchHighlightOptions;
 }