diff --git a/.changeset/famous-ladybugs-swim.md b/.changeset/famous-ladybugs-swim.md new file mode 100644 index 0000000000..67341fd906 --- /dev/null +++ b/.changeset/famous-ladybugs-swim.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Enable normalization in postgres query to change the behavior of the search. diff --git a/plugins/search-backend-module-pg/README.md b/plugins/search-backend-module-pg/README.md index 49804a927c..cce66e83bc 100644 --- a/plugins/search-backend-module-pg/README.md +++ b/plugins/search-backend-module-pg/README.md @@ -17,7 +17,7 @@ for details on how to setup Postgres based search for your Backstage instance. ## Optional Configuration -The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature: +The following is an example of the optional configuration that can be applied when using Postgres as the search backend. ```yaml search: @@ -30,6 +30,20 @@ search: highlightAll: false # If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. maxFragments: 0 # Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). fragmentDelimiter: ' ... ' # Delimiter string used to concatenate fragments. Defaults to " ... ". + normalization: 0 # Ranking functions use an integer bit mask to control document length impact on rank. The default value is 0 +``` + +The Normalization option controls several behaviors. It is a bit mask. [Ranking Search Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING) has more details. + +Example with bit mask specifying more behaviours: + +- 2 divides the rank by the document length +- 4 divides the rank by the mean harmonic distance between extents + +```yaml +search: + pg: + normalization: 2 | 4 ``` **Note:** the highlight search term feature uses `ts_headline` which has been known to potentially impact performance. You only need this minimal config to disable it should you have issues: diff --git a/plugins/search-backend-module-pg/config.d.ts b/plugins/search-backend-module-pg/config.d.ts index 3039cd1d28..090885209c 100644 --- a/plugins/search-backend-module-pg/config.d.ts +++ b/plugins/search-backend-module-pg/config.d.ts @@ -57,6 +57,10 @@ export interface Config { */ fragmentDelimiter?: string; }; + /** + * Ranking functions use an integer bit mask to control document length impact on rank. The default value is 0 + */ + normalization?: number | string; /** * Batch size to use when indexing */ diff --git a/plugins/search-backend-module-pg/report.api.md b/plugins/search-backend-module-pg/report.api.md index 104821328c..ff2129b4bb 100644 --- a/plugins/search-backend-module-pg/report.api.md +++ b/plugins/search-backend-module-pg/report.api.md @@ -164,6 +164,8 @@ export interface PgSearchQuery { // (undocumented) limit: number; // (undocumented) + normalization?: number; + // (undocumented) offset: number; // (undocumented) options: PgSearchHighlightOptions; @@ -182,6 +184,7 @@ export type PgSearchQueryTranslator = ( // @public export type PgSearchQueryTranslatorOptions = { highlightOptions: PgSearchHighlightOptions; + normalization?: number; }; // @public (undocumented) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 23f8983074..c56816b475 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -86,7 +86,10 @@ describe('PgSearchEngine', () => { term: 'testTerm', filters: {}, }, - { highlightOptions }, + { + highlightOptions, + normalization: 0, + }, ); }); @@ -237,6 +240,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 0, limit: 26, + normalization: 0, options: highlightOptions, }); }); @@ -294,6 +298,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 0, limit: 26, + normalization: 0, options: highlightOptions, }); }); @@ -354,6 +359,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 25, limit: 26, + normalization: 0, options: highlightOptions, }); }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 4ff787071a..5b24d71100 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -45,6 +45,7 @@ export type ConcretePgSearchQuery = { */ export type PgSearchQueryTranslatorOptions = { highlightOptions: PgSearchHighlightOptions; + normalization?: number; }; /** @@ -86,6 +87,7 @@ export class PgSearchEngine implements SearchEngine { private readonly logger?: LoggerService; private readonly highlightOptions: PgSearchHighlightOptions; private readonly indexerBatchSize: number; + private readonly normalization: number; /** * @deprecated This will be marked as private in a future release, please us fromConfig instead @@ -116,9 +118,42 @@ export class PgSearchEngine implements SearchEngine { this.highlightOptions = highlightOptions; this.indexerBatchSize = config.getOptionalNumber('search.pg.indexerBatchSize') ?? 1000; + + this.normalization = this.getNormalizationValue(config); this.logger = logger; } + private getNormalizationValue(config: Config) { + const normalizationConfig = + config.getOptional('search.pg.normalization') ?? 0; + if (typeof normalizationConfig === 'number') { + return normalizationConfig; + } else if (typeof normalizationConfig === 'string') { + return this.evaluateBitwiseOrExpression(normalizationConfig); + } + this.logger?.error( + `Unknown normalization configuration: ${normalizationConfig}`, + ); + + return 0; + } + + private evaluateBitwiseOrExpression(expression: string) { + const tokens = expression.split('|').map(token => token.trim()); + + const numbers = tokens.map(token => { + const num = parseInt(token, 10); + if (isNaN(num)) { + this.logger?.error( + `Unknown expression for normalization: ${expression}`, + ); + return 0; + } + return num; + }); + return numbers.reduce((acc, num) => acc | num, 0); + } + /** * @deprecated This will be removed in a future release, please use fromConfig instead */ @@ -155,6 +190,7 @@ export class PgSearchEngine implements SearchEngine { const offset = page * pageSize; // We request more result to know whether there is another page const limit = pageSize + 1; + const normalization = options.normalization || 0; return { pgQuery: { @@ -168,6 +204,7 @@ export class PgSearchEngine implements SearchEngine { types: query.types, offset, limit, + normalization, options: options.highlightOptions, }, pageSize, @@ -190,6 +227,7 @@ export class PgSearchEngine implements SearchEngine { async query(query: SearchQuery): Promise { const { pgQuery, pageSize } = this.translator(query, { highlightOptions: this.highlightOptions, + normalization: this.normalization, }); const rows = await this.databaseStore.transaction(async tx => diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index d68b36fb54..0029b67bd6 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -240,6 +240,7 @@ describe('DatabaseDocumentStore', () => { pgTerm: 'Hello & World', offset: 1, limit: 1, + normalization: 0, options: highlightOptions, }), ); @@ -285,6 +286,7 @@ describe('DatabaseDocumentStore', () => { pgTerm: 'Hello & World', offset: 0, limit: 25, + normalization: 0, options: highlightOptions, }), ); @@ -346,6 +348,7 @@ describe('DatabaseDocumentStore', () => { types: ['my-type'], offset: 0, limit: 25, + normalization: 0, options: highlightOptions, }), ); @@ -399,6 +402,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, + normalization: 0, options: highlightOptions, }), ); @@ -465,6 +469,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: ['this', 'that'] }, offset: 0, limit: 25, + normalization: 0, options: highlightOptions, }), ); @@ -546,6 +551,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this', otherField: 'another' }, offset: 0, limit: 25, + normalization: 0, options: highlightOptions, }), ); @@ -595,6 +601,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, + normalization: 0, options: highlightOptions, }), ); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 0487097d9a..8bff8c0a60 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -148,10 +148,18 @@ export class DatabaseDocumentStore implements DatabaseStore { tx: Knex.Transaction, searchQuery: PgSearchQuery, ): Promise { - const { types, pgTerm, fields, offset, limit, options } = searchQuery; + const { + types, + pgTerm, + fields, + offset, + limit, + normalization = 0, + options, + } = searchQuery; // TODO(awanlin): We should make the language a parameter so that we can support more then just english // Builds a query like: - // SELECT ts_rank_cd(body, query) AS rank, type, document, + // SELECT ts_rank_cd(body, query, 0) AS rank, type, document, // ts_headline('english', document, query) AS highlight // FROM documents, to_tsquery('english', 'consent') query // WHERE query @@ body AND (document @> '{"kind": "API"}') @@ -194,7 +202,7 @@ export class DatabaseDocumentStore implements DatabaseStore { if (pgTerm && options.useHighlight) { const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`; query - .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS "rank"`)) .select( tx.raw( `ts_headline(\'english\', document, query, '${headlineOptions}') as "highlight"`, @@ -203,7 +211,7 @@ export class DatabaseDocumentStore implements DatabaseStore { .orderBy('rank', 'desc'); } else if (pgTerm && !options.useHighlight) { query - .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .select(tx.raw(`ts_rank_cd(body, query, ${normalization}) AS "rank"`)) .orderBy('rank', 'desc'); } else { query.select(tx.raw('1 as rank')); diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index f330651cef..5093a170ac 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -24,6 +24,7 @@ export interface PgSearchQuery { pgTerm?: string; offset: number; limit: number; + normalization?: number; options: PgSearchHighlightOptions; }