diff --git a/.changeset/metal-windows-share.md b/.changeset/metal-windows-share.md new file mode 100644 index 0000000000..b4443a620f --- /dev/null +++ b/.changeset/metal-windows-share.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +**DEPRECATED**: `PgSearchEngine` static `from` has been deprecated and will be removed in a future release. Use static `fromConfig` method to instantiate. + +Added support for highlighting matched terms in search result data diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index a5ff780c8a..ecd8a85109 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -59,10 +59,38 @@ configured and make the following changes to your backend: // Initialize a connection to a search engine. const searchEngine = (await PgSearchEngine.supported(env.database)) - ? await PgSearchEngine.from({ database: env.database }) + ? await PgSearchEngine.fromConfig(env.config, { database: env.database }) : new LunrSearchEngine({ logger: env.logger }); ``` +## Optional Configuration + +The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature: + +```yaml +search: + pg: + highlightOptions: + useHighlight: true # Used to enable to disable the highlight feature. The default value is true + maxWord: 35 # Used to set the longest headlines to output. The default value is 35. + minWord: 15 # Used to set the shortest headlines to output. The default value is 15. + shortWord: 3 # Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. The default value of three (3) eliminates common English articles. + highlightAll: false # If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + maxFragments: 0 # Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + fragmentDelimiter: ' ... ' # Delimiter string used to concatenate fragments. Defaults to " ... ". +``` + +**Note:** the highlight search term feature uses `ts_headline` which has been known to potentially impact performance. You only need this minimal config to disable it should you have issues: + +```yaml +search: + pg: + highlightOptions: + useHighlight: false +``` + +The Postgres documentation on [Highlighting Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE) has more details. + ## ElasticSearch Backstage supports ElasticSearch search engine connections, indexing and diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index a0ea83ab61..c478ecdd05 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -39,7 +39,9 @@ async function createSearchEngine( } if (await PgSearchEngine.supported(env.database)) { - return await PgSearchEngine.from({ database: env.database }); + return await PgSearchEngine.fromConfig(env.config, { + database: env.database, + }); } return new LunrSearchEngine({ logger: env.logger }); diff --git a/plugins/search-backend-module-pg/README.md b/plugins/search-backend-module-pg/README.md index d3f98132aa..49804a927c 100644 --- a/plugins/search-backend-module-pg/README.md +++ b/plugins/search-backend-module-pg/README.md @@ -14,3 +14,31 @@ other plugins. See [Backstage documentation](https://backstage.io/docs/features/search/search-engines#postgres) for details on how to setup Postgres based search for your Backstage instance. + +## Optional Configuration + +The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature: + +```yaml +search: + pg: + highlightOptions: + useHighlight: true # Used to enable to disable the highlight feature. The default value is true + maxWord: 35 # Used to set the longest headlines to output. The default value is 35. + minWord: 15 # Used to set the shortest headlines to output. The default value is 15. + shortWord: 3 # Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. The default value of three (3) eliminates common English articles. + highlightAll: false # If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + maxFragments: 0 # Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + fragmentDelimiter: ' ... ' # Delimiter string used to concatenate fragments. Defaults to " ... ". +``` + +**Note:** the highlight search term feature uses `ts_headline` which has been known to potentially impact performance. You only need this minimal config to disable it should you have issues: + +```yaml +search: + pg: + highlightOptions: + useHighlight: false +``` + +The Postgres documentation on [Highlighting Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE) has more details. diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index 71ceac996e..f7a188a14e 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -4,6 +4,7 @@ ```ts import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { Config } from '@backstage/config'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; @@ -11,9 +12,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-backend-node'; import { SearchQuery } from '@backstage/plugin-search-common'; -// Warning: (ae-missing-release-tag) "ConcretePgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; pageSize: number; @@ -43,7 +42,7 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit }: PgSearchQuery, + searchQuery: PgSearchQuery, ): Promise; // (undocumented) static supported(knex: Knex): Promise; @@ -80,23 +79,31 @@ export interface DatabaseStore { // // @public (undocumented) export class PgSearchEngine implements SearchEngine { - constructor(databaseStore: DatabaseStore); - // (undocumented) + // @deprecated + constructor(databaseStore: DatabaseStore, config: Config); + // @deprecated (undocumented) static from(options: { database: PluginDatabaseManager; + config: Config; }): Promise; // (undocumented) + static fromConfig( + config: Config, + options: PgSearchOptions, + ): Promise; + // (undocumented) getIndexer(type: string): Promise; // (undocumented) query(query: SearchQuery): Promise; // (undocumented) - setTranslator( - translator: (query: SearchQuery) => ConcretePgSearchQuery, - ): void; + setTranslator(translator: PgSearchQueryTranslator): void; // (undocumented) static supported(database: PluginDatabaseManager): Promise; // (undocumented) - translator(query: SearchQuery): ConcretePgSearchQuery; + translator( + query: SearchQuery, + options: PgSearchQueryTranslatorOptions, + ): ConcretePgSearchQuery; } // Warning: (ae-missing-release-tag) "PgSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -121,6 +128,24 @@ export type PgSearchEngineIndexerOptions = { databaseStore: DatabaseStore; }; +// @public +export type PgSearchHighlightOptions = { + useHighlight?: boolean; + maxWords?: number; + minWords?: number; + shortWord?: number; + highlightAll?: boolean; + maxFragments?: number; + fragmentDelimiter?: string; + preTag: string; + postTag: string; +}; + +// @public +export type PgSearchOptions = { + database: PluginDatabaseManager; +}; + // Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -132,11 +157,24 @@ export interface PgSearchQuery { // (undocumented) offset: number; // (undocumented) + options: PgSearchHighlightOptions; + // (undocumented) pgTerm?: string; // (undocumented) types?: string[]; } +// @public +export type PgSearchQueryTranslator = ( + query: SearchQuery, + options: PgSearchQueryTranslatorOptions, +) => ConcretePgSearchQuery; + +// @public +export type PgSearchQueryTranslatorOptions = { + highlightOptions: PgSearchHighlightOptions; +}; + // Warning: (ae-missing-release-tag) "RawDocumentRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/search-backend-module-pg/config.d.ts b/plugins/search-backend-module-pg/config.d.ts new file mode 100644 index 0000000000..7367b770bc --- /dev/null +++ b/plugins/search-backend-module-pg/config.d.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the search plugin */ + search?: { + /** + * Options for PG + */ + pg?: { + /** + * Options for configuring highlight settings + * See https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE + */ + highlightOptions?: { + /** + * Used to enable to disable the highlight feature. The default value is true + */ + useHighlight?: boolean; + /** + * Used to set the longest headlines to output. The default value is 35. + */ + maxWords?: number; + /** + * Used to set the shortest headlines to output. The default value is 15. + */ + minWords?: number; + /** + * Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. + * The default value of three (3) eliminates common English articles. + */ + shortWord?: number; + /** + * If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + */ + highlightAll?: boolean; + /** + * Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. + * A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + */ + maxFragments?: number; + /** + * Delimiter string used to concatenate fragments. Defaults to " ... ". + */ + fragmentDelimiter?: string; + }; + }; + }; +} diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index fb8cd4afc4..16367b9ef1 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -24,10 +24,12 @@ }, "dependencies": { "@backstage/backend-common": "^0.14.1-next.1", + "@backstage/config": "^1.0.1", "@backstage/plugin-search-backend-node": "^0.6.3-next.1", "@backstage/plugin-search-common": "^0.3.6-next.0", "lodash": "^4.17.21", - "knex": "^2.0.0" + "knex": "^2.0.0", + "uuid": "^8.3.2" }, "devDependencies": { "@backstage/backend-test-utils": "^0.1.26-next.1", @@ -35,6 +37,8 @@ }, "files": [ "dist", - "migrations" - ] + "migrations", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 3ed002567e..23f8983074 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -13,15 +13,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; import { DatabaseStore } from '../database'; import { ConcretePgSearchQuery, decodePageCursor, encodePageCursor, PgSearchEngine, + PgSearchHighlightOptions, } from './PgSearchEngine'; import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; +const highlightOptions: PgSearchHighlightOptions = { + preTag: '', + postTag: '', + useHighlight: true, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', +}; + +jest.mock('uuid', () => ({ v4: () => 'tag' })); + jest.mock('./PgSearchEngineIndexer', () => ({ PgSearchEngineIndexer: jest .fn() @@ -32,6 +48,13 @@ describe('PgSearchEngine', () => { const tx: any = {} as any; let searchEngine: PgSearchEngine; let database: jest.Mocked; + const config = { + search: { + pg: { + highlightOptions, + }, + }, + }; beforeEach(() => { database = { @@ -42,7 +65,7 @@ describe('PgSearchEngine', () => { completeInsert: jest.fn(), prepareInsert: jest.fn(), }; - searchEngine = new PgSearchEngine(database); + searchEngine = new PgSearchEngine(database, new ConfigReader(config)); }); describe('translator', () => { @@ -58,17 +81,23 @@ describe('PgSearchEngine', () => { filters: {}, }); - expect(translatorSpy).toHaveBeenCalledWith({ - term: 'testTerm', - filters: {}, - }); + expect(translatorSpy).toHaveBeenCalledWith( + { + term: 'testTerm', + filters: {}, + }, + { highlightOptions }, + ); }); it('should pass page cursor', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'Hello', - pageCursor: 'MQ==', - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'Hello', + pageCursor: 'MQ==', + }, + { highlightOptions }, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -81,9 +110,12 @@ describe('PgSearchEngine', () => { }); it('should return translated query term', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'Hello World', - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'Hello World', + }, + { highlightOptions }, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -96,10 +128,13 @@ describe('PgSearchEngine', () => { }); it('should sanitize query term', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'H&e|l!l*o W\0o(r)l:d', - pageCursor: '', - }) as ConcretePgSearchQuery; + const actualTranslatedQuery = searchEngine.translator( + { + term: 'H&e|l!l*o W\0o(r)l:d', + pageCursor: '', + }, + { highlightOptions }, + ) as ConcretePgSearchQuery; expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -110,11 +145,14 @@ describe('PgSearchEngine', () => { }); it('should return translated query with filters', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'testTerm', - filters: { kind: 'testKind' }, - types: ['my-filter'], - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'testTerm', + filters: { kind: 'testKind' }, + types: ['my-filter'], + }, + { highlightOptions }, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -158,6 +196,11 @@ describe('PgSearchEngine', () => { location: 'location-1', }, type: 'my-type', + highlight: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, }, ]); @@ -175,6 +218,16 @@ describe('PgSearchEngine', () => { }, type: 'my-type', rank: 1, + highlight: { + preTag: '', + postTag: '', + fields: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + path: '', + }, + }, }, ], nextPageCursor: undefined, @@ -184,6 +237,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 0, limit: 26, + options: highlightOptions, }); }); @@ -199,6 +253,11 @@ describe('PgSearchEngine', () => { location: `location-${i}`, }, type: 'my-type', + highlight: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, })), ); @@ -217,6 +276,16 @@ describe('PgSearchEngine', () => { }, type: 'my-type', rank: i + 1, + highlight: { + preTag: '', + postTag: '', + fields: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + path: '', + }, + }, })), nextPageCursor: 'MQ==', }); @@ -225,6 +294,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 0, limit: 26, + options: highlightOptions, }); }); @@ -240,6 +310,11 @@ describe('PgSearchEngine', () => { location: `location-${i}`, }, type: 'my-type', + highlight: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, })) .slice(25), ); @@ -260,6 +335,16 @@ describe('PgSearchEngine', () => { }, type: 'my-type', rank: i + 1, + highlight: { + preTag: '', + postTag: '', + fields: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + path: '', + }, + }, })) .slice(25), previousPageCursor: 'MA==', @@ -269,6 +354,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 25, limit: 26, + options: highlightOptions, }); }); }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index bf98c58de9..abf7cdbf5c 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -26,20 +26,104 @@ import { DatabaseStore, PgSearchQuery, } from '../database'; +import { v4 as uuid } from 'uuid'; +import { Config } from '@backstage/config'; +/** + * Search query that the Postgres search engine understands. + * @public + */ export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; pageSize: number; }; -export class PgSearchEngine implements SearchEngine { - constructor(private readonly databaseStore: DatabaseStore) {} +/** + * Options available for the Postgres specific query translator. + * @public + */ +export type PgSearchQueryTranslatorOptions = { + highlightOptions: PgSearchHighlightOptions; +}; +/** + * Postgres specific query translator. + * @public + */ +export type PgSearchQueryTranslator = ( + query: SearchQuery, + options: PgSearchQueryTranslatorOptions, +) => ConcretePgSearchQuery; + +/** + * Options to instantiate PgSearchEngine + * @public + */ +export type PgSearchOptions = { + database: PluginDatabaseManager; +}; + +/** + * Options for highlighting search terms + * @public + */ +export type PgSearchHighlightOptions = { + useHighlight?: boolean; + maxWords?: number; + minWords?: number; + shortWord?: number; + highlightAll?: boolean; + maxFragments?: number; + fragmentDelimiter?: string; + preTag: string; + postTag: string; +}; + +export class PgSearchEngine implements SearchEngine { + private readonly highlightOptions: PgSearchHighlightOptions; + + /** + * @deprecated This will be marked as private in a future release, please us fromConfig instead + */ + constructor(private readonly databaseStore: DatabaseStore, config: Config) { + const uuidTag = uuid(); + const highlightConfig = config.getOptionalConfig( + 'search.pg.highlightOptions', + ); + + const highlightOptions: PgSearchHighlightOptions = { + preTag: `<${uuidTag}>`, + postTag: ``, + useHighlight: highlightConfig?.getOptionalBoolean('useHighlight') ?? true, + maxWords: highlightConfig?.getOptionalNumber('maxWords') ?? 35, + minWords: highlightConfig?.getOptionalNumber('minWords') ?? 15, + shortWord: highlightConfig?.getOptionalNumber('shortWord') ?? 3, + highlightAll: + highlightConfig?.getOptionalBoolean('highlightAll') ?? false, + maxFragments: highlightConfig?.getOptionalNumber('maxFragments') ?? 0, + fragmentDelimiter: + highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ', + }; + this.highlightOptions = highlightOptions; + } + + /** + * @deprecated This will be removed in a future release, please us fromConfig instead + */ static async from(options: { database: PluginDatabaseManager; + config: Config; }): Promise { return new PgSearchEngine( await DatabaseDocumentStore.create(await options.database.getClient()), + options.config, + ); + } + + static async fromConfig(config: Config, options: PgSearchOptions) { + return new PgSearchEngine( + await DatabaseDocumentStore.create(await options.database.getClient()), + config, ); } @@ -47,7 +131,10 @@ export class PgSearchEngine implements SearchEngine { return await DatabaseDocumentStore.supported(await database.getClient()); } - translator(query: SearchQuery): ConcretePgSearchQuery { + translator( + query: SearchQuery, + options: PgSearchQueryTranslatorOptions, + ): ConcretePgSearchQuery { const pageSize = 25; const { page } = decodePageCursor(query.pageCursor); const offset = page * pageSize; @@ -66,14 +153,13 @@ export class PgSearchEngine implements SearchEngine { types: query.types, offset, limit, + options: options.highlightOptions, }, pageSize, }; } - setTranslator( - translator: (query: SearchQuery) => ConcretePgSearchQuery, - ): void { + setTranslator(translator: PgSearchQueryTranslator) { this.translator = translator; } @@ -86,7 +172,9 @@ export class PgSearchEngine implements SearchEngine { } async query(query: SearchQuery): Promise { - const { pgQuery, pageSize } = this.translator(query); + const { pgQuery, pageSize } = this.translator(query, { + highlightOptions: this.highlightOptions, + }); const rows = await this.databaseStore.transaction(async tx => this.databaseStore.query(tx, pgQuery), @@ -106,10 +194,22 @@ export class PgSearchEngine implements SearchEngine { : undefined; const results = pageRows.map( - ({ type, document }, index): IndexableResult => ({ + ({ type, document, highlight }, index): IndexableResult => ({ type, document, rank: page * pageSize + index + 1, + highlight: { + preTag: pgQuery.options.preTag, + postTag: pgQuery.options.postTag, + fields: highlight + ? { + text: highlight.text, + title: highlight.title, + location: highlight.location, + path: '', + } + : {}, + }, }), ); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index 7f8e297648..e9ea04e5c4 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -14,7 +14,13 @@ * limitations under the License. */ export { PgSearchEngine } from './PgSearchEngine'; -export type { ConcretePgSearchQuery } from './PgSearchEngine'; +export type { + ConcretePgSearchQuery, + PgSearchQueryTranslatorOptions, + PgSearchQueryTranslator, + PgSearchOptions, + PgSearchHighlightOptions, +} from './PgSearchEngine'; export type { PgSearchEngineIndexer, PgSearchEngineIndexerOptions, diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index b973c2e14f..355e41352d 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -15,8 +15,21 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { IndexableDocument } from '@backstage/plugin-search-common'; +import { PgSearchHighlightOptions } from '../PgSearchEngine'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; +const highlightOptions: PgSearchHighlightOptions = { + preTag: '', + postTag: '', + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', +}; + describe('DatabaseDocumentStore', () => { describe('unsupported', () => { const databases = TestDatabases.create({ @@ -220,7 +233,12 @@ describe('DatabaseDocumentStore', () => { }); const rows = await store.transaction(tx => - store.query(tx, { pgTerm: 'Hello & World', offset: 1, limit: 1 }), + store.query(tx, { + pgTerm: 'Hello & World', + offset: 1, + limit: 1, + options: highlightOptions, + }), ); expect(rows).toEqual([ @@ -261,7 +279,12 @@ describe('DatabaseDocumentStore', () => { }); const rows = await store.transaction(tx => - store.query(tx, { pgTerm: 'Hello & World', offset: 0, limit: 25 }), + store.query(tx, { + pgTerm: 'Hello & World', + offset: 0, + limit: 25, + options: highlightOptions, + }), ); expect(rows).toEqual([ @@ -322,6 +345,7 @@ describe('DatabaseDocumentStore', () => { types: ['my-type'], offset: 0, limit: 25, + options: highlightOptions, }), ); @@ -375,6 +399,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, + options: highlightOptions, }), ); @@ -429,6 +454,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: ['this', 'that'] }, offset: 0, limit: 25, + options: highlightOptions, }), ); @@ -490,6 +516,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this', otherField: 'another' }, offset: 0, limit: 25, + options: highlightOptions, }), ); @@ -539,6 +566,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, + options: highlightOptions, }), ); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 33b2d0c0d6..8e73c0f16f 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -132,10 +132,13 @@ export class DatabaseDocumentStore implements DatabaseStore { async query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit }: PgSearchQuery, + searchQuery: PgSearchQuery, ): Promise { + const { types, pgTerm, fields, offset, limit, options } = searchQuery; + // TODO(awanlin): We should make the language a parameter so that we can support more then just english // Builds a query like: - // SELECT ts_rank_cd(body, query) AS rank, type, document + // SELECT ts_rank_cd(body, query) AS rank, type, document, + // ts_headline('english', document, query) AS highlight // FROM documents, to_tsquery('english', 'consent') query // WHERE query @@ body AND (document @> '{"kind": "API"}') // ORDER BY rank DESC @@ -170,7 +173,17 @@ export class DatabaseDocumentStore implements DatabaseStore { query.select('type', 'document'); - if (pgTerm) { + if (pgTerm && options.useHighlight) { + const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`; + query + .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .select( + tx.raw( + `ts_headline(\'english\', document, query, '${headlineOptions}') as "highlight"`, + ), + ) + .orderBy('rank', 'desc'); + } else if (pgTerm && !options.useHighlight) { query .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) .orderBy('rank', 'desc'); diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 7e87658745..151be7d13f 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -15,6 +15,7 @@ */ import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { PgSearchHighlightOptions } from '../PgSearchEngine'; export interface PgSearchQuery { fields?: Record; @@ -22,6 +23,7 @@ export interface PgSearchQuery { pgTerm?: string; offset: number; limit: number; + options: PgSearchHighlightOptions; } export interface DatabaseStore { @@ -49,4 +51,5 @@ export interface RawDocumentRow { export interface DocumentResultRow { document: IndexableDocument; type: string; + highlight: IndexableDocument; }