From 423e3d8e957606438755a7c712e7c8450d4b91ea Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Fri, 17 Jun 2022 19:51:40 -0500 Subject: [PATCH 01/10] Highlight PG search results Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .changeset/metal-windows-share.md | 5 ++++ .../search-backend-module-pg/api-report.md | 6 ++++- plugins/search-backend-module-pg/package.json | 3 ++- .../src/PgSearchEngine/PgSearchEngine.test.ts | 15 +++++++++++ .../src/PgSearchEngine/PgSearchEngine.ts | 18 ++++++++++++- .../database/DatabaseDocumentStore.test.ts | 26 +++++++++++++++++-- .../src/database/DatabaseDocumentStore.ts | 10 +++++-- .../src/database/types.ts | 3 +++ 8 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 .changeset/metal-windows-share.md diff --git a/.changeset/metal-windows-share.md b/.changeset/metal-windows-share.md new file mode 100644 index 0000000000..4fc568986a --- /dev/null +++ b/.changeset/metal-windows-share.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Added support for highlighting matched terms in search result data diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index 71ceac996e..6a8b68a042 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -43,7 +43,7 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit }: PgSearchQuery, + { types, pgTerm, fields, offset, limit, preTag, postTag }: PgSearchQuery, ): Promise; // (undocumented) static supported(knex: Knex): Promise; @@ -134,6 +134,10 @@ export interface PgSearchQuery { // (undocumented) pgTerm?: string; // (undocumented) + postTag: string; + // (undocumented) + preTag: string; + // (undocumented) types?: string[]; } diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index fb8cd4afc4..7809f9dd06 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -27,7 +27,8 @@ "@backstage/plugin-search-backend-node": "^0.6.3-next.1", "@backstage/plugin-search-common": "^0.3.6-next.0", "lodash": "^4.17.21", - "knex": "^2.0.0" + "knex": "^2.0.0", + "uuid": "^8.3.2" }, "devDependencies": { "@backstage/backend-test-utils": "^0.1.26-next.1", diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 3ed002567e..5f3bc3c5d8 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -158,6 +158,11 @@ describe('PgSearchEngine', () => { location: 'location-1', }, type: 'my-type', + highlight: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, }, ]); @@ -199,6 +204,11 @@ describe('PgSearchEngine', () => { location: `location-${i}`, }, type: 'my-type', + highlight: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, })), ); @@ -240,6 +250,11 @@ describe('PgSearchEngine', () => { location: `location-${i}`, }, type: 'my-type', + highlight: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, })) .slice(25), ); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index bf98c58de9..77aa003c5d 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -26,6 +26,7 @@ import { DatabaseStore, PgSearchQuery, } from '../database'; +import { v4 as uuid } from 'uuid'; export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; @@ -53,6 +54,7 @@ export class PgSearchEngine implements SearchEngine { const offset = page * pageSize; // We request more result to know whether there is another page const limit = pageSize + 1; + const uuidTag = uuid(); return { pgQuery: { @@ -66,6 +68,8 @@ export class PgSearchEngine implements SearchEngine { types: query.types, offset, limit, + preTag: `<${uuidTag}>`, + postTag: ``, }, pageSize, }; @@ -106,10 +110,22 @@ export class PgSearchEngine implements SearchEngine { : undefined; const results = pageRows.map( - ({ type, document }, index): IndexableResult => ({ + ({ type, document, highlight }, index): IndexableResult => ({ type, document, rank: page * pageSize + index + 1, + highlight: { + preTag: pgQuery.preTag, + postTag: pgQuery.postTag, + fields: highlight + ? { + text: highlight.text, + title: highlight.title, + location: highlight.location, + path: '', + } + : {}, + }, }), ); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index b973c2e14f..e2303d90b2 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -220,7 +220,13 @@ describe('DatabaseDocumentStore', () => { }); const rows = await store.transaction(tx => - store.query(tx, { pgTerm: 'Hello & World', offset: 1, limit: 1 }), + store.query(tx, { + pgTerm: 'Hello & World', + offset: 1, + limit: 1, + preTag: '', + postTag: '', + }), ); expect(rows).toEqual([ @@ -261,7 +267,13 @@ describe('DatabaseDocumentStore', () => { }); const rows = await store.transaction(tx => - store.query(tx, { pgTerm: 'Hello & World', offset: 0, limit: 25 }), + store.query(tx, { + pgTerm: 'Hello & World', + offset: 0, + limit: 25, + preTag: '', + postTag: '', + }), ); expect(rows).toEqual([ @@ -322,6 +334,8 @@ describe('DatabaseDocumentStore', () => { types: ['my-type'], offset: 0, limit: 25, + preTag: '', + postTag: '', }), ); @@ -375,6 +389,8 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, + preTag: '', + postTag: '', }), ); @@ -429,6 +445,8 @@ describe('DatabaseDocumentStore', () => { fields: { myField: ['this', 'that'] }, offset: 0, limit: 25, + preTag: '', + postTag: '', }), ); @@ -490,6 +508,8 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this', otherField: 'another' }, offset: 0, limit: 25, + preTag: '', + postTag: '', }), ); @@ -539,6 +559,8 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, + preTag: '', + postTag: '', }), ); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 33b2d0c0d6..3a1ec128ef 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -132,10 +132,11 @@ export class DatabaseDocumentStore implements DatabaseStore { async query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit }: PgSearchQuery, + { types, pgTerm, fields, offset, limit, preTag, postTag }: PgSearchQuery, ): Promise { // Builds a query like: - // SELECT ts_rank_cd(body, query) AS rank, type, document + // SELECT ts_rank_cd(body, query) AS rank, type, document, + // ts_headline('english', document, query) AS highlight // FROM documents, to_tsquery('english', 'consent') query // WHERE query @@ body AND (document @> '{"kind": "API"}') // ORDER BY rank DESC @@ -173,6 +174,11 @@ export class DatabaseDocumentStore implements DatabaseStore { if (pgTerm) { query .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .select( + tx.raw( + `ts_headline(\'english\', document, query, 'StartSel=${preTag}, StopSel=${postTag}') as "highlight"`, + ), + ) .orderBy('rank', 'desc'); } else { query.select(tx.raw('1 as rank')); diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 7e87658745..23679d100f 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -22,6 +22,8 @@ export interface PgSearchQuery { pgTerm?: string; offset: number; limit: number; + preTag: string; + postTag: string; } export interface DatabaseStore { @@ -49,4 +51,5 @@ export interface RawDocumentRow { export interface DocumentResultRow { document: IndexableDocument; type: string; + highlight: IndexableDocument; } From d43228a1832118dfa2b1e60c6e37c9d8c6568dc5 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Sat, 18 Jun 2022 07:39:23 -0500 Subject: [PATCH 02/10] Refactored to support config with options Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- packages/backend/src/plugins/search.ts | 5 +- plugins/search-backend-module-pg/config.d.ts | 62 +++++++++++++++++++ plugins/search-backend-module-pg/package.json | 7 ++- .../src/PgSearchEngine/PgSearchEngine.test.ts | 57 ++++++++++++----- .../src/PgSearchEngine/PgSearchEngine.ts | 42 ++++++++++--- .../database/DatabaseDocumentStore.test.ts | 34 +++++----- .../src/database/DatabaseDocumentStore.ts | 11 +++- .../src/database/types.ts | 4 +- plugins/search-backend-module-pg/src/types.ts | 40 ++++++++++++ 9 files changed, 216 insertions(+), 46 deletions(-) create mode 100644 plugins/search-backend-module-pg/config.d.ts create mode 100644 plugins/search-backend-module-pg/src/types.ts diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index a0ea83ab61..db92d79d71 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -39,7 +39,10 @@ async function createSearchEngine( } if (await PgSearchEngine.supported(env.database)) { - return await PgSearchEngine.from({ database: env.database }); + return await PgSearchEngine.from({ + database: env.database, + config: env.config, + }); } return new LunrSearchEngine({ logger: env.logger }); diff --git a/plugins/search-backend-module-pg/config.d.ts b/plugins/search-backend-module-pg/config.d.ts new file mode 100644 index 0000000000..6cdda27156 --- /dev/null +++ b/plugins/search-backend-module-pg/config.d.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the search plugin */ + search?: { + /** + * Options for PG + */ + pg?: { + /** + * Options for configuring highlight settings + * See https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE + */ + highlightOptions?: { + /** + * Used to enable to disable the highlight feature. The default value is true + */ + useHighlight?: boolean; + /** + * Used to set the longest headlines to output. The default value is 35. + */ + maxWords?: number; + /** + * Used to set the shortest headlines to output. The default value is 15. + */ + minWords?: number; + /** + * Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. + * The default value of three (3) eliminates common English articles. + */ + shortWord?: number; + /** + * If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + */ + highlightAll?: boolean; + /** + * maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. + * A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + */ + maxFragments?: number; + /** + * Delimiter string used to concatenate fragments. Defaults to " ... ". + */ + fragmentDelimiter?: string; + }; + }; + }; +} diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index 7809f9dd06..16367b9ef1 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.14.1-next.1", + "@backstage/config": "^1.0.1", "@backstage/plugin-search-backend-node": "^0.6.3-next.1", "@backstage/plugin-search-common": "^0.3.6-next.0", "lodash": "^4.17.21", @@ -36,6 +37,8 @@ }, "files": [ "dist", - "migrations" - ] + "migrations", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 5f3bc3c5d8..1d0e31757f 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import { DatabaseStore } from '../database'; +import { PgSearchHighlightConfig } from '../types'; import { ConcretePgSearchQuery, decodePageCursor, @@ -22,6 +23,18 @@ import { } from './PgSearchEngine'; import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; +const highlightOptions: PgSearchHighlightConfig = { + preTag: '', + postTag: '', + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', +}; + jest.mock('./PgSearchEngineIndexer', () => ({ PgSearchEngineIndexer: jest .fn() @@ -65,10 +78,13 @@ describe('PgSearchEngine', () => { }); it('should pass page cursor', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'Hello', - pageCursor: 'MQ==', - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'Hello', + pageCursor: 'MQ==', + }, + highlightOptions, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -81,9 +97,12 @@ describe('PgSearchEngine', () => { }); it('should return translated query term', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'Hello World', - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'Hello World', + }, + highlightOptions, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -96,10 +115,13 @@ describe('PgSearchEngine', () => { }); it('should sanitize query term', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'H&e|l!l*o W\0o(r)l:d', - pageCursor: '', - }) as ConcretePgSearchQuery; + const actualTranslatedQuery = searchEngine.translator( + { + term: 'H&e|l!l*o W\0o(r)l:d', + pageCursor: '', + }, + highlightOptions, + ) as ConcretePgSearchQuery; expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -110,11 +132,14 @@ describe('PgSearchEngine', () => { }); it('should return translated query with filters', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'testTerm', - filters: { kind: 'testKind' }, - types: ['my-filter'], - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'testTerm', + filters: { kind: 'testKind' }, + types: ['my-filter'], + }, + highlightOptions, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 77aa003c5d..764e530435 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -27,6 +27,8 @@ import { PgSearchQuery, } from '../database'; import { v4 as uuid } from 'uuid'; +import { Config } from '@backstage/config'; +import { PgSearchHighlightConfig, PgSearchHighlightOptions } from '../types'; export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; @@ -34,13 +36,36 @@ export type ConcretePgSearchQuery = { }; export class PgSearchEngine implements SearchEngine { - constructor(private readonly databaseStore: DatabaseStore) {} + private readonly highlightOptions: PgSearchHighlightConfig; + + constructor( + private readonly databaseStore: DatabaseStore, + highlightOptions?: PgSearchHighlightOptions, + ) { + const uuidTag = uuid(); + this.highlightOptions = { + preTag: `<${uuidTag}>`, + postTag: ``, + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', + ...highlightOptions, + }; + } static async from(options: { database: PluginDatabaseManager; + config: Config; }): Promise { return new PgSearchEngine( await DatabaseDocumentStore.create(await options.database.getClient()), + options.config.getOptional( + 'search.pg.highlightOptions', + ), ); } @@ -48,13 +73,15 @@ export class PgSearchEngine implements SearchEngine { return await DatabaseDocumentStore.supported(await database.getClient()); } - translator(query: SearchQuery): ConcretePgSearchQuery { + translator( + query: SearchQuery, + options: PgSearchHighlightConfig, + ): ConcretePgSearchQuery { const pageSize = 25; const { page } = decodePageCursor(query.pageCursor); const offset = page * pageSize; // We request more result to know whether there is another page const limit = pageSize + 1; - const uuidTag = uuid(); return { pgQuery: { @@ -68,8 +95,7 @@ export class PgSearchEngine implements SearchEngine { types: query.types, offset, limit, - preTag: `<${uuidTag}>`, - postTag: ``, + options, }, pageSize, }; @@ -90,7 +116,7 @@ export class PgSearchEngine implements SearchEngine { } async query(query: SearchQuery): Promise { - const { pgQuery, pageSize } = this.translator(query); + const { pgQuery, pageSize } = this.translator(query, this.highlightOptions); const rows = await this.databaseStore.transaction(async tx => this.databaseStore.query(tx, pgQuery), @@ -115,8 +141,8 @@ export class PgSearchEngine implements SearchEngine { document, rank: page * pageSize + index + 1, highlight: { - preTag: pgQuery.preTag, - postTag: pgQuery.postTag, + preTag: pgQuery.options.preTag, + postTag: pgQuery.options.postTag, fields: highlight ? { text: highlight.text, diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index e2303d90b2..693a3ea3c0 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -15,8 +15,21 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { IndexableDocument } from '@backstage/plugin-search-common'; +import { PgSearchHighlightConfig } from '../types'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; +const highlightOptions: PgSearchHighlightConfig = { + preTag: '', + postTag: '', + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', +}; + describe('DatabaseDocumentStore', () => { describe('unsupported', () => { const databases = TestDatabases.create({ @@ -224,8 +237,7 @@ describe('DatabaseDocumentStore', () => { pgTerm: 'Hello & World', offset: 1, limit: 1, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -271,8 +283,7 @@ describe('DatabaseDocumentStore', () => { pgTerm: 'Hello & World', offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -334,8 +345,7 @@ describe('DatabaseDocumentStore', () => { types: ['my-type'], offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -389,8 +399,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -445,8 +454,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: ['this', 'that'] }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -508,8 +516,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this', otherField: 'another' }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -559,8 +566,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 3a1ec128ef..1695db3cf5 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -132,7 +132,7 @@ export class DatabaseDocumentStore implements DatabaseStore { async query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit, preTag, postTag }: PgSearchQuery, + { types, pgTerm, fields, offset, limit, options }: PgSearchQuery, ): Promise { // Builds a query like: // SELECT ts_rank_cd(body, query) AS rank, type, document, @@ -171,15 +171,20 @@ export class DatabaseDocumentStore implements DatabaseStore { query.select('type', 'document'); - if (pgTerm) { + if (pgTerm && options.useHighlight) { + const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`; query .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) .select( tx.raw( - `ts_headline(\'english\', document, query, 'StartSel=${preTag}, StopSel=${postTag}') as "highlight"`, + `ts_headline(\'english\', document, query, '${headlineOptions}') as "highlight"`, ), ) .orderBy('rank', 'desc'); + } else if (pgTerm && !options.useHighlight) { + query + .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .orderBy('rank', 'desc'); } else { query.select(tx.raw('1 as rank')); } diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 23679d100f..d572eba146 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -15,6 +15,7 @@ */ import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { PgSearchHighlightConfig } from '../types'; export interface PgSearchQuery { fields?: Record; @@ -22,8 +23,7 @@ export interface PgSearchQuery { pgTerm?: string; offset: number; limit: number; - preTag: string; - postTag: string; + options: PgSearchHighlightConfig; } export interface DatabaseStore { diff --git a/plugins/search-backend-module-pg/src/types.ts b/plugins/search-backend-module-pg/src/types.ts new file mode 100644 index 0000000000..9ad56b9b12 --- /dev/null +++ b/plugins/search-backend-module-pg/src/types.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type PgSearchHighlightOptions = { + useHighlight?: boolean; + maxWords?: number; + minWords?: number; + shortWord?: number; + highlightAll?: boolean; + maxFragments?: number; + fragmentDelimiter?: string; +}; + +/** + * @public + */ +export type PgSearchHighlightConfig = { + useHighlight: boolean; + maxWords: number; + minWords: number; + shortWord: number; + highlightAll: boolean; + maxFragments: number; + fragmentDelimiter: string; + preTag: string; + postTag: string; +}; From b69dd94766690fab7443b6f7af701ddc2f16bf72 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Sat, 18 Jun 2022 12:29:54 -0500 Subject: [PATCH 03/10] Fixed failing tests Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .../src/PgSearchEngine/PgSearchEngine.test.ts | 48 +++++++++++++++++-- .../src/PgSearchEngine/PgSearchEngine.ts | 2 +- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 1d0e31757f..c1ed730053 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -26,7 +26,7 @@ import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; const highlightOptions: PgSearchHighlightConfig = { preTag: '', postTag: '', - useHighlight: false, + useHighlight: true, maxWords: 35, minWords: 15, shortWord: 3, @@ -35,6 +35,8 @@ const highlightOptions: PgSearchHighlightConfig = { fragmentDelimiter: ' ... ', }; +jest.mock('uuid', () => ({ v4: () => 'tag' })); + jest.mock('./PgSearchEngineIndexer', () => ({ PgSearchEngineIndexer: jest .fn() @@ -71,10 +73,13 @@ describe('PgSearchEngine', () => { filters: {}, }); - expect(translatorSpy).toHaveBeenCalledWith({ - term: 'testTerm', - filters: {}, - }); + expect(translatorSpy).toHaveBeenCalledWith( + { + term: 'testTerm', + filters: {}, + }, + highlightOptions, + ); }); it('should pass page cursor', async () => { @@ -205,6 +210,16 @@ describe('PgSearchEngine', () => { }, type: 'my-type', rank: 1, + highlight: { + preTag: '', + postTag: '', + fields: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + path: '', + }, + }, }, ], nextPageCursor: undefined, @@ -214,6 +229,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 0, limit: 26, + options: highlightOptions, }); }); @@ -252,6 +268,16 @@ describe('PgSearchEngine', () => { }, type: 'my-type', rank: i + 1, + highlight: { + preTag: '', + postTag: '', + fields: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + path: '', + }, + }, })), nextPageCursor: 'MQ==', }); @@ -260,6 +286,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 0, limit: 26, + options: highlightOptions, }); }); @@ -300,6 +327,16 @@ describe('PgSearchEngine', () => { }, type: 'my-type', rank: i + 1, + highlight: { + preTag: '', + postTag: '', + fields: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + path: '', + }, + }, })) .slice(25), previousPageCursor: 'MA==', @@ -309,6 +346,7 @@ describe('PgSearchEngine', () => { pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', offset: 25, limit: 26, + options: highlightOptions, }); }); }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 764e530435..d0c1571991 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -46,7 +46,7 @@ export class PgSearchEngine implements SearchEngine { this.highlightOptions = { preTag: `<${uuidTag}>`, postTag: ``, - useHighlight: false, + useHighlight: true, maxWords: 35, minWords: 15, shortWord: 3, From fe664999df0312348f7d17745bbe22f13e8d96a0 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Sat, 18 Jun 2022 12:48:34 -0500 Subject: [PATCH 04/10] Added documentation about optional config Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- plugins/search-backend-module-pg/README.md | 28 ++++++++++++++++++++ plugins/search-backend-module-pg/config.d.ts | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/plugins/search-backend-module-pg/README.md b/plugins/search-backend-module-pg/README.md index d3f98132aa..49804a927c 100644 --- a/plugins/search-backend-module-pg/README.md +++ b/plugins/search-backend-module-pg/README.md @@ -14,3 +14,31 @@ other plugins. See [Backstage documentation](https://backstage.io/docs/features/search/search-engines#postgres) for details on how to setup Postgres based search for your Backstage instance. + +## Optional Configuration + +The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature: + +```yaml +search: + pg: + highlightOptions: + useHighlight: true # Used to enable to disable the highlight feature. The default value is true + maxWord: 35 # Used to set the longest headlines to output. The default value is 35. + minWord: 15 # Used to set the shortest headlines to output. The default value is 15. + shortWord: 3 # Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. The default value of three (3) eliminates common English articles. + highlightAll: false # If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + maxFragments: 0 # Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + fragmentDelimiter: ' ... ' # Delimiter string used to concatenate fragments. Defaults to " ... ". +``` + +**Note:** the highlight search term feature uses `ts_headline` which has been known to potentially impact performance. You only need this minimal config to disable it should you have issues: + +```yaml +search: + pg: + highlightOptions: + useHighlight: false +``` + +The Postgres documentation on [Highlighting Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE) has more details. diff --git a/plugins/search-backend-module-pg/config.d.ts b/plugins/search-backend-module-pg/config.d.ts index 6cdda27156..7367b770bc 100644 --- a/plugins/search-backend-module-pg/config.d.ts +++ b/plugins/search-backend-module-pg/config.d.ts @@ -48,7 +48,7 @@ export interface Config { */ highlightAll?: boolean; /** - * maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. + * Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. * A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). */ maxFragments?: number; From a678795d252c01b3101b1ab56bd04533081f4bf1 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Sat, 18 Jun 2022 12:54:28 -0500 Subject: [PATCH 05/10] Updated api-report Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .../search-backend-module-pg/api-report.md | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index 6a8b68a042..23e261ca4d 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -4,6 +4,7 @@ ```ts import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { Config } from '@backstage/config'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; @@ -43,7 +44,7 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit, preTag, postTag }: PgSearchQuery, + { types, pgTerm, fields, offset, limit, options }: PgSearchQuery, ): Promise; // (undocumented) static supported(knex: Knex): Promise; @@ -80,10 +81,15 @@ export interface DatabaseStore { // // @public (undocumented) export class PgSearchEngine implements SearchEngine { - constructor(databaseStore: DatabaseStore); + // Warning: (ae-forgotten-export) The symbol "PgSearchHighlightOptions" needs to be exported by the entry point index.d.ts + constructor( + databaseStore: DatabaseStore, + highlightOptions?: PgSearchHighlightOptions, + ); // (undocumented) static from(options: { database: PluginDatabaseManager; + config: Config; }): Promise; // (undocumented) getIndexer(type: string): Promise; @@ -95,8 +101,13 @@ export class PgSearchEngine implements SearchEngine { ): void; // (undocumented) static supported(database: PluginDatabaseManager): Promise; + // Warning: (ae-forgotten-export) The symbol "PgSearchHighlightConfig" needs to be exported by the entry point index.d.ts + // // (undocumented) - translator(query: SearchQuery): ConcretePgSearchQuery; + translator( + query: SearchQuery, + options: PgSearchHighlightConfig, + ): ConcretePgSearchQuery; } // Warning: (ae-missing-release-tag) "PgSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -132,12 +143,10 @@ export interface PgSearchQuery { // (undocumented) offset: number; // (undocumented) + options: PgSearchHighlightConfig; + // (undocumented) pgTerm?: string; // (undocumented) - postTag: string; - // (undocumented) - preTag: string; - // (undocumented) types?: string[]; } From 90aca31689a7ed778804a21013191dd59790cb81 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:29:45 -0500 Subject: [PATCH 06/10] Refactored config/options based on feedback Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- packages/backend/src/plugins/search.ts | 2 +- .../src/PgSearchEngine/PgSearchEngine.test.ts | 14 ++++-- .../src/PgSearchEngine/PgSearchEngine.ts | 50 +++++++++++-------- .../database/DatabaseDocumentStore.test.ts | 4 +- .../src/database/DatabaseDocumentStore.ts | 3 +- .../src/database/types.ts | 4 +- plugins/search-backend-module-pg/src/index.ts | 1 + plugins/search-backend-module-pg/src/types.ts | 29 ++++++----- 8 files changed, 65 insertions(+), 42 deletions(-) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index db92d79d71..51ab665026 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -39,7 +39,7 @@ async function createSearchEngine( } if (await PgSearchEngine.supported(env.database)) { - return await PgSearchEngine.from({ + return await PgSearchEngine.fromConfig({ database: env.database, config: env.config, }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index c1ed730053..5ad9d12897 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { ConfigReader } from '@backstage/config'; import { DatabaseStore } from '../database'; -import { PgSearchHighlightConfig } from '../types'; +import { PgSearchHighlightOptions } from '../types'; import { ConcretePgSearchQuery, decodePageCursor, @@ -23,7 +24,7 @@ import { } from './PgSearchEngine'; import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; -const highlightOptions: PgSearchHighlightConfig = { +const highlightOptions: PgSearchHighlightOptions = { preTag: '', postTag: '', useHighlight: true, @@ -47,6 +48,13 @@ describe('PgSearchEngine', () => { const tx: any = {} as any; let searchEngine: PgSearchEngine; let database: jest.Mocked; + const config = { + search: { + pg: { + highlightOptions, + }, + }, + }; beforeEach(() => { database = { @@ -57,7 +65,7 @@ describe('PgSearchEngine', () => { completeInsert: jest.fn(), prepareInsert: jest.fn(), }; - searchEngine = new PgSearchEngine(database); + searchEngine = new PgSearchEngine(database, new ConfigReader(config)); }); describe('translator', () => { diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index d0c1571991..b6e57552ba 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -28,7 +28,7 @@ import { } from '../database'; import { v4 as uuid } from 'uuid'; import { Config } from '@backstage/config'; -import { PgSearchHighlightConfig, PgSearchHighlightOptions } from '../types'; +import { PgSearchHighlightOptions, PgSearchOptions } from '../types'; export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; @@ -36,36 +36,46 @@ export type ConcretePgSearchQuery = { }; export class PgSearchEngine implements SearchEngine { - private readonly highlightOptions: PgSearchHighlightConfig; - - constructor( - private readonly databaseStore: DatabaseStore, - highlightOptions?: PgSearchHighlightOptions, - ) { + private readonly highlightOptions: PgSearchHighlightOptions; + constructor(private readonly databaseStore: DatabaseStore, config: Config) { const uuidTag = uuid(); - this.highlightOptions = { + const highlightConfig = config.getOptionalConfig( + 'search.pg.highlightOptions', + ); + + const highlightOptions: PgSearchHighlightOptions = { preTag: `<${uuidTag}>`, postTag: ``, - useHighlight: true, - maxWords: 35, - minWords: 15, - shortWord: 3, - highlightAll: false, - maxFragments: 0, - fragmentDelimiter: ' ... ', - ...highlightOptions, + useHighlight: highlightConfig?.getOptionalBoolean('useHighlight') ?? true, + maxWords: highlightConfig?.getOptionalNumber('maxWords') ?? 35, + minWords: highlightConfig?.getOptionalNumber('minWords') ?? 15, + shortWord: highlightConfig?.getOptionalNumber('shortWord') ?? 3, + highlightAll: + highlightConfig?.getOptionalBoolean('highlightAll') ?? false, + maxFragments: highlightConfig?.getOptionalNumber('maxFragments') ?? 0, + fragmentDelimiter: + highlightConfig?.getOptionalString('fragmentDelimiter') ?? ' ... ', }; + this.highlightOptions = highlightOptions; } + /** + * @deprecated This will be removed in a future release, please us fromConfig instead + */ static async from(options: { database: PluginDatabaseManager; config: Config; }): Promise { return new PgSearchEngine( await DatabaseDocumentStore.create(await options.database.getClient()), - options.config.getOptional( - 'search.pg.highlightOptions', - ), + options.config, + ); + } + + static async fromConfig({ config, database }: PgSearchOptions) { + return new PgSearchEngine( + await DatabaseDocumentStore.create(await database.getClient()), + config, ); } @@ -75,7 +85,7 @@ export class PgSearchEngine implements SearchEngine { translator( query: SearchQuery, - options: PgSearchHighlightConfig, + options: PgSearchHighlightOptions, ): ConcretePgSearchQuery { const pageSize = 25; const { page } = decodePageCursor(query.pageCursor); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index 693a3ea3c0..f4b3e6fb0b 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -15,10 +15,10 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { IndexableDocument } from '@backstage/plugin-search-common'; -import { PgSearchHighlightConfig } from '../types'; +import { PgSearchHighlightOptions } from '../types'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; -const highlightOptions: PgSearchHighlightConfig = { +const highlightOptions: PgSearchHighlightOptions = { preTag: '', postTag: '', useHighlight: false, diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 1695db3cf5..040145f7b8 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -132,8 +132,9 @@ export class DatabaseDocumentStore implements DatabaseStore { async query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit, options }: PgSearchQuery, + searchQuery: PgSearchQuery, ): Promise { + const { types, pgTerm, fields, offset, limit, options } = searchQuery; // Builds a query like: // SELECT ts_rank_cd(body, query) AS rank, type, document, // ts_headline('english', document, query) AS highlight diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index d572eba146..43bda32050 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -15,7 +15,7 @@ */ import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; -import { PgSearchHighlightConfig } from '../types'; +import { PgSearchHighlightOptions } from '../types'; export interface PgSearchQuery { fields?: Record; @@ -23,7 +23,7 @@ export interface PgSearchQuery { pgTerm?: string; offset: number; limit: number; - options: PgSearchHighlightConfig; + options: PgSearchHighlightOptions; } export interface DatabaseStore { diff --git a/plugins/search-backend-module-pg/src/index.ts b/plugins/search-backend-module-pg/src/index.ts index fbc52735dd..4cb54f9676 100644 --- a/plugins/search-backend-module-pg/src/index.ts +++ b/plugins/search-backend-module-pg/src/index.ts @@ -22,3 +22,4 @@ export * from './database'; export * from './PgSearchEngine'; +export type { PgSearchHighlightOptions } from './types'; diff --git a/plugins/search-backend-module-pg/src/types.ts b/plugins/search-backend-module-pg/src/types.ts index 9ad56b9b12..5e32c69e7a 100644 --- a/plugins/search-backend-module-pg/src/types.ts +++ b/plugins/search-backend-module-pg/src/types.ts @@ -14,6 +14,22 @@ * limitations under the License. */ +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; + +/** + * Options to instantiate PgSearchEngine + * @public + */ +export type PgSearchOptions = { + config: Config; + database: PluginDatabaseManager; +}; + +/** + * Options for highlighting search terms + * @public + */ export type PgSearchHighlightOptions = { useHighlight?: boolean; maxWords?: number; @@ -22,19 +38,6 @@ export type PgSearchHighlightOptions = { highlightAll?: boolean; maxFragments?: number; fragmentDelimiter?: string; -}; - -/** - * @public - */ -export type PgSearchHighlightConfig = { - useHighlight: boolean; - maxWords: number; - minWords: number; - shortWord: number; - highlightAll: boolean; - maxFragments: number; - fragmentDelimiter: string; preTag: string; postTag: string; }; From 8803c2f2dfc2fbea04cf8fe604cf1b8b0af1c7d2 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:57:55 -0500 Subject: [PATCH 07/10] Updated changeset and api-report Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .changeset/metal-windows-share.md | 2 + .../search-backend-module-pg/api-report.md | 40 ++++++++++++++----- plugins/search-backend-module-pg/src/index.ts | 2 +- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.changeset/metal-windows-share.md b/.changeset/metal-windows-share.md index 4fc568986a..0c5e6bb26e 100644 --- a/.changeset/metal-windows-share.md +++ b/.changeset/metal-windows-share.md @@ -2,4 +2,6 @@ '@backstage/plugin-search-backend-module-pg': patch --- +**DEPRECATED**: `PgSearchEngine` static `from` has been depracated and will be removed in a future release. Use static `fromConfig` method to instantiate. + Added support for highlighting matched terms in search result data diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index 23e261ca4d..0d03a83d34 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -44,7 +44,7 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit, options }: PgSearchQuery, + searchQuery: PgSearchQuery, ): Promise; // (undocumented) static supported(knex: Knex): Promise; @@ -81,17 +81,18 @@ export interface DatabaseStore { // // @public (undocumented) export class PgSearchEngine implements SearchEngine { - // Warning: (ae-forgotten-export) The symbol "PgSearchHighlightOptions" needs to be exported by the entry point index.d.ts - constructor( - databaseStore: DatabaseStore, - highlightOptions?: PgSearchHighlightOptions, - ); - // (undocumented) + constructor(databaseStore: DatabaseStore, config: Config); + // @deprecated (undocumented) static from(options: { database: PluginDatabaseManager; config: Config; }): Promise; // (undocumented) + static fromConfig({ + config, + database, + }: PgSearchOptions): Promise; + // (undocumented) getIndexer(type: string): Promise; // (undocumented) query(query: SearchQuery): Promise; @@ -101,12 +102,10 @@ export class PgSearchEngine implements SearchEngine { ): void; // (undocumented) static supported(database: PluginDatabaseManager): Promise; - // Warning: (ae-forgotten-export) The symbol "PgSearchHighlightConfig" needs to be exported by the entry point index.d.ts - // // (undocumented) translator( query: SearchQuery, - options: PgSearchHighlightConfig, + options: PgSearchHighlightOptions, ): ConcretePgSearchQuery; } @@ -132,6 +131,25 @@ export type PgSearchEngineIndexerOptions = { databaseStore: DatabaseStore; }; +// @public +export type PgSearchHighlightOptions = { + useHighlight?: boolean; + maxWords?: number; + minWords?: number; + shortWord?: number; + highlightAll?: boolean; + maxFragments?: number; + fragmentDelimiter?: string; + preTag: string; + postTag: string; +}; + +// @public +export type PgSearchOptions = { + config: Config; + database: PluginDatabaseManager; +}; + // Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -143,7 +161,7 @@ export interface PgSearchQuery { // (undocumented) offset: number; // (undocumented) - options: PgSearchHighlightConfig; + options: PgSearchHighlightOptions; // (undocumented) pgTerm?: string; // (undocumented) diff --git a/plugins/search-backend-module-pg/src/index.ts b/plugins/search-backend-module-pg/src/index.ts index 4cb54f9676..86184d1e4e 100644 --- a/plugins/search-backend-module-pg/src/index.ts +++ b/plugins/search-backend-module-pg/src/index.ts @@ -22,4 +22,4 @@ export * from './database'; export * from './PgSearchEngine'; -export type { PgSearchHighlightOptions } from './types'; +export type { PgSearchOptions, PgSearchHighlightOptions } from './types'; From cb62dd8a8a716a75add3388997cfb1bee2dd5d65 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:12:51 -0500 Subject: [PATCH 08/10] Fixed typo in changeset Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .changeset/metal-windows-share.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/metal-windows-share.md b/.changeset/metal-windows-share.md index 0c5e6bb26e..b4443a620f 100644 --- a/.changeset/metal-windows-share.md +++ b/.changeset/metal-windows-share.md @@ -2,6 +2,6 @@ '@backstage/plugin-search-backend-module-pg': patch --- -**DEPRECATED**: `PgSearchEngine` static `from` has been depracated and will be removed in a future release. Use static `fromConfig` method to instantiate. +**DEPRECATED**: `PgSearchEngine` static `from` has been deprecated and will be removed in a future release. Use static `fromConfig` method to instantiate. Added support for highlighting matched terms in search result data From a45d796451ca35984d1ed76e6221644e5c144981 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:03:12 -0500 Subject: [PATCH 09/10] Refactored types based on feedback Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- .../search-backend-module-pg/api-report.md | 21 ++++--- .../src/PgSearchEngine/PgSearchEngine.test.ts | 12 ++-- .../src/PgSearchEngine/PgSearchEngine.ts | 59 ++++++++++++++++--- .../src/PgSearchEngine/index.ts | 8 ++- .../database/DatabaseDocumentStore.test.ts | 2 +- .../src/database/types.ts | 2 +- plugins/search-backend-module-pg/src/index.ts | 1 - plugins/search-backend-module-pg/src/types.ts | 43 -------------- 8 files changed, 81 insertions(+), 67 deletions(-) delete mode 100644 plugins/search-backend-module-pg/src/types.ts diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index 0d03a83d34..cd47df837c 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -12,9 +12,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-backend-node'; import { SearchQuery } from '@backstage/plugin-search-common'; -// Warning: (ae-missing-release-tag) "ConcretePgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; pageSize: number; @@ -97,15 +95,13 @@ export class PgSearchEngine implements SearchEngine { // (undocumented) query(query: SearchQuery): Promise; // (undocumented) - setTranslator( - translator: (query: SearchQuery) => ConcretePgSearchQuery, - ): void; + setTranslator(translator: PgSearchQueryTranslator): void; // (undocumented) static supported(database: PluginDatabaseManager): Promise; // (undocumented) translator( query: SearchQuery, - options: PgSearchHighlightOptions, + options: PgSearchQueryTranslatorOptions, ): ConcretePgSearchQuery; } @@ -168,6 +164,17 @@ export interface PgSearchQuery { types?: string[]; } +// @public +export type PgSearchQueryTranslator = ( + query: SearchQuery, + options: PgSearchQueryTranslatorOptions, +) => ConcretePgSearchQuery; + +// @public +export type PgSearchQueryTranslatorOptions = { + highlightOptions: PgSearchHighlightOptions; +}; + // Warning: (ae-missing-release-tag) "RawDocumentRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 5ad9d12897..23f8983074 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -15,12 +15,12 @@ */ import { ConfigReader } from '@backstage/config'; import { DatabaseStore } from '../database'; -import { PgSearchHighlightOptions } from '../types'; import { ConcretePgSearchQuery, decodePageCursor, encodePageCursor, PgSearchEngine, + PgSearchHighlightOptions, } from './PgSearchEngine'; import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; @@ -86,7 +86,7 @@ describe('PgSearchEngine', () => { term: 'testTerm', filters: {}, }, - highlightOptions, + { highlightOptions }, ); }); @@ -96,7 +96,7 @@ describe('PgSearchEngine', () => { term: 'Hello', pageCursor: 'MQ==', }, - highlightOptions, + { highlightOptions }, ); expect(actualTranslatedQuery).toMatchObject({ @@ -114,7 +114,7 @@ describe('PgSearchEngine', () => { { term: 'Hello World', }, - highlightOptions, + { highlightOptions }, ); expect(actualTranslatedQuery).toMatchObject({ @@ -133,7 +133,7 @@ describe('PgSearchEngine', () => { term: 'H&e|l!l*o W\0o(r)l:d', pageCursor: '', }, - highlightOptions, + { highlightOptions }, ) as ConcretePgSearchQuery; expect(actualTranslatedQuery).toMatchObject({ @@ -151,7 +151,7 @@ describe('PgSearchEngine', () => { filters: { kind: 'testKind' }, types: ['my-filter'], }, - highlightOptions, + { highlightOptions }, ); expect(actualTranslatedQuery).toMatchObject({ diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index b6e57552ba..3f4d419fab 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -28,13 +28,58 @@ import { } from '../database'; import { v4 as uuid } from 'uuid'; import { Config } from '@backstage/config'; -import { PgSearchHighlightOptions, PgSearchOptions } from '../types'; +/** + * Search query that the Postgres search engine understands. + * @public + */ export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; pageSize: number; }; +/** + * Options available for the Postgres specific query translator. + * @public + */ +export type PgSearchQueryTranslatorOptions = { + highlightOptions: PgSearchHighlightOptions; +}; + +/** + * Postgres specific query translator. + * @public + */ +export type PgSearchQueryTranslator = ( + query: SearchQuery, + options: PgSearchQueryTranslatorOptions, +) => ConcretePgSearchQuery; + +/** + * Options to instantiate PgSearchEngine + * @public + */ +export type PgSearchOptions = { + config: Config; + database: PluginDatabaseManager; +}; + +/** + * Options for highlighting search terms + * @public + */ +export type PgSearchHighlightOptions = { + useHighlight?: boolean; + maxWords?: number; + minWords?: number; + shortWord?: number; + highlightAll?: boolean; + maxFragments?: number; + fragmentDelimiter?: string; + preTag: string; + postTag: string; +}; + export class PgSearchEngine implements SearchEngine { private readonly highlightOptions: PgSearchHighlightOptions; constructor(private readonly databaseStore: DatabaseStore, config: Config) { @@ -85,7 +130,7 @@ export class PgSearchEngine implements SearchEngine { translator( query: SearchQuery, - options: PgSearchHighlightOptions, + options: PgSearchQueryTranslatorOptions, ): ConcretePgSearchQuery { const pageSize = 25; const { page } = decodePageCursor(query.pageCursor); @@ -105,15 +150,13 @@ export class PgSearchEngine implements SearchEngine { types: query.types, offset, limit, - options, + options: options.highlightOptions, }, pageSize, }; } - setTranslator( - translator: (query: SearchQuery) => ConcretePgSearchQuery, - ): void { + setTranslator(translator: PgSearchQueryTranslator) { this.translator = translator; } @@ -126,7 +169,9 @@ export class PgSearchEngine implements SearchEngine { } async query(query: SearchQuery): Promise { - const { pgQuery, pageSize } = this.translator(query, this.highlightOptions); + const { pgQuery, pageSize } = this.translator(query, { + highlightOptions: this.highlightOptions, + }); const rows = await this.databaseStore.transaction(async tx => this.databaseStore.query(tx, pgQuery), diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index 7f8e297648..e9ea04e5c4 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -14,7 +14,13 @@ * limitations under the License. */ export { PgSearchEngine } from './PgSearchEngine'; -export type { ConcretePgSearchQuery } from './PgSearchEngine'; +export type { + ConcretePgSearchQuery, + PgSearchQueryTranslatorOptions, + PgSearchQueryTranslator, + PgSearchOptions, + PgSearchHighlightOptions, +} from './PgSearchEngine'; export type { PgSearchEngineIndexer, PgSearchEngineIndexerOptions, diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index f4b3e6fb0b..355e41352d 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -15,7 +15,7 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { IndexableDocument } from '@backstage/plugin-search-common'; -import { PgSearchHighlightOptions } from '../types'; +import { PgSearchHighlightOptions } from '../PgSearchEngine'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; const highlightOptions: PgSearchHighlightOptions = { diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 43bda32050..151be7d13f 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -15,7 +15,7 @@ */ import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; -import { PgSearchHighlightOptions } from '../types'; +import { PgSearchHighlightOptions } from '../PgSearchEngine'; export interface PgSearchQuery { fields?: Record; diff --git a/plugins/search-backend-module-pg/src/index.ts b/plugins/search-backend-module-pg/src/index.ts index 86184d1e4e..fbc52735dd 100644 --- a/plugins/search-backend-module-pg/src/index.ts +++ b/plugins/search-backend-module-pg/src/index.ts @@ -22,4 +22,3 @@ export * from './database'; export * from './PgSearchEngine'; -export type { PgSearchOptions, PgSearchHighlightOptions } from './types'; diff --git a/plugins/search-backend-module-pg/src/types.ts b/plugins/search-backend-module-pg/src/types.ts deleted file mode 100644 index 5e32c69e7a..0000000000 --- a/plugins/search-backend-module-pg/src/types.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PluginDatabaseManager } from '@backstage/backend-common'; -import { Config } from '@backstage/config'; - -/** - * Options to instantiate PgSearchEngine - * @public - */ -export type PgSearchOptions = { - config: Config; - database: PluginDatabaseManager; -}; - -/** - * Options for highlighting search terms - * @public - */ -export type PgSearchHighlightOptions = { - useHighlight?: boolean; - maxWords?: number; - minWords?: number; - shortWord?: number; - highlightAll?: boolean; - maxFragments?: number; - fragmentDelimiter?: string; - preTag: string; - postTag: string; -}; From e52f7801b0ff54d8cea619bb9219b5020d96b1ba Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Tue, 28 Jun 2022 07:34:57 -0500 Subject: [PATCH 10/10] Adjustments based on feedback Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- docs/features/search/search-engines.md | 30 ++++++++++++++++++- packages/backend/src/plugins/search.ts | 3 +- .../search-backend-module-pg/api-report.md | 10 +++---- .../src/PgSearchEngine/PgSearchEngine.ts | 9 ++++-- .../src/database/DatabaseDocumentStore.ts | 1 + 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index a5ff780c8a..ecd8a85109 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -59,10 +59,38 @@ configured and make the following changes to your backend: // Initialize a connection to a search engine. const searchEngine = (await PgSearchEngine.supported(env.database)) - ? await PgSearchEngine.from({ database: env.database }) + ? await PgSearchEngine.fromConfig(env.config, { database: env.database }) : new LunrSearchEngine({ logger: env.logger }); ``` +## Optional Configuration + +The following is an example of the optional configuration that can be applied when using Postgres as the search backend. Currently this is mostly for just the highlight feature: + +```yaml +search: + pg: + highlightOptions: + useHighlight: true # Used to enable to disable the highlight feature. The default value is true + maxWord: 35 # Used to set the longest headlines to output. The default value is 35. + minWord: 15 # Used to set the shortest headlines to output. The default value is 15. + shortWord: 3 # Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. The default value of three (3) eliminates common English articles. + highlightAll: false # If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + maxFragments: 0 # Maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + fragmentDelimiter: ' ... ' # Delimiter string used to concatenate fragments. Defaults to " ... ". +``` + +**Note:** the highlight search term feature uses `ts_headline` which has been known to potentially impact performance. You only need this minimal config to disable it should you have issues: + +```yaml +search: + pg: + highlightOptions: + useHighlight: false +``` + +The Postgres documentation on [Highlighting Results](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE) has more details. + ## ElasticSearch Backstage supports ElasticSearch search engine connections, indexing and diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 51ab665026..c478ecdd05 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -39,9 +39,8 @@ async function createSearchEngine( } if (await PgSearchEngine.supported(env.database)) { - return await PgSearchEngine.fromConfig({ + return await PgSearchEngine.fromConfig(env.config, { database: env.database, - config: env.config, }); } diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index cd47df837c..f7a188a14e 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -79,6 +79,7 @@ export interface DatabaseStore { // // @public (undocumented) export class PgSearchEngine implements SearchEngine { + // @deprecated constructor(databaseStore: DatabaseStore, config: Config); // @deprecated (undocumented) static from(options: { @@ -86,10 +87,10 @@ export class PgSearchEngine implements SearchEngine { config: Config; }): Promise; // (undocumented) - static fromConfig({ - config, - database, - }: PgSearchOptions): Promise; + static fromConfig( + config: Config, + options: PgSearchOptions, + ): Promise; // (undocumented) getIndexer(type: string): Promise; // (undocumented) @@ -142,7 +143,6 @@ export type PgSearchHighlightOptions = { // @public export type PgSearchOptions = { - config: Config; database: PluginDatabaseManager; }; diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 3f4d419fab..abf7cdbf5c 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -60,7 +60,6 @@ export type PgSearchQueryTranslator = ( * @public */ export type PgSearchOptions = { - config: Config; database: PluginDatabaseManager; }; @@ -82,6 +81,10 @@ export type PgSearchHighlightOptions = { export class PgSearchEngine implements SearchEngine { private readonly highlightOptions: PgSearchHighlightOptions; + + /** + * @deprecated This will be marked as private in a future release, please us fromConfig instead + */ constructor(private readonly databaseStore: DatabaseStore, config: Config) { const uuidTag = uuid(); const highlightConfig = config.getOptionalConfig( @@ -117,9 +120,9 @@ export class PgSearchEngine implements SearchEngine { ); } - static async fromConfig({ config, database }: PgSearchOptions) { + static async fromConfig(config: Config, options: PgSearchOptions) { return new PgSearchEngine( - await DatabaseDocumentStore.create(await database.getClient()), + await DatabaseDocumentStore.create(await options.database.getClient()), config, ); } diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 040145f7b8..8e73c0f16f 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -135,6 +135,7 @@ export class DatabaseDocumentStore implements DatabaseStore { searchQuery: PgSearchQuery, ): Promise { const { types, pgTerm, fields, offset, limit, options } = searchQuery; + // TODO(awanlin): We should make the language a parameter so that we can support more then just english // Builds a query like: // SELECT ts_rank_cd(body, query) AS rank, type, document, // ts_headline('english', document, query) AS highlight