From d43228a1832118dfa2b1e60c6e37c9d8c6568dc5 Mon Sep 17 00:00:00 2001 From: Andre Wanlin <67169551+awanlin@users.noreply.github.com> Date: Sat, 18 Jun 2022 07:39:23 -0500 Subject: [PATCH] Refactored to support config with options Signed-off-by: Andre Wanlin <67169551+awanlin@users.noreply.github.com> --- packages/backend/src/plugins/search.ts | 5 +- plugins/search-backend-module-pg/config.d.ts | 62 +++++++++++++++++++ plugins/search-backend-module-pg/package.json | 7 ++- .../src/PgSearchEngine/PgSearchEngine.test.ts | 57 ++++++++++++----- .../src/PgSearchEngine/PgSearchEngine.ts | 42 ++++++++++--- .../database/DatabaseDocumentStore.test.ts | 34 +++++----- .../src/database/DatabaseDocumentStore.ts | 11 +++- .../src/database/types.ts | 4 +- plugins/search-backend-module-pg/src/types.ts | 40 ++++++++++++ 9 files changed, 216 insertions(+), 46 deletions(-) create mode 100644 plugins/search-backend-module-pg/config.d.ts create mode 100644 plugins/search-backend-module-pg/src/types.ts diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index a0ea83ab61..db92d79d71 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -39,7 +39,10 @@ async function createSearchEngine( } if (await PgSearchEngine.supported(env.database)) { - return await PgSearchEngine.from({ database: env.database }); + return await PgSearchEngine.from({ + database: env.database, + config: env.config, + }); } return new LunrSearchEngine({ logger: env.logger }); diff --git a/plugins/search-backend-module-pg/config.d.ts b/plugins/search-backend-module-pg/config.d.ts new file mode 100644 index 0000000000..6cdda27156 --- /dev/null +++ b/plugins/search-backend-module-pg/config.d.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the search plugin */ + search?: { + /** + * Options for PG + */ + pg?: { + /** + * Options for configuring highlight settings + * See https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE + */ + highlightOptions?: { + /** + * Used to enable to disable the highlight feature. The default value is true + */ + useHighlight?: boolean; + /** + * Used to set the longest headlines to output. The default value is 35. + */ + maxWords?: number; + /** + * Used to set the shortest headlines to output. The default value is 15. + */ + minWords?: number; + /** + * Words of this length or less will be dropped at the start and end of a headline, unless they are query terms. + * The default value of three (3) eliminates common English articles. + */ + shortWord?: number; + /** + * If true the whole document will be used as the headline, ignoring the preceding three parameters. The default is false. + */ + highlightAll?: boolean; + /** + * maximum number of text fragments to display. The default value of zero selects a non-fragment-based headline generation method. + * A value greater than zero selects fragment-based headline generation (see the linked documentation above for more details). + */ + maxFragments?: number; + /** + * Delimiter string used to concatenate fragments. Defaults to " ... ". + */ + fragmentDelimiter?: string; + }; + }; + }; +} diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json index 7809f9dd06..16367b9ef1 100644 --- a/plugins/search-backend-module-pg/package.json +++ b/plugins/search-backend-module-pg/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.14.1-next.1", + "@backstage/config": "^1.0.1", "@backstage/plugin-search-backend-node": "^0.6.3-next.1", "@backstage/plugin-search-common": "^0.3.6-next.0", "lodash": "^4.17.21", @@ -36,6 +37,8 @@ }, "files": [ "dist", - "migrations" - ] + "migrations", + "config.d.ts" + ], + "configSchema": "config.d.ts" } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 5f3bc3c5d8..1d0e31757f 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import { DatabaseStore } from '../database'; +import { PgSearchHighlightConfig } from '../types'; import { ConcretePgSearchQuery, decodePageCursor, @@ -22,6 +23,18 @@ import { } from './PgSearchEngine'; import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; +const highlightOptions: PgSearchHighlightConfig = { + preTag: '', + postTag: '', + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', +}; + jest.mock('./PgSearchEngineIndexer', () => ({ PgSearchEngineIndexer: jest .fn() @@ -65,10 +78,13 @@ describe('PgSearchEngine', () => { }); it('should pass page cursor', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'Hello', - pageCursor: 'MQ==', - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'Hello', + pageCursor: 'MQ==', + }, + highlightOptions, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -81,9 +97,12 @@ describe('PgSearchEngine', () => { }); it('should return translated query term', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'Hello World', - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'Hello World', + }, + highlightOptions, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -96,10 +115,13 @@ describe('PgSearchEngine', () => { }); it('should sanitize query term', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'H&e|l!l*o W\0o(r)l:d', - pageCursor: '', - }) as ConcretePgSearchQuery; + const actualTranslatedQuery = searchEngine.translator( + { + term: 'H&e|l!l*o W\0o(r)l:d', + pageCursor: '', + }, + highlightOptions, + ) as ConcretePgSearchQuery; expect(actualTranslatedQuery).toMatchObject({ pgQuery: { @@ -110,11 +132,14 @@ describe('PgSearchEngine', () => { }); it('should return translated query with filters', async () => { - const actualTranslatedQuery = searchEngine.translator({ - term: 'testTerm', - filters: { kind: 'testKind' }, - types: ['my-filter'], - }); + const actualTranslatedQuery = searchEngine.translator( + { + term: 'testTerm', + filters: { kind: 'testKind' }, + types: ['my-filter'], + }, + highlightOptions, + ); expect(actualTranslatedQuery).toMatchObject({ pgQuery: { diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 77aa003c5d..764e530435 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -27,6 +27,8 @@ import { PgSearchQuery, } from '../database'; import { v4 as uuid } from 'uuid'; +import { Config } from '@backstage/config'; +import { PgSearchHighlightConfig, PgSearchHighlightOptions } from '../types'; export type ConcretePgSearchQuery = { pgQuery: PgSearchQuery; @@ -34,13 +36,36 @@ export type ConcretePgSearchQuery = { }; export class PgSearchEngine implements SearchEngine { - constructor(private readonly databaseStore: DatabaseStore) {} + private readonly highlightOptions: PgSearchHighlightConfig; + + constructor( + private readonly databaseStore: DatabaseStore, + highlightOptions?: PgSearchHighlightOptions, + ) { + const uuidTag = uuid(); + this.highlightOptions = { + preTag: `<${uuidTag}>`, + postTag: ``, + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', + ...highlightOptions, + }; + } static async from(options: { database: PluginDatabaseManager; + config: Config; }): Promise { return new PgSearchEngine( await DatabaseDocumentStore.create(await options.database.getClient()), + options.config.getOptional( + 'search.pg.highlightOptions', + ), ); } @@ -48,13 +73,15 @@ export class PgSearchEngine implements SearchEngine { return await DatabaseDocumentStore.supported(await database.getClient()); } - translator(query: SearchQuery): ConcretePgSearchQuery { + translator( + query: SearchQuery, + options: PgSearchHighlightConfig, + ): ConcretePgSearchQuery { const pageSize = 25; const { page } = decodePageCursor(query.pageCursor); const offset = page * pageSize; // We request more result to know whether there is another page const limit = pageSize + 1; - const uuidTag = uuid(); return { pgQuery: { @@ -68,8 +95,7 @@ export class PgSearchEngine implements SearchEngine { types: query.types, offset, limit, - preTag: `<${uuidTag}>`, - postTag: ``, + options, }, pageSize, }; @@ -90,7 +116,7 @@ export class PgSearchEngine implements SearchEngine { } async query(query: SearchQuery): Promise { - const { pgQuery, pageSize } = this.translator(query); + const { pgQuery, pageSize } = this.translator(query, this.highlightOptions); const rows = await this.databaseStore.transaction(async tx => this.databaseStore.query(tx, pgQuery), @@ -115,8 +141,8 @@ export class PgSearchEngine implements SearchEngine { document, rank: page * pageSize + index + 1, highlight: { - preTag: pgQuery.preTag, - postTag: pgQuery.postTag, + preTag: pgQuery.options.preTag, + postTag: pgQuery.options.postTag, fields: highlight ? { text: highlight.text, diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index e2303d90b2..693a3ea3c0 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -15,8 +15,21 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { IndexableDocument } from '@backstage/plugin-search-common'; +import { PgSearchHighlightConfig } from '../types'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; +const highlightOptions: PgSearchHighlightConfig = { + preTag: '', + postTag: '', + useHighlight: false, + maxWords: 35, + minWords: 15, + shortWord: 3, + highlightAll: false, + maxFragments: 0, + fragmentDelimiter: ' ... ', +}; + describe('DatabaseDocumentStore', () => { describe('unsupported', () => { const databases = TestDatabases.create({ @@ -224,8 +237,7 @@ describe('DatabaseDocumentStore', () => { pgTerm: 'Hello & World', offset: 1, limit: 1, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -271,8 +283,7 @@ describe('DatabaseDocumentStore', () => { pgTerm: 'Hello & World', offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -334,8 +345,7 @@ describe('DatabaseDocumentStore', () => { types: ['my-type'], offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -389,8 +399,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -445,8 +454,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: ['this', 'that'] }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -508,8 +516,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this', otherField: 'another' }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); @@ -559,8 +566,7 @@ describe('DatabaseDocumentStore', () => { fields: { myField: 'this' }, offset: 0, limit: 25, - preTag: '', - postTag: '', + options: highlightOptions, }), ); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 3a1ec128ef..1695db3cf5 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -132,7 +132,7 @@ export class DatabaseDocumentStore implements DatabaseStore { async query( tx: Knex.Transaction, - { types, pgTerm, fields, offset, limit, preTag, postTag }: PgSearchQuery, + { types, pgTerm, fields, offset, limit, options }: PgSearchQuery, ): Promise { // Builds a query like: // SELECT ts_rank_cd(body, query) AS rank, type, document, @@ -171,15 +171,20 @@ export class DatabaseDocumentStore implements DatabaseStore { query.select('type', 'document'); - if (pgTerm) { + if (pgTerm && options.useHighlight) { + const headlineOptions = `MaxWords=${options.maxWords}, MinWords=${options.minWords}, ShortWord=${options.shortWord}, HighlightAll=${options.highlightAll}, MaxFragments=${options.maxFragments}, FragmentDelimiter=${options.fragmentDelimiter}, StartSel=${options.preTag}, StopSel=${options.postTag}`; query .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) .select( tx.raw( - `ts_headline(\'english\', document, query, 'StartSel=${preTag}, StopSel=${postTag}') as "highlight"`, + `ts_headline(\'english\', document, query, '${headlineOptions}') as "highlight"`, ), ) .orderBy('rank', 'desc'); + } else if (pgTerm && !options.useHighlight) { + query + .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .orderBy('rank', 'desc'); } else { query.select(tx.raw('1 as rank')); } diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 23679d100f..d572eba146 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -15,6 +15,7 @@ */ import { IndexableDocument } from '@backstage/plugin-search-common'; import { Knex } from 'knex'; +import { PgSearchHighlightConfig } from '../types'; export interface PgSearchQuery { fields?: Record; @@ -22,8 +23,7 @@ export interface PgSearchQuery { pgTerm?: string; offset: number; limit: number; - preTag: string; - postTag: string; + options: PgSearchHighlightConfig; } export interface DatabaseStore { diff --git a/plugins/search-backend-module-pg/src/types.ts b/plugins/search-backend-module-pg/src/types.ts new file mode 100644 index 0000000000..9ad56b9b12 --- /dev/null +++ b/plugins/search-backend-module-pg/src/types.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type PgSearchHighlightOptions = { + useHighlight?: boolean; + maxWords?: number; + minWords?: number; + shortWord?: number; + highlightAll?: boolean; + maxFragments?: number; + fragmentDelimiter?: string; +}; + +/** + * @public + */ +export type PgSearchHighlightConfig = { + useHighlight: boolean; + maxWords: number; + minWords: number; + shortWord: number; + highlightAll: boolean; + maxFragments: number; + fragmentDelimiter: string; + preTag: string; + postTag: string; +};