From 9255e143099beaabbea637c1c51c69b7d6bfe0ef Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Mon, 2 Aug 2021 11:17:25 +0200 Subject: [PATCH] Add postgres based `SearchEngine` Signed-off-by: Oliver Sand --- .changeset/search-early-horses-sip.md | 6 + packages/backend/package.json | 1 + packages/backend/src/plugins/search.ts | 10 +- plugins/search-backend-module-pg/.eslintrc.js | 3 + plugins/search-backend-module-pg/README.md | 28 + .../migrations/20210311110000_init.js | 52 ++ .../migrations/20210727180000_delta_update.js | 45 ++ plugins/search-backend-module-pg/package.json | 36 ++ .../src/PgSearchEngine/PgSearchEngine.test.ts | 164 ++++++ .../src/PgSearchEngine/PgSearchEngine.ts | 92 ++++ .../src/PgSearchEngine/index.ts | 16 + .../database/DatabaseDocumentStore.test.ts | 512 ++++++++++++++++++ .../src/database/DatabaseDocumentStore.ts | 179 ++++++ .../src/database/index.ts | 17 + .../src/database/types.ts | 49 ++ .../src/database/util.test.ts | 62 +++ .../src/database/util.ts | 28 + plugins/search-backend-module-pg/src/index.ts | 17 + .../src/setupTests.ts | 16 + 19 files changed, 1330 insertions(+), 3 deletions(-) create mode 100644 .changeset/search-early-horses-sip.md create mode 100644 plugins/search-backend-module-pg/.eslintrc.js create mode 100644 plugins/search-backend-module-pg/README.md create mode 100644 plugins/search-backend-module-pg/migrations/20210311110000_init.js create mode 100644 plugins/search-backend-module-pg/migrations/20210727180000_delta_update.js create mode 100644 plugins/search-backend-module-pg/package.json create mode 100644 plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts create mode 100644 plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts create mode 100644 plugins/search-backend-module-pg/src/PgSearchEngine/index.ts create mode 100644 plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts create mode 100644 plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts create mode 100644 plugins/search-backend-module-pg/src/database/index.ts create mode 100644 plugins/search-backend-module-pg/src/database/types.ts create mode 100644 plugins/search-backend-module-pg/src/database/util.test.ts create mode 100644 plugins/search-backend-module-pg/src/database/util.ts create mode 100644 plugins/search-backend-module-pg/src/index.ts create mode 100644 plugins/search-backend-module-pg/src/setupTests.ts diff --git a/.changeset/search-early-horses-sip.md b/.changeset/search-early-horses-sip.md new file mode 100644 index 0000000000..ce5612ae86 --- /dev/null +++ b/.changeset/search-early-horses-sip.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Add `plugin-search-backend-module-pg` providing a postgres based search engine. +See the [README of `search-backend-module-pg`](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-pg/README.md) for usage instructions. diff --git a/packages/backend/package.json b/packages/backend/package.json index 5d4ff60d68..33221bde01 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -48,6 +48,7 @@ "@backstage/plugin-search-backend": "^0.2.3", "@backstage/plugin-search-backend-node": "^0.4.0", "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.1", + "@backstage/plugin-search-backend-module-pg": "^0.1.0", "@backstage/plugin-techdocs-backend": "^0.9.0", "@backstage/plugin-todo-backend": "^0.1.8", "@gitbeaker/node": "^30.2.0", diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 78ce09ee07..ac82e22093 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -14,20 +14,22 @@ * limitations under the License. */ import { useHotCleanup } from '@backstage/backend-common'; +import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; import { createRouter } from '@backstage/plugin-search-backend'; +import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; import { IndexBuilder, LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; -import { PluginEnvironment } from '../types'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; -import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { PluginEnvironment } from '../types'; export default async function createPlugin({ logger, discovery, config, + database, }: PluginEnvironment) { // Initialize a connection to a search engine. const searchEngine = config.has('search.elasticsearch') @@ -35,6 +37,8 @@ export default async function createPlugin({ logger, config, }) + : (await PgSearchEngine.supported(database)) + ? await PgSearchEngine.from({ database }) : new LunrSearchEngine({ logger }); const indexBuilder = new IndexBuilder({ logger, searchEngine }); diff --git a/plugins/search-backend-module-pg/.eslintrc.js b/plugins/search-backend-module-pg/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/search-backend-module-pg/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/search-backend-module-pg/README.md b/plugins/search-backend-module-pg/README.md new file mode 100644 index 0000000000..ae069ad848 --- /dev/null +++ b/plugins/search-backend-module-pg/README.md @@ -0,0 +1,28 @@ +# search-backend-module-pg + +This plugin provides an easy to use `SearchEngine` implementation to use with the +`@backstage/plugin-search-backend` based on Postgres. +Therefore it targets setups that want to avoid maintaining another external +service like elastic search. The search provides decent results and performs +well with ten thousands of indexed documents. +The connection to postgres is established via the database manager also used by +other plugins. + +> **Important**: The search plugin requires at least Postgres 11! + +## Setup + +To use the `SearchEngine`, make sure that you have a Postgres database +configured and make the following changes to your backend: + +1. Add a dependency on `@backstage/plugin-search-backend-module-pg` to your backend's `package.json`. +2. Initialize the search engine. It is recommended to initialize it with a fallback to the lunr search engine if you are running Backstage for development locally with SQLite: + +```typescript +// In packages/backend/src/plugins/search.ts + +// Initialize a connection to a search engine. +const searchEngine = (await PgSearchEngine.supported(database)) + ? await PgSearchEngine.from({ database }) + : new LunrSearchEngine({ logger }); +``` diff --git a/plugins/search-backend-module-pg/migrations/20210311110000_init.js b/plugins/search-backend-module-pg/migrations/20210311110000_init.js new file mode 100644 index 0000000000..8eaf86d2b4 --- /dev/null +++ b/plugins/search-backend-module-pg/migrations/20210311110000_init.js @@ -0,0 +1,52 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.createTable('documents', table => { + table.comment('The table of documents'); + table + .text('type') + .notNullable() + .index() + .comment('The index type of the document'); + table.jsonb('document').notNullable().comment('The document'); + table.specificType( + 'body', + 'tsvector NOT NULL GENERATED ALWAYS AS (' + + "setweight(to_tsvector('english', document->>'title'), 'A') || " + + "setweight(to_tsvector('english', document->>'text'), 'B') || " + + "setweight(to_tsvector('english', document - 'location' - 'title' - 'text'), 'C')" + + ') STORED', + ); + + table + .index(['body'], 'documents_body_index', 'GIN') + .comment('Optimize full text queries on documents'); + table + .index(['document'], 'documents_document_index', 'GIN') + .comment('Optimize filter queries on documents'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.dropTable('documents'); +}; diff --git a/plugins/search-backend-module-pg/migrations/20210727180000_delta_update.js b/plugins/search-backend-module-pg/migrations/20210727180000_delta_update.js new file mode 100644 index 0000000000..9ba4649b65 --- /dev/null +++ b/plugins/search-backend-module-pg/migrations/20210727180000_delta_update.js @@ -0,0 +1,45 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + await knex.schema.alterTable('documents', table => { + // Extend the documents table with a column that allows to check whether a + // document with the same content is already indexed. + table.specificType('hash', 'bytea'); + }); + + await knex('documents').update({ + hash: knex.raw( + "sha256(replace(document::text || type, '\\', '\\\\')::bytea)", + ), + }); + + await knex.schema.alterTable('documents', table => { + table.specificType('hash', 'bytea').notNullable().primary().alter(); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('documents', table => { + table.dropColumn('hash'); + }); +}; diff --git a/plugins/search-backend-module-pg/package.json b/plugins/search-backend-module-pg/package.json new file mode 100644 index 0000000000..9463e5c5a9 --- /dev/null +++ b/plugins/search-backend-module-pg/package.json @@ -0,0 +1,36 @@ +{ + "name": "@backstage/plugin-search-backend-module-pg", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.8.7", + "@backstage/search-common": "^0.1.2", + "@backstage/plugin-search-backend-node": "^0.4.0", + "lodash": "^4.17.15", + "knex": "^0.95.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "^0.1.4", + "@backstage/cli": "^0.7.6" + }, + "files": [ + "dist", + "migrations" + ] +} diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts new file mode 100644 index 0000000000..49fb169bf8 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -0,0 +1,164 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { range } from 'lodash'; +import { DatabaseStore, PgSearchQuery } from '../database'; +import { PgSearchEngine } from './PgSearchEngine'; + +describe('PgSearchEngine', () => { + const tx: any = {} as any; + let searchEngine: PgSearchEngine; + let database: jest.Mocked; + + beforeEach(() => { + database = { + transaction: jest.fn(), + insertDocuments: jest.fn(), + query: jest.fn(), + completeInsert: jest.fn(), + prepareInsert: jest.fn(), + }; + searchEngine = new PgSearchEngine(database); + }); + + describe('translator', () => { + it('query translator invoked', async () => { + database.transaction.mockResolvedValue([]); + const translatorSpy = jest.fn().mockReturnValue({ + pgSearchTerm: 'testTerm', + }); + searchEngine.setTranslator(translatorSpy); + + await searchEngine.query({ + term: 'testTerm', + filters: {}, + pageCursor: '', + }); + + expect(translatorSpy).toHaveBeenCalledWith({ + term: 'testTerm', + filters: {}, + pageCursor: '', + }); + }); + + it('should return translated query term', async () => { + const actualTranslatedQuery = searchEngine.translator({ + term: 'Hello World', + pageCursor: '', + }) as PgSearchQuery; + + expect(actualTranslatedQuery).toMatchObject({ + pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', + }); + }); + + it('should return translated query with filters', async () => { + const actualTranslatedQuery = searchEngine.translator({ + term: 'testTerm', + filters: { kind: 'testKind' }, + types: ['my-filter'], + pageCursor: '', + }) as PgSearchQuery; + + expect(actualTranslatedQuery).toMatchObject({ + pgTerm: '("testTerm" | "testTerm":*)', + fields: { kind: 'testKind' }, + types: ['my-filter'], + }); + }); + }); + + describe('insert', () => { + it('should insert documents', async () => { + database.transaction.mockImplementation(fn => fn(tx)); + + const documents = [ + { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, + { + location: 'location-2', + text: 'Hello World', + title: 'Dolor sit amet', + }, + ]; + + await searchEngine.index('my-type', documents); + + expect(database.transaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledWith( + tx, + 'my-type', + documents, + ); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + }); + + it('should batch insert documents', async () => { + database.transaction.mockImplementation(fn => fn(tx)); + + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await searchEngine.index('my-type', documents); + + expect(database.transaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toBeCalledTimes(4); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + }); + }); + + describe('query', () => { + it('should perform query', async () => { + database.transaction.mockImplementation(fn => fn(tx)); + database.query.mockResolvedValue([ + { + document: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, + type: 'my-type', + }, + ]); + + const results = await searchEngine.query({ + term: 'Hello World', + pageCursor: '', + }); + + expect(results).toEqual({ + results: [ + { + document: { + title: 'Hello World', + text: 'Lorem Ipsum', + location: 'location-1', + }, + type: 'my-type', + }, + ], + }); + expect(database.transaction).toHaveBeenCalledTimes(1); + expect(database.query).toHaveBeenCalledWith(tx, { + pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)', + }); + }); + }); +}); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts new file mode 100644 index 0000000000..0b052a499d --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { SearchEngine } from '@backstage/plugin-search-backend-node'; +import { + IndexableDocument, + SearchQuery, + SearchResultSet, +} from '@backstage/search-common'; +import { chunk } from 'lodash'; +import { + DatabaseDocumentStore, + DatabaseStore, + PgSearchQuery, +} from '../database'; + +// TODO: Support paging using page cursor (return cursor and parse cursor) + +export class PgSearchEngine implements SearchEngine { + constructor(private readonly databaseStore: DatabaseStore) {} + + static async from({ + database, + }: { + database: PluginDatabaseManager; + }): Promise { + return new PgSearchEngine( + await DatabaseDocumentStore.create(await database.getClient()), + ); + } + + static async supported(database: PluginDatabaseManager): Promise { + return await DatabaseDocumentStore.supported(await database.getClient()); + } + + translator(query: SearchQuery): PgSearchQuery { + return { + pgTerm: query.term + .split(/\s/) + .map(p => p.trim()) + .filter(p => p !== '') + .map(p => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`) + .join('&'), + fields: query.filters as Record, + types: query.types, + }; + } + + setTranslator(translator: (query: SearchQuery) => PgSearchQuery): void { + this.translator = translator; + } + + async index(type: string, documents: IndexableDocument[]): Promise { + await this.databaseStore.transaction(async tx => { + await this.databaseStore.prepareInsert(tx); + + const batchSize = 100; + for (const documentBatch of chunk(documents, batchSize)) { + await this.databaseStore.insertDocuments(tx, type, documentBatch); + } + + await this.databaseStore.completeInsert(tx, type); + }); + } + + async query(query: SearchQuery): Promise { + const pgQuery = this.translator(query); + + const rows = await this.databaseStore.transaction(async tx => + this.databaseStore.query(tx, pgQuery), + ); + const results = rows.map(({ type, document }) => ({ + type, + document, + })); + + return { results }; + } +} diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts new file mode 100644 index 0000000000..b28e03ee5c --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { PgSearchEngine } from './PgSearchEngine'; diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts new file mode 100644 index 0000000000..6f5075f112 --- /dev/null +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -0,0 +1,512 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { IndexableDocument } from '@backstage/search-common'; +import { DatabaseDocumentStore } from './DatabaseDocumentStore'; + +describe('DatabaseDocumentStore', () => { + describe('unsupported', () => { + const databases = TestDatabases.create({ + ids: ['MYSQL_8', 'POSTGRES_9', 'SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + 'should return support state, %p', + async databaseId => { + const knex = await databases.init(databaseId); + const supported = await DatabaseDocumentStore.supported(knex); + + expect(supported).toBe(false); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should fail to create, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await expect( + async () => await DatabaseDocumentStore.create(knex), + ).rejects.toThrow(); + }, + 60_000, + ); + }); + + describe('supported', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13'], + }); + + async function createStore(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + const store = await DatabaseDocumentStore.create(knex); + return { store, knex }; + } + + if (databases.eachSupportedId().length < 1) { + // Only execute tests if at least on database engine is available, e.g. if + // not in CI=1. it.each doesn't support an empty array. + return; + } + + it.each(databases.eachSupportedId())( + 'should return support state, %p', + async databaseId => { + const knex = await databases.init(databaseId); + const supported = await DatabaseDocumentStore.supported(knex); + + expect(supported).toBe(true); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should insert documents, %p', + async databaseId => { + const { store, knex } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '2' }]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should insert documents in batches, %p', + async databaseId => { + const { store, knex } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', + }, + ]); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 3', + text: 'TEXT 3', + location: 'LOCATION-3', + }, + { + title: 'TITLE 4', + text: 'TEXT 4', + location: 'LOCATION-4', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '4' }]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should clear index for type, %p', + async databaseId => { + const { store, knex } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ + { + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'test'); + }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 1', + text: 'TEXT 1', + location: 'LOCATION-1', + }, + { + title: 'TITLE 2', + text: 'TEXT 2', + location: 'LOCATION-2', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.completeInsert(tx, 'my-type'); + }); + + expect( + await knex.count('*').where('type', 'test').from('documents'), + ).toEqual([{ count: '1' }]); + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '0' }]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'query by term, %p', + async databaseId => { + const { store } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + location: 'LOCATION-1', + }, + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'test'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { pgTerm: 'Hello & World' }), + ); + + expect(rows).toEqual([ + { + document: { + location: 'LOCATION-1', + text: 'Around the world', + title: 'Hello World', + }, + rank: expect.any(Number), + type: 'test', + }, + { + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + }, + rank: expect.any(Number), + type: 'test', + }, + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'query by term for specific type, %p', + async databaseId => { + const { store } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + { + title: 'Lorem Ipsum', + text: 'Hello World', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'test', [ + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'test'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { pgTerm: 'Hello & World', types: ['my-type'] }), + ); + + expect(rows).toEqual([ + { + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'query by term and filter by field, %p', + async databaseId => { + const { store } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + ({ + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + ({ + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'that', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + fields: { myField: 'this' }, + }), + ); + + expect(rows).toEqual([ + { + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + myField: 'this', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'query by term and filter by field (any of), %p', + async databaseId => { + const { store } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + ({ + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + ({ + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'that', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + { + title: 'Hello World', + text: 'Around the world', + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + fields: { myField: ['this', 'that'] }, + }), + ); + + expect(rows).toEqual([ + { + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + myField: 'this', + }, + rank: expect.any(Number), + type: 'my-type', + }, + { + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Dolor sit amet', + myField: 'that', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'query by term and filter by fields, %p', + async databaseId => { + const { store } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + ({ + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + otherField: 'another', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + ({ + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'this', + otherField: 'unknown', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + pgTerm: 'Hello & World', + fields: { myField: 'this', otherField: 'another' }, + }), + ); + + expect(rows).toEqual([ + { + document: { + location: 'LOCATION-1', + text: 'Hello World', + title: 'Lorem Ipsum', + myField: 'this', + otherField: 'another', + }, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'query without term and filter by field, %p', + async databaseId => { + const { store } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + await store.insertDocuments(tx, 'my-type', [ + ({ + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + ({ + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + ]); + await store.completeInsert(tx, 'my-type'); + }); + + const rows = await store.transaction(tx => + store.query(tx, { + fields: { myField: 'this' }, + }), + ); + + expect(rows).toEqual([ + { + document: ({ + title: 'Lorem Ipsum', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + rank: expect.any(Number), + type: 'my-type', + }, + { + document: ({ + title: 'Dolor sit amet', + text: 'Hello World', + myField: 'this', + location: 'LOCATION-1', + } as unknown) as IndexableDocument, + rank: expect.any(Number), + type: 'my-type', + }, + ]); + }, + 60_000, + ); + }); +}); diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts new file mode 100644 index 0000000000..ecaa3e2f0f --- /dev/null +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -0,0 +1,179 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { resolvePackagePath } from '@backstage/backend-common'; +import { IndexableDocument } from '@backstage/search-common'; +import { Knex } from 'knex'; +import { + DatabaseStore, + DocumentResultRow, + PgSearchQuery, + RawDocumentRow, +} from './types'; +import { queryPostgresMajorVersion } from './util'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-search-backend-module-pg', + 'migrations', +); + +export class DatabaseDocumentStore implements DatabaseStore { + static async create(knex: Knex): Promise { + try { + const majorVersion = await queryPostgresMajorVersion(knex); + + if (majorVersion < 11) { + // We are using some features (like generated columns) that aren't + // available in older postgres versions. + throw new Error( + `The PgSearchEngine requires at least postgres version 11 (but is running on ${majorVersion})`, + ); + } + } catch { + // Actually both mysql and sqlite have a full text search, too. We could + // implement them separately or add them here. + throw new Error( + 'The PgSearchEngine is only supported when using a postgres database (>=11.x)', + ); + } + + await knex.migrate.latest({ + directory: migrationsDir, + }); + return new DatabaseDocumentStore(knex); + } + + static async supported(knex: Knex): Promise { + try { + const majorVersion = await queryPostgresMajorVersion(knex); + + return majorVersion >= 11; + } catch { + return false; + } + } + + constructor(private readonly db: Knex) {} + + async transaction(fn: (tx: Knex.Transaction) => Promise): Promise { + return await this.db.transaction(fn); + } + + async prepareInsert(tx: Knex.Transaction): Promise { + // We create a temporary table to collect the hashes of the documents that + // we expect to be in the documents table at the end. The table is deleted + // at the end of the transaction. + // The hash makes sure that we generate a new row for every change. + await tx.raw( + 'CREATE TEMP TABLE documents_to_insert (' + + 'type text NOT NULL, ' + + 'document jsonb NOT NULL, ' + + // Generating the hash requires a trick, as the text to bytea + // conversation runs into errors in case the text contains a backslash. + // Therefore we have to escape them. + "hash bytea NOT NULL GENERATED ALWAYS AS (sha256(replace(document::text || type, '\\', '\\\\')::bytea)) STORED" + + ') ON COMMIT DROP', + ); + } + + async completeInsert(tx: Knex.Transaction, type: string): Promise { + // Copy all new rows into the documents table + await tx + .insert( + tx('documents_to_insert').select( + 'type', + 'document', + 'hash', + ), + ) + .into(tx.raw('documents (type, document, hash)')) + .onConflict('hash') + .ignore(); + + // Delete all documents that we don't expect (deleted and changed) + await tx('documents') + .where({ type }) + .whereNotIn( + 'hash', + tx('documents_to_insert').select('hash'), + ) + .delete(); + } + + async insertDocuments( + tx: Knex.Transaction, + type: string, + documents: IndexableDocument[], + ): Promise { + // Insert all documents into the temporary table to process them later + await tx('documents_to_insert').insert( + documents.map(document => ({ + type, + document, + })), + ); + } + + async query( + tx: Knex.Transaction, + { types, pgTerm, fields }: PgSearchQuery, + ): Promise { + // Builds a query like: + // SELECT ts_rank_cd(body, query) AS rank, type, document + // FROM documents, to_tsquery('english', 'consent') query + // WHERE query @@ body AND (document @> '{"kind": "API"}') + // ORDER BY rank DESC + // LIMIT 10; + const query = tx('documents'); + + if (pgTerm) { + query + .from(tx.raw("documents, to_tsquery('english', ?) query", pgTerm)) + .whereRaw('query @@ body'); + } else { + query.from('documents'); + } + + if (types) { + query.whereIn('type', types); + } + + if (fields) { + Object.keys(fields).forEach(key => { + const value = fields[key]; + const valueArray = Array.isArray(value) ? value : [value]; + const valueCompare = valueArray + .map(v => ({ [key]: v })) + .map(v => JSON.stringify(v)); + query.whereRaw( + `(${valueCompare.map(() => 'document @> ?').join(' OR ')})`, + valueCompare, + ); + }); + } + + query.select('type', 'document'); + + if (pgTerm) { + query + .select(tx.raw('ts_rank_cd(body, query) AS "rank"')) + .orderBy('rank', 'desc'); + } else { + query.select(tx.raw('1 as rank')); + } + + return await query.limit(100); + } +} diff --git a/plugins/search-backend-module-pg/src/database/index.ts b/plugins/search-backend-module-pg/src/database/index.ts new file mode 100644 index 0000000000..d2ca3ba3da --- /dev/null +++ b/plugins/search-backend-module-pg/src/database/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { DatabaseDocumentStore } from './DatabaseDocumentStore'; +export type { DatabaseStore, PgSearchQuery, RawDocumentRow } from './types'; diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts new file mode 100644 index 0000000000..8c98628caf --- /dev/null +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { IndexableDocument } from '@backstage/search-common'; +import { Knex } from 'knex'; + +export interface PgSearchQuery { + fields?: Record; + types?: string[]; + pgTerm?: string; +} + +export interface DatabaseStore { + transaction(fn: (tx: Knex.Transaction) => Promise): Promise; + prepareInsert(tx: Knex.Transaction): Promise; + insertDocuments( + tx: Knex.Transaction, + type: string, + documents: IndexableDocument[], + ): Promise; + completeInsert(tx: Knex.Transaction, type: string): Promise; + query( + tx: Knex.Transaction, + pgQuery: PgSearchQuery, + ): Promise; +} + +export interface RawDocumentRow { + document: IndexableDocument; + type: string; + hash: unknown; +} + +export interface DocumentResultRow { + document: IndexableDocument; + type: string; +} diff --git a/plugins/search-backend-module-pg/src/database/util.test.ts b/plugins/search-backend-module-pg/src/database/util.test.ts new file mode 100644 index 0000000000..09eb70c096 --- /dev/null +++ b/plugins/search-backend-module-pg/src/database/util.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TestDatabases } from '@backstage/backend-test-utils'; +import { queryPostgresMajorVersion } from './util'; + +describe('util', () => { + describe('unsupported', () => { + const databases = TestDatabases.create({ + ids: ['SQLITE_3', 'MYSQL_8'], + }); + + it.each(databases.eachSupportedId())( + 'should fail on get postgres major version, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await expect( + async () => await queryPostgresMajorVersion(knex), + ).rejects.toThrow(); + }, + 60_000, + ); + }); + + describe('supported', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9'], + }); + + if (databases.eachSupportedId().length < 1) { + // Only execute tests if at least on database engine is available, e.g. if + // not in CI=1. it.each doesn't support an empty array. + return; + } + + it.each(databases.eachSupportedId())( + 'should get postgres major version, %p', + async databaseId => { + const knex = await databases.init(databaseId); + const expectedVersion = +databaseId.substr(9); + + await expect(queryPostgresMajorVersion(knex)).resolves.toBe( + expectedVersion, + ); + }, + 60_000, + ); + }); +}); diff --git a/plugins/search-backend-module-pg/src/database/util.ts b/plugins/search-backend-module-pg/src/database/util.ts new file mode 100644 index 0000000000..674e780625 --- /dev/null +++ b/plugins/search-backend-module-pg/src/database/util.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; + +export async function queryPostgresMajorVersion(knex: Knex): Promise { + if (knex.client.config.client !== 'pg') { + throw new Error("Can' resolve version, not a postgres database"); + } + + const { rows } = await knex.raw('SHOW server_version_num'); + const [result] = rows; + const version = +result.server_version_num; + const majorVersion = Math.floor(version / 10000); + return majorVersion; +} diff --git a/plugins/search-backend-module-pg/src/index.ts b/plugins/search-backend-module-pg/src/index.ts new file mode 100644 index 0000000000..7c5716cfc4 --- /dev/null +++ b/plugins/search-backend-module-pg/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './database'; +export * from './PgSearchEngine'; diff --git a/plugins/search-backend-module-pg/src/setupTests.ts b/plugins/search-backend-module-pg/src/setupTests.ts new file mode 100644 index 0000000000..fb7d1a181a --- /dev/null +++ b/plugins/search-backend-module-pg/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {};