Add postgres based SearchEngine

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>

.changeset/search-early-horses-sip.md

@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-search-backend-module-pg': patch
+---
+
+Add `plugin-search-backend-module-pg` providing a postgres based search engine.
+See the [README of `search-backend-module-pg`](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-pg/README.md) for usage instructions.

packages/backend/package.json

@@ -48,6 +48,7 @@
     "@backstage/plugin-search-backend": "^0.2.3",
     "@backstage/plugin-search-backend-node": "^0.4.0",
     "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.1",
+    "@backstage/plugin-search-backend-module-pg": "^0.1.0",
     "@backstage/plugin-techdocs-backend": "^0.9.0",
     "@backstage/plugin-todo-backend": "^0.1.8",
     "@gitbeaker/node": "^30.2.0",

packages/backend/src/plugins/search.ts

@@ -14,20 +14,22 @@
  * limitations under the License.
  */
 import { useHotCleanup } from '@backstage/backend-common';
+import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
 import { createRouter } from '@backstage/plugin-search-backend';
+import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
+import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg';
 import {
   IndexBuilder,
   LunrSearchEngine,
 } from '@backstage/plugin-search-backend-node';
-import { PluginEnvironment } from '../types';
-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
 import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
-import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
+import { PluginEnvironment } from '../types';
 
 export default async function createPlugin({
   logger,
   discovery,
   config,
+  database,
 }: PluginEnvironment) {
   // Initialize a connection to a search engine.
   const searchEngine = config.has('search.elasticsearch')
@@ -35,6 +37,8 @@ export default async function createPlugin({
         logger,
         config,
       })
+    : (await PgSearchEngine.supported(database))
+    ? await PgSearchEngine.from({ database })
     : new LunrSearchEngine({ logger });
   const indexBuilder = new IndexBuilder({ logger, searchEngine });
 

plugins/search-backend-module-pg/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/search-backend-module-pg/README.md

@@ -0,0 +1,28 @@
+# search-backend-module-pg
+
+This plugin provides an easy to use `SearchEngine` implementation to use with the
+`@backstage/plugin-search-backend` based on Postgres.
+Therefore it targets setups that want to avoid maintaining another external
+service like elastic search. The search provides decent results and performs
+well with ten thousands of indexed documents.
+The connection to postgres is established via the database manager also used by
+other plugins.
+
+> **Important**: The search plugin requires at least Postgres 11!
+
+## Setup
+
+To use the `SearchEngine`, make sure that you have a Postgres database
+configured and make the following changes to your backend:
+
+1. Add a dependency on `@backstage/plugin-search-backend-module-pg` to your backend's `package.json`.
+2. Initialize the search engine. It is recommended to initialize it with a fallback to the lunr search engine if you are running Backstage for development locally with SQLite:
+
+```typescript
+// In packages/backend/src/plugins/search.ts
+
+// Initialize a connection to a search engine.
+const searchEngine = (await PgSearchEngine.supported(database))
+  ? await PgSearchEngine.from({ database })
+  : new LunrSearchEngine({ logger });
+```

plugins/search-backend-module-pg/migrations/20210311110000_init.js

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.createTable('documents', table => {
+    table.comment('The table of documents');
+    table
+      .text('type')
+      .notNullable()
+      .index()
+      .comment('The index type of the document');
+    table.jsonb('document').notNullable().comment('The document');
+    table.specificType(
+      'body',
+      'tsvector NOT NULL GENERATED ALWAYS AS (' +
+        "setweight(to_tsvector('english', document->>'title'), 'A') || " +
+        "setweight(to_tsvector('english', document->>'text'), 'B') || " +
+        "setweight(to_tsvector('english', document - 'location' - 'title' - 'text'), 'C')" +
+        ') STORED',
+    );
+
+    table
+      .index(['body'], 'documents_body_index', 'GIN')
+      .comment('Optimize full text queries on documents');
+    table
+      .index(['document'], 'documents_document_index', 'GIN')
+      .comment('Optimize filter queries on documents');
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.dropTable('documents');
+};

plugins/search-backend-module-pg/migrations/20210727180000_delta_update.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('documents', table => {
+    // Extend the documents table with a column that allows to check whether a
+    // document with the same content is already indexed.
+    table.specificType('hash', 'bytea');
+  });
+
+  await knex('documents').update({
+    hash: knex.raw(
+      "sha256(replace(document::text || type, '\\', '\\\\')::bytea)",
+    ),
+  });
+
+  await knex.schema.alterTable('documents', table => {
+    table.specificType('hash', 'bytea').notNullable().primary().alter();
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('documents', table => {
+    table.dropColumn('hash');
+  });
+};

plugins/search-backend-module-pg/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@backstage/plugin-search-backend-module-pg",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "scripts": {
+    "start": "backstage-cli backend:dev",
+    "build": "backstage-cli backend:build",
+    "lint": "backstage-cli lint",
+    "test": "backstage-cli test",
+    "prepack": "backstage-cli prepack",
+    "postpack": "backstage-cli postpack",
+    "clean": "backstage-cli clean"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.8.7",
+    "@backstage/search-common": "^0.1.2",
+    "@backstage/plugin-search-backend-node": "^0.4.0",
+    "lodash": "^4.17.15",
+    "knex": "^0.95.1"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^0.1.4",
+    "@backstage/cli": "^0.7.6"
+  },
+  "files": [
+    "dist",
+    "migrations"
+  ]
+}

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { range } from 'lodash';
+import { DatabaseStore, PgSearchQuery } from '../database';
+import { PgSearchEngine } from './PgSearchEngine';
+
+describe('PgSearchEngine', () => {
+  const tx: any = {} as any;
+  let searchEngine: PgSearchEngine;
+  let database: jest.Mocked<DatabaseStore>;
+
+  beforeEach(() => {
+    database = {
+      transaction: jest.fn(),
+      insertDocuments: jest.fn(),
+      query: jest.fn(),
+      completeInsert: jest.fn(),
+      prepareInsert: jest.fn(),
+    };
+    searchEngine = new PgSearchEngine(database);
+  });
+
+  describe('translator', () => {
+    it('query translator invoked', async () => {
+      database.transaction.mockResolvedValue([]);
+      const translatorSpy = jest.fn().mockReturnValue({
+        pgSearchTerm: 'testTerm',
+      });
+      searchEngine.setTranslator(translatorSpy);
+
+      await searchEngine.query({
+        term: 'testTerm',
+        filters: {},
+        pageCursor: '',
+      });
+
+      expect(translatorSpy).toHaveBeenCalledWith({
+        term: 'testTerm',
+        filters: {},
+        pageCursor: '',
+      });
+    });
+
+    it('should return translated query term', async () => {
+      const actualTranslatedQuery = searchEngine.translator({
+        term: 'Hello World',
+        pageCursor: '',
+      }) as PgSearchQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
+      });
+    });
+
+    it('should return translated query with filters', async () => {
+      const actualTranslatedQuery = searchEngine.translator({
+        term: 'testTerm',
+        filters: { kind: 'testKind' },
+        types: ['my-filter'],
+        pageCursor: '',
+      }) as PgSearchQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        pgTerm: '("testTerm" | "testTerm":*)',
+        fields: { kind: 'testKind' },
+        types: ['my-filter'],
+      });
+    });
+  });
+
+  describe('insert', () => {
+    it('should insert documents', async () => {
+      database.transaction.mockImplementation(fn => fn(tx));
+
+      const documents = [
+        { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' },
+        {
+          location: 'location-2',
+          text: 'Hello World',
+          title: 'Dolor sit amet',
+        },
+      ];
+
+      await searchEngine.index('my-type', documents);
+
+      expect(database.transaction).toHaveBeenCalledTimes(1);
+      expect(database.prepareInsert).toHaveBeenCalledTimes(1);
+      expect(database.insertDocuments).toHaveBeenCalledWith(
+        tx,
+        'my-type',
+        documents,
+      );
+      expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
+    });
+
+    it('should batch insert documents', async () => {
+      database.transaction.mockImplementation(fn => fn(tx));
+
+      const documents = range(350).map(i => ({
+        title: `Hello World ${i}`,
+        text: 'Lorem Ipsum',
+        location: `location-${i}`,
+      }));
+
+      await searchEngine.index('my-type', documents);
+
+      expect(database.transaction).toHaveBeenCalledTimes(1);
+      expect(database.prepareInsert).toHaveBeenCalledTimes(1);
+      expect(database.insertDocuments).toBeCalledTimes(4);
+      expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
+    });
+  });
+
+  describe('query', () => {
+    it('should perform query', async () => {
+      database.transaction.mockImplementation(fn => fn(tx));
+      database.query.mockResolvedValue([
+        {
+          document: {
+            title: 'Hello World',
+            text: 'Lorem Ipsum',
+            location: 'location-1',
+          },
+          type: 'my-type',
+        },
+      ]);
+
+      const results = await searchEngine.query({
+        term: 'Hello World',
+        pageCursor: '',
+      });
+
+      expect(results).toEqual({
+        results: [
+          {
+            document: {
+              title: 'Hello World',
+              text: 'Lorem Ipsum',
+              location: 'location-1',
+            },
+            type: 'my-type',
+          },
+        ],
+      });
+      expect(database.transaction).toHaveBeenCalledTimes(1);
+      expect(database.query).toHaveBeenCalledWith(tx, {
+        pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
+      });
+    });
+  });
+});

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { PluginDatabaseManager } from '@backstage/backend-common';
+import { SearchEngine } from '@backstage/plugin-search-backend-node';
+import {
+  IndexableDocument,
+  SearchQuery,
+  SearchResultSet,
+} from '@backstage/search-common';
+import { chunk } from 'lodash';
+import {
+  DatabaseDocumentStore,
+  DatabaseStore,
+  PgSearchQuery,
+} from '../database';
+
+// TODO: Support paging using page cursor (return cursor and parse cursor)
+
+export class PgSearchEngine implements SearchEngine {
+  constructor(private readonly databaseStore: DatabaseStore) {}
+
+  static async from({
+    database,
+  }: {
+    database: PluginDatabaseManager;
+  }): Promise<PgSearchEngine> {
+    return new PgSearchEngine(
+      await DatabaseDocumentStore.create(await database.getClient()),
+    );
+  }
+
+  static async supported(database: PluginDatabaseManager): Promise<boolean> {
+    return await DatabaseDocumentStore.supported(await database.getClient());
+  }
+
+  translator(query: SearchQuery): PgSearchQuery {
+    return {
+      pgTerm: query.term
+        .split(/\s/)
+        .map(p => p.trim())
+        .filter(p => p !== '')
+        .map(p => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`)
+        .join('&'),
+      fields: query.filters as Record<string, string | string[]>,
+      types: query.types,
+    };
+  }
+
+  setTranslator(translator: (query: SearchQuery) => PgSearchQuery): void {
+    this.translator = translator;
+  }
+
+  async index(type: string, documents: IndexableDocument[]): Promise<void> {
+    await this.databaseStore.transaction(async tx => {
+      await this.databaseStore.prepareInsert(tx);
+
+      const batchSize = 100;
+      for (const documentBatch of chunk(documents, batchSize)) {
+        await this.databaseStore.insertDocuments(tx, type, documentBatch);
+      }
+
+      await this.databaseStore.completeInsert(tx, type);
+    });
+  }
+
+  async query(query: SearchQuery): Promise<SearchResultSet> {
+    const pgQuery = this.translator(query);
+
+    const rows = await this.databaseStore.transaction(async tx =>
+      this.databaseStore.query(tx, pgQuery),
+    );
+    const results = rows.map(({ type, document }) => ({
+      type,
+      document,
+    }));
+
+    return { results };
+  }
+}

plugins/search-backend-module-pg/src/PgSearchEngine/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { PgSearchEngine } from './PgSearchEngine';

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts

@@ -0,0 +1,512 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
+import { IndexableDocument } from '@backstage/search-common';
+import { DatabaseDocumentStore } from './DatabaseDocumentStore';
+
+describe('DatabaseDocumentStore', () => {
+  describe('unsupported', () => {
+    const databases = TestDatabases.create({
+      ids: ['MYSQL_8', 'POSTGRES_9', 'SQLITE_3'],
+    });
+
+    it.each(databases.eachSupportedId())(
+      'should return support state, %p',
+      async databaseId => {
+        const knex = await databases.init(databaseId);
+        const supported = await DatabaseDocumentStore.supported(knex);
+
+        expect(supported).toBe(false);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should fail to create, %p',
+      async databaseId => {
+        const knex = await databases.init(databaseId);
+
+        await expect(
+          async () => await DatabaseDocumentStore.create(knex),
+        ).rejects.toThrow();
+      },
+      60_000,
+    );
+  });
+
+  describe('supported', () => {
+    const databases = TestDatabases.create({
+      ids: ['POSTGRES_13'],
+    });
+
+    async function createStore(databaseId: TestDatabaseId) {
+      const knex = await databases.init(databaseId);
+      const store = await DatabaseDocumentStore.create(knex);
+      return { store, knex };
+    }
+
+    if (databases.eachSupportedId().length < 1) {
+      // Only execute tests if at least on database engine is available, e.g. if
+      // not in CI=1. it.each doesn't support an empty array.
+      return;
+    }
+
+    it.each(databases.eachSupportedId())(
+      'should return support state, %p',
+      async databaseId => {
+        const knex = await databases.init(databaseId);
+        const supported = await DatabaseDocumentStore.supported(knex);
+
+        expect(supported).toBe(true);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should insert documents, %p',
+      async databaseId => {
+        const { store, knex } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            {
+              title: 'TITLE 1',
+              text: 'TEXT 1',
+              location: 'LOCATION-1',
+            },
+            {
+              title: 'TITLE 2',
+              text: 'TEXT 2',
+              location: 'LOCATION-2',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        expect(
+          await knex.count('*').where('type', 'my-type').from('documents'),
+        ).toEqual([{ count: '2' }]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should insert documents in batches, %p',
+      async databaseId => {
+        const { store, knex } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            {
+              title: 'TITLE 1',
+              text: 'TEXT 1',
+              location: 'LOCATION-1',
+            },
+            {
+              title: 'TITLE 2',
+              text: 'TEXT 2',
+              location: 'LOCATION-2',
+            },
+          ]);
+          await store.insertDocuments(tx, 'my-type', [
+            {
+              title: 'TITLE 3',
+              text: 'TEXT 3',
+              location: 'LOCATION-3',
+            },
+            {
+              title: 'TITLE 4',
+              text: 'TEXT 4',
+              location: 'LOCATION-4',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        expect(
+          await knex.count('*').where('type', 'my-type').from('documents'),
+        ).toEqual([{ count: '4' }]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'should clear index for type, %p',
+      async databaseId => {
+        const { store, knex } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'test', [
+            {
+              title: 'TITLE 1',
+              text: 'TEXT 1',
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'test');
+        });
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            {
+              title: 'TITLE 1',
+              text: 'TEXT 1',
+              location: 'LOCATION-1',
+            },
+            {
+              title: 'TITLE 2',
+              text: 'TEXT 2',
+              location: 'LOCATION-2',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        expect(
+          await knex.count('*').where('type', 'test').from('documents'),
+        ).toEqual([{ count: '1' }]);
+        expect(
+          await knex.count('*').where('type', 'my-type').from('documents'),
+        ).toEqual([{ count: '0' }]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'query by term, %p',
+      async databaseId => {
+        const { store } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'test', [
+            {
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              location: 'LOCATION-1',
+            },
+            {
+              title: 'Hello World',
+              text: 'Around the world',
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'test');
+        });
+
+        const rows = await store.transaction(tx =>
+          store.query(tx, { pgTerm: 'Hello & World' }),
+        );
+
+        expect(rows).toEqual([
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Around the world',
+              title: 'Hello World',
+            },
+            rank: expect.any(Number),
+            type: 'test',
+          },
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Hello World',
+              title: 'Lorem Ipsum',
+            },
+            rank: expect.any(Number),
+            type: 'test',
+          },
+        ]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'query by term for specific type, %p',
+      async databaseId => {
+        const { store } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            {
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'test', [
+            {
+              title: 'Hello World',
+              text: 'Around the world',
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'test');
+        });
+
+        const rows = await store.transaction(tx =>
+          store.query(tx, { pgTerm: 'Hello & World', types: ['my-type'] }),
+        );
+
+        expect(rows).toEqual([
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Hello World',
+              title: 'Lorem Ipsum',
+            },
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+        ]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'query by term and filter by field, %p',
+      async databaseId => {
+        const { store } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            ({
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              myField: 'this',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            ({
+              title: 'Dolor sit amet',
+              text: 'Hello World',
+              myField: 'that',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            {
+              title: 'Hello World',
+              text: 'Around the world',
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        const rows = await store.transaction(tx =>
+          store.query(tx, {
+            pgTerm: 'Hello & World',
+            fields: { myField: 'this' },
+          }),
+        );
+
+        expect(rows).toEqual([
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Hello World',
+              title: 'Lorem Ipsum',
+              myField: 'this',
+            },
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+        ]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'query by term and filter by field (any of), %p',
+      async databaseId => {
+        const { store } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            ({
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              myField: 'this',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            ({
+              title: 'Dolor sit amet',
+              text: 'Hello World',
+              myField: 'that',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            {
+              title: 'Hello World',
+              text: 'Around the world',
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        const rows = await store.transaction(tx =>
+          store.query(tx, {
+            pgTerm: 'Hello & World',
+            fields: { myField: ['this', 'that'] },
+          }),
+        );
+
+        expect(rows).toEqual([
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Hello World',
+              title: 'Lorem Ipsum',
+              myField: 'this',
+            },
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Hello World',
+              title: 'Dolor sit amet',
+              myField: 'that',
+            },
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+        ]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'query by term and filter by fields, %p',
+      async databaseId => {
+        const { store } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            ({
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              myField: 'this',
+              otherField: 'another',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            ({
+              title: 'Dolor sit amet',
+              text: 'Hello World',
+              myField: 'this',
+              otherField: 'unknown',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        const rows = await store.transaction(tx =>
+          store.query(tx, {
+            pgTerm: 'Hello & World',
+            fields: { myField: 'this', otherField: 'another' },
+          }),
+        );
+
+        expect(rows).toEqual([
+          {
+            document: {
+              location: 'LOCATION-1',
+              text: 'Hello World',
+              title: 'Lorem Ipsum',
+              myField: 'this',
+              otherField: 'another',
+            },
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+        ]);
+      },
+      60_000,
+    );
+
+    it.each(databases.eachSupportedId())(
+      'query without term and filter by field, %p',
+      async databaseId => {
+        const { store } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+          await store.insertDocuments(tx, 'my-type', [
+            ({
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              myField: 'this',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            ({
+              title: 'Dolor sit amet',
+              text: 'Hello World',
+              myField: 'this',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+          ]);
+          await store.completeInsert(tx, 'my-type');
+        });
+
+        const rows = await store.transaction(tx =>
+          store.query(tx, {
+            fields: { myField: 'this' },
+          }),
+        );
+
+        expect(rows).toEqual([
+          {
+            document: ({
+              title: 'Lorem Ipsum',
+              text: 'Hello World',
+              myField: 'this',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+          {
+            document: ({
+              title: 'Dolor sit amet',
+              text: 'Hello World',
+              myField: 'this',
+              location: 'LOCATION-1',
+            } as unknown) as IndexableDocument,
+            rank: expect.any(Number),
+            type: 'my-type',
+          },
+        ]);
+      },
+      60_000,
+    );
+  });
+});

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { resolvePackagePath } from '@backstage/backend-common';
+import { IndexableDocument } from '@backstage/search-common';
+import { Knex } from 'knex';
+import {
+  DatabaseStore,
+  DocumentResultRow,
+  PgSearchQuery,
+  RawDocumentRow,
+} from './types';
+import { queryPostgresMajorVersion } from './util';
+
+const migrationsDir = resolvePackagePath(
+  '@backstage/plugin-search-backend-module-pg',
+  'migrations',
+);
+
+export class DatabaseDocumentStore implements DatabaseStore {
+  static async create(knex: Knex): Promise<DatabaseDocumentStore> {
+    try {
+      const majorVersion = await queryPostgresMajorVersion(knex);
+
+      if (majorVersion < 11) {
+        // We are using some features (like generated columns) that aren't
+        // available in older postgres versions.
+        throw new Error(
+          `The PgSearchEngine requires at least postgres version 11 (but is running on ${majorVersion})`,
+        );
+      }
+    } catch {
+      // Actually both mysql and sqlite have a full text search, too. We could
+      // implement them separately or add them here.
+      throw new Error(
+        'The PgSearchEngine is only supported when using a postgres database (>=11.x)',
+      );
+    }
+
+    await knex.migrate.latest({
+      directory: migrationsDir,
+    });
+    return new DatabaseDocumentStore(knex);
+  }
+
+  static async supported(knex: Knex): Promise<boolean> {
+    try {
+      const majorVersion = await queryPostgresMajorVersion(knex);
+
+      return majorVersion >= 11;
+    } catch {
+      return false;
+    }
+  }
+
+  constructor(private readonly db: Knex) {}
+
+  async transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T> {
+    return await this.db.transaction(fn);
+  }
+
+  async prepareInsert(tx: Knex.Transaction): Promise<void> {
+    // We create a temporary table to collect the hashes of the documents that
+    // we expect to be in the documents table at the end. The table is deleted
+    // at the end of the transaction.
+    // The hash makes sure that we generate a new row for every change.
+    await tx.raw(
+      'CREATE TEMP TABLE documents_to_insert (' +
+        'type text NOT NULL, ' +
+        'document jsonb NOT NULL, ' +
+        // Generating the hash requires a trick, as the text to bytea
+        // conversation runs into errors in case the text contains a backslash.
+        // Therefore we have to escape them.
+        "hash bytea NOT NULL GENERATED ALWAYS AS (sha256(replace(document::text || type, '\\', '\\\\')::bytea)) STORED" +
+        ') ON COMMIT DROP',
+    );
+  }
+
+  async completeInsert(tx: Knex.Transaction, type: string): Promise<void> {
+    // Copy all new rows into the documents table
+    await tx
+      .insert(
+        tx<RawDocumentRow>('documents_to_insert').select(
+          'type',
+          'document',
+          'hash',
+        ),
+      )
+      .into(tx.raw('documents (type, document, hash)'))
+      .onConflict('hash')
+      .ignore();
+
+    // Delete all documents that we don't expect (deleted and changed)
+    await tx<RawDocumentRow>('documents')
+      .where({ type })
+      .whereNotIn(
+        'hash',
+        tx<RawDocumentRow>('documents_to_insert').select('hash'),
+      )
+      .delete();
+  }
+
+  async insertDocuments(
+    tx: Knex.Transaction,
+    type: string,
+    documents: IndexableDocument[],
+  ): Promise<void> {
+    // Insert all documents into the temporary table to process them later
+    await tx<DocumentResultRow>('documents_to_insert').insert(
+      documents.map(document => ({
+        type,
+        document,
+      })),
+    );
+  }
+
+  async query(
+    tx: Knex.Transaction,
+    { types, pgTerm, fields }: PgSearchQuery,
+  ): Promise<DocumentResultRow[]> {
+    // Builds a query like:
+    // SELECT ts_rank_cd(body, query) AS rank,  type, document
+    // FROM documents, to_tsquery('english', 'consent') query
+    // WHERE query @@ body AND (document @> '{"kind": "API"}')
+    // ORDER BY rank DESC
+    // LIMIT 10;
+    const query = tx<DocumentResultRow>('documents');
+
+    if (pgTerm) {
+      query
+        .from(tx.raw("documents, to_tsquery('english', ?) query", pgTerm))
+        .whereRaw('query @@ body');
+    } else {
+      query.from('documents');
+    }
+
+    if (types) {
+      query.whereIn('type', types);
+    }
+
+    if (fields) {
+      Object.keys(fields).forEach(key => {
+        const value = fields[key];
+        const valueArray = Array.isArray(value) ? value : [value];
+        const valueCompare = valueArray
+          .map(v => ({ [key]: v }))
+          .map(v => JSON.stringify(v));
+        query.whereRaw(
+          `(${valueCompare.map(() => 'document @> ?').join(' OR ')})`,
+          valueCompare,
+        );
+      });
+    }
+
+    query.select('type', 'document');
+
+    if (pgTerm) {
+      query
+        .select(tx.raw('ts_rank_cd(body, query) AS "rank"'))
+        .orderBy('rank', 'desc');
+    } else {
+      query.select(tx.raw('1 as rank'));
+    }
+
+    return await query.limit(100);
+  }
+}

plugins/search-backend-module-pg/src/database/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { DatabaseDocumentStore } from './DatabaseDocumentStore';
+export type { DatabaseStore, PgSearchQuery, RawDocumentRow } from './types';

plugins/search-backend-module-pg/src/database/types.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { IndexableDocument } from '@backstage/search-common';
+import { Knex } from 'knex';
+
+export interface PgSearchQuery {
+  fields?: Record<string, string | string[]>;
+  types?: string[];
+  pgTerm?: string;
+}
+
+export interface DatabaseStore {
+  transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T>;
+  prepareInsert(tx: Knex.Transaction): Promise<void>;
+  insertDocuments(
+    tx: Knex.Transaction,
+    type: string,
+    documents: IndexableDocument[],
+  ): Promise<void>;
+  completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
+  query(
+    tx: Knex.Transaction,
+    pgQuery: PgSearchQuery,
+  ): Promise<DocumentResultRow[]>;
+}
+
+export interface RawDocumentRow {
+  document: IndexableDocument;
+  type: string;
+  hash: unknown;
+}
+
+export interface DocumentResultRow {
+  document: IndexableDocument;
+  type: string;
+}

plugins/search-backend-module-pg/src/database/util.test.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TestDatabases } from '@backstage/backend-test-utils';
+import { queryPostgresMajorVersion } from './util';
+
+describe('util', () => {
+  describe('unsupported', () => {
+    const databases = TestDatabases.create({
+      ids: ['SQLITE_3', 'MYSQL_8'],
+    });
+
+    it.each(databases.eachSupportedId())(
+      'should fail on get postgres major version, %p',
+      async databaseId => {
+        const knex = await databases.init(databaseId);
+
+        await expect(
+          async () => await queryPostgresMajorVersion(knex),
+        ).rejects.toThrow();
+      },
+      60_000,
+    );
+  });
+
+  describe('supported', () => {
+    const databases = TestDatabases.create({
+      ids: ['POSTGRES_13', 'POSTGRES_9'],
+    });
+
+    if (databases.eachSupportedId().length < 1) {
+      // Only execute tests if at least on database engine is available, e.g. if
+      // not in CI=1. it.each doesn't support an empty array.
+      return;
+    }
+
+    it.each(databases.eachSupportedId())(
+      'should get postgres major version, %p',
+      async databaseId => {
+        const knex = await databases.init(databaseId);
+        const expectedVersion = +databaseId.substr(9);
+
+        await expect(queryPostgresMajorVersion(knex)).resolves.toBe(
+          expectedVersion,
+        );
+      },
+      60_000,
+    );
+  });
+});

plugins/search-backend-module-pg/src/database/util.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Knex } from 'knex';
+
+export async function queryPostgresMajorVersion(knex: Knex): Promise<number> {
+  if (knex.client.config.client !== 'pg') {
+    throw new Error("Can' resolve version, not a postgres database");
+  }
+
+  const { rows } = await knex.raw('SHOW server_version_num');
+  const [result] = rows;
+  const version = +result.server_version_num;
+  const majorVersion = Math.floor(version / 10000);
+  return majorVersion;
+}

plugins/search-backend-module-pg/src/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './database';
+export * from './PgSearchEngine';

plugins/search-backend-module-pg/src/setupTests.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export {};