Merge pull request #6682 from SDA-SE/feat/search-postgres
Add postgres based `SearchEngine`
This commit is contained in:
Oliver Sand
@@ -0,0 +1,6 @@
|
||||
---
|
||||
'@backstage/plugin-search-backend-module-pg': patch
|
||||
---
|
||||
|
||||
Add `plugin-search-backend-module-pg` providing a postgres based search engine.
|
||||
See the [README of `search-backend-module-pg`](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-pg/README.md) for usage instructions.
|
||||
@@ -4,8 +4,6 @@ title: Search Architecture
|
||||
description: Documentation on Search Architecture
|
||||
---
|
||||
|
||||
# Search Architecture
|
||||
|
||||
> _This architecture has not been fully implemented yet. Find our milestones to
|
||||
> follow our progress and help contribute on the
|
||||
> [Search Roadmap](./README.md#project-roadmap)._
|
||||
|
||||
@@ -4,8 +4,6 @@ title: Search Concepts
|
||||
description: Documentation on Backstage Search Concepts
|
||||
---
|
||||
|
||||
# Search Concepts
|
||||
|
||||
Backstage Search lets you find the right information you are looking for in the
|
||||
Backstage ecosystem.
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ title: Getting Started with Search
|
||||
description: How to set up and install Backstage Search
|
||||
---
|
||||
|
||||
# Getting Started
|
||||
|
||||
Search functions as a plugin to Backstage, so you will need to use Backstage to
|
||||
use Search.
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ title: Search Engines
|
||||
description: Choosing and configuring your search engine for Backstage
|
||||
---
|
||||
|
||||
# Search Engines
|
||||
|
||||
Backstage supports 2 search engines by default, an in-memory engine called Lunr
|
||||
and ElasticSearch. You can configure your own search engines by implementing the
|
||||
provided interface as mentioned in the
|
||||
@@ -18,7 +16,7 @@ QueryTranslator interface. This modification can be done without touching
|
||||
provided search engines by using the exposed setter to set the modified query
|
||||
translator into the instance.
|
||||
|
||||
```
|
||||
```typescript
|
||||
const searchEngine = new LunrSearchEngine({ logger });
|
||||
searchEngine.setTranslator(new MyNewAndBetterQueryTranslator());
|
||||
```
|
||||
@@ -36,6 +34,35 @@ const searchEngine = new LunrSearchEngine({ logger });
|
||||
const indexBuilder = new IndexBuilder({ logger, searchEngine });
|
||||
```
|
||||
|
||||
## Postgres
|
||||
|
||||
The Postgres based search engine only requires that postgres being configured as
|
||||
the database engine for Backstage. Therefore it targets setups that want to
|
||||
avoid maintaining another external service like elastic search. The search
|
||||
provides decent results and performs well with ten thousands of indexed
|
||||
documents. The connection to postgres is established via the database manager
|
||||
also used by other plugins.
|
||||
|
||||
> **Important**: The search plugin requires at least Postgres 11!
|
||||
|
||||
To use the `PgSearchEngine`, make sure that you have a Postgres database
|
||||
configured and make the following changes to your backend:
|
||||
|
||||
1. Add a dependency on `@backstage/plugin-search-backend-module-pg` to your
|
||||
backend's `package.json`.
|
||||
2. Initialize the search engine. It is recommended to initialize it with a
|
||||
fallback to the lunr search engine if you are running Backstage for
|
||||
development locally with SQLite:
|
||||
|
||||
```typescript
|
||||
// In packages/backend/src/plugins/search.ts
|
||||
|
||||
// Initialize a connection to a search engine.
|
||||
const searchEngine = (await PgSearchEngine.supported(database))
|
||||
? await PgSearchEngine.from({ database })
|
||||
: new LunrSearchEngine({ logger });
|
||||
```
|
||||
|
||||
## ElasticSearch
|
||||
|
||||
Backstage supports ElasticSearch search engine connections, indexing and
|
||||
|
||||
@@ -80,7 +80,8 @@
|
||||
"features/search/search-overview",
|
||||
"features/search/getting-started",
|
||||
"features/search/concepts",
|
||||
"features/search/architecture"
|
||||
"features/search/architecture",
|
||||
"features/search/search-engines"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -64,6 +64,7 @@ nav:
|
||||
- Getting Started: 'features/search/getting-started.md'
|
||||
- Concepts: 'features/search/concepts.md'
|
||||
- Search Architecture: 'features/search/architecture.md'
|
||||
- Search Engines: 'features/search/search-engines.md'
|
||||
- TechDocs:
|
||||
- Overview: 'features/techdocs/README.md'
|
||||
- Getting Started: 'features/techdocs/getting-started.md'
|
||||
|
||||
@@ -48,6 +48,7 @@
|
||||
"@backstage/plugin-search-backend": "^0.2.3",
|
||||
"@backstage/plugin-search-backend-node": "^0.4.0",
|
||||
"@backstage/plugin-search-backend-module-elasticsearch": "^0.0.1",
|
||||
"@backstage/plugin-search-backend-module-pg": "^0.1.0",
|
||||
"@backstage/plugin-techdocs-backend": "^0.9.0",
|
||||
"@backstage/plugin-todo-backend": "^0.1.8",
|
||||
"@gitbeaker/node": "^30.2.0",
|
||||
|
||||
@@ -13,29 +13,55 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { useHotCleanup } from '@backstage/backend-common';
|
||||
import {
|
||||
PluginDatabaseManager,
|
||||
useHotCleanup,
|
||||
} from '@backstage/backend-common';
|
||||
import { Config } from '@backstage/config';
|
||||
import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
|
||||
import { createRouter } from '@backstage/plugin-search-backend';
|
||||
import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
|
||||
import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg';
|
||||
import {
|
||||
IndexBuilder,
|
||||
LunrSearchEngine,
|
||||
SearchEngine,
|
||||
} from '@backstage/plugin-search-backend-node';
|
||||
import { PluginEnvironment } from '../types';
|
||||
import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend';
|
||||
import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
|
||||
import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch';
|
||||
import { Logger } from 'winston';
|
||||
import { PluginEnvironment } from '../types';
|
||||
|
||||
async function createSearchEngine({
|
||||
logger,
|
||||
database,
|
||||
config,
|
||||
}: {
|
||||
logger: Logger;
|
||||
database: PluginDatabaseManager;
|
||||
config: Config;
|
||||
}): Promise<SearchEngine> {
|
||||
if (config.has('search.elasticsearch')) {
|
||||
return await ElasticSearchSearchEngine.fromConfig({
|
||||
logger,
|
||||
config,
|
||||
});
|
||||
}
|
||||
|
||||
if (await PgSearchEngine.supported(database)) {
|
||||
return await PgSearchEngine.from({ database });
|
||||
}
|
||||
|
||||
return new LunrSearchEngine({ logger });
|
||||
}
|
||||
|
||||
export default async function createPlugin({
|
||||
logger,
|
||||
discovery,
|
||||
config,
|
||||
database,
|
||||
}: PluginEnvironment) {
|
||||
// Initialize a connection to a search engine.
|
||||
const searchEngine = config.has('search.elasticsearch')
|
||||
? await ElasticSearchSearchEngine.fromConfig({
|
||||
logger,
|
||||
config,
|
||||
})
|
||||
: new LunrSearchEngine({ logger });
|
||||
const searchEngine = await createSearchEngine({ config, logger, database });
|
||||
const indexBuilder = new IndexBuilder({ logger, searchEngine });
|
||||
|
||||
// Collators are responsible for gathering documents known to plugins. This
|
||||
|
||||
@@ -6,4 +6,4 @@ This module provides functionality to index and implement querying using Elastic
|
||||
|
||||
## Getting started
|
||||
|
||||
See [Backstage documentation](https://backstage.io/docs/features/search/getting-started) for details on how to install and configure ElasticSearch for your Backstage instance.
|
||||
See [Backstage documentation](https://backstage.io/docs/features/search/search-engines#elasticsearch) for details on how to install and configure ElasticSearch for your Backstage instance.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module.exports = {
|
||||
extends: [require.resolve('@backstage/cli/config/eslint.backend')],
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
# search-backend-module-pg
|
||||
|
||||
This plugin provides an easy to use `SearchEngine` implementation to use with the
|
||||
`@backstage/plugin-search-backend` based on Postgres.
|
||||
Therefore it targets setups that want to avoid maintaining another external
|
||||
service like elastic search. The search provides decent results and performs
|
||||
well with ten thousands of indexed documents.
|
||||
The connection to postgres is established via the database manager also used by
|
||||
other plugins.
|
||||
|
||||
> **Important**: The search plugin requires at least Postgres 11!
|
||||
|
||||
## Getting started
|
||||
|
||||
See [Backstage documentation](https://backstage.io/docs/features/search/search-engines#postgres)
|
||||
for details on how to setup Postgres based search for your Backstage instance.
|
||||
@@ -0,0 +1,114 @@
|
||||
## API Report File for "@backstage/plugin-search-backend-module-pg"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Knex } from 'knex';
|
||||
import { PluginDatabaseManager } from '@backstage/backend-common';
|
||||
import { SearchEngine } from '@backstage/plugin-search-backend-node';
|
||||
import { SearchQuery } from '@backstage/search-common';
|
||||
import { SearchResultSet } from '@backstage/search-common';
|
||||
|
||||
// Warning: (ae-missing-release-tag) "DatabaseDocumentStore" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export class DatabaseDocumentStore implements DatabaseStore {
|
||||
constructor(db: Knex);
|
||||
// (undocumented)
|
||||
completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
|
||||
// (undocumented)
|
||||
static create(knex: Knex): Promise<DatabaseDocumentStore>;
|
||||
// (undocumented)
|
||||
insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
type: string,
|
||||
documents: IndexableDocument[],
|
||||
): Promise<void>;
|
||||
// (undocumented)
|
||||
prepareInsert(tx: Knex.Transaction): Promise<void>;
|
||||
// Warning: (ae-forgotten-export) The symbol "DocumentResultRow" needs to be exported by the entry point index.d.ts
|
||||
//
|
||||
// (undocumented)
|
||||
query(
|
||||
tx: Knex.Transaction,
|
||||
{ types, pgTerm, fields }: PgSearchQuery,
|
||||
): Promise<DocumentResultRow[]>;
|
||||
// (undocumented)
|
||||
static supported(knex: Knex): Promise<boolean>;
|
||||
// (undocumented)
|
||||
transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "DatabaseStore" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export interface DatabaseStore {
|
||||
// (undocumented)
|
||||
completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
|
||||
// (undocumented)
|
||||
insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
type: string,
|
||||
documents: IndexableDocument[],
|
||||
): Promise<void>;
|
||||
// (undocumented)
|
||||
prepareInsert(tx: Knex.Transaction): Promise<void>;
|
||||
// (undocumented)
|
||||
query(
|
||||
tx: Knex.Transaction,
|
||||
pgQuery: PgSearchQuery,
|
||||
): Promise<DocumentResultRow[]>;
|
||||
// (undocumented)
|
||||
transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T>;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "PgSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export class PgSearchEngine implements SearchEngine {
|
||||
constructor(databaseStore: DatabaseStore);
|
||||
// (undocumented)
|
||||
static from({
|
||||
database,
|
||||
}: {
|
||||
database: PluginDatabaseManager;
|
||||
}): Promise<PgSearchEngine>;
|
||||
// (undocumented)
|
||||
index(type: string, documents: IndexableDocument[]): Promise<void>;
|
||||
// (undocumented)
|
||||
query(query: SearchQuery): Promise<SearchResultSet>;
|
||||
// (undocumented)
|
||||
setTranslator(translator: (query: SearchQuery) => PgSearchQuery): void;
|
||||
// (undocumented)
|
||||
static supported(database: PluginDatabaseManager): Promise<boolean>;
|
||||
// (undocumented)
|
||||
translator(query: SearchQuery): PgSearchQuery;
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export interface PgSearchQuery {
|
||||
// (undocumented)
|
||||
fields?: Record<string, string | string[]>;
|
||||
// (undocumented)
|
||||
pgTerm?: string;
|
||||
// (undocumented)
|
||||
types?: string[];
|
||||
}
|
||||
|
||||
// Warning: (ae-missing-release-tag) "RawDocumentRow" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
|
||||
//
|
||||
// @public (undocumented)
|
||||
export interface RawDocumentRow {
|
||||
// (undocumented)
|
||||
document: IndexableDocument;
|
||||
// (undocumented)
|
||||
hash: unknown;
|
||||
// (undocumented)
|
||||
type: string;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
```
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.up = async function up(knex) {
|
||||
// This database schema uses some postgres specific features (like tsvector
|
||||
// and jsonb columns) and can not be used with other database engines.
|
||||
await knex.schema.createTable('documents', table => {
|
||||
table.comment('The table of documents');
|
||||
table
|
||||
.text('type')
|
||||
.notNullable()
|
||||
.index()
|
||||
.comment('The index type of the document');
|
||||
table.jsonb('document').notNullable().comment('The document');
|
||||
table.specificType(
|
||||
'body',
|
||||
'tsvector NOT NULL GENERATED ALWAYS AS (' +
|
||||
"setweight(to_tsvector('english', document->>'title'), 'A') || " +
|
||||
"setweight(to_tsvector('english', document->>'text'), 'B') || " +
|
||||
"setweight(to_tsvector('english', document - 'location' - 'title' - 'text'), 'C')" +
|
||||
') STORED',
|
||||
);
|
||||
|
||||
table
|
||||
.index(['body'], 'documents_body_index', 'GIN')
|
||||
.comment('Optimize full text queries on documents');
|
||||
table
|
||||
.index(['document'], 'documents_document_index', 'GIN')
|
||||
.comment('Optimize filter queries on documents');
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.down = async function down(knex) {
|
||||
await knex.schema.dropTable('documents');
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.up = async function up(knex) {
|
||||
// This database migration uses some postgres specific features (like bytea
|
||||
// columns and build in functions) and can not be used with other database
|
||||
// engines.
|
||||
await knex.schema.alterTable('documents', table => {
|
||||
// Extend the documents table with a column that allows to check whether a
|
||||
// document with the same content is already indexed.
|
||||
table.specificType('hash', 'bytea');
|
||||
});
|
||||
|
||||
await knex('documents').update({
|
||||
hash: knex.raw(
|
||||
"sha256(replace(document::text || type, '\\', '\\\\')::bytea)",
|
||||
),
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('documents', table => {
|
||||
table.specificType('hash', 'bytea').notNullable().primary().alter();
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.down = async function down(knex) {
|
||||
await knex.schema.alterTable('documents', table => {
|
||||
table.dropColumn('hash');
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "@backstage/plugin-search-backend-module-pg",
|
||||
"version": "0.1.0",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"license": "Apache-2.0",
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
"main": "dist/index.cjs.js",
|
||||
"types": "dist/index.d.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "backstage-cli backend:dev",
|
||||
"build": "backstage-cli backend:build",
|
||||
"lint": "backstage-cli lint",
|
||||
"test": "backstage-cli test",
|
||||
"prepack": "backstage-cli prepack",
|
||||
"postpack": "backstage-cli postpack",
|
||||
"clean": "backstage-cli clean"
|
||||
},
|
||||
"dependencies": {
|
||||
"@backstage/backend-common": "^0.8.7",
|
||||
"@backstage/search-common": "^0.1.2",
|
||||
"@backstage/plugin-search-backend-node": "^0.4.0",
|
||||
"lodash": "^4.17.15",
|
||||
"knex": "^0.95.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/backend-test-utils": "^0.1.4",
|
||||
"@backstage/cli": "^0.7.6"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"migrations"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { range } from 'lodash';
|
||||
import { DatabaseStore, PgSearchQuery } from '../database';
|
||||
import { PgSearchEngine } from './PgSearchEngine';
|
||||
|
||||
describe('PgSearchEngine', () => {
|
||||
const tx: any = {} as any;
|
||||
let searchEngine: PgSearchEngine;
|
||||
let database: jest.Mocked<DatabaseStore>;
|
||||
|
||||
beforeEach(() => {
|
||||
database = {
|
||||
transaction: jest.fn(),
|
||||
insertDocuments: jest.fn(),
|
||||
query: jest.fn(),
|
||||
completeInsert: jest.fn(),
|
||||
prepareInsert: jest.fn(),
|
||||
};
|
||||
searchEngine = new PgSearchEngine(database);
|
||||
});
|
||||
|
||||
describe('translator', () => {
|
||||
it('query translator invoked', async () => {
|
||||
database.transaction.mockResolvedValue([]);
|
||||
const translatorSpy = jest.fn().mockReturnValue({
|
||||
pgSearchTerm: 'testTerm',
|
||||
});
|
||||
searchEngine.setTranslator(translatorSpy);
|
||||
|
||||
await searchEngine.query({
|
||||
term: 'testTerm',
|
||||
filters: {},
|
||||
pageCursor: '',
|
||||
});
|
||||
|
||||
expect(translatorSpy).toHaveBeenCalledWith({
|
||||
term: 'testTerm',
|
||||
filters: {},
|
||||
pageCursor: '',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return translated query term', async () => {
|
||||
const actualTranslatedQuery = searchEngine.translator({
|
||||
term: 'Hello World',
|
||||
pageCursor: '',
|
||||
}) as PgSearchQuery;
|
||||
|
||||
expect(actualTranslatedQuery).toMatchObject({
|
||||
pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return translated query with filters', async () => {
|
||||
const actualTranslatedQuery = searchEngine.translator({
|
||||
term: 'testTerm',
|
||||
filters: { kind: 'testKind' },
|
||||
types: ['my-filter'],
|
||||
pageCursor: '',
|
||||
}) as PgSearchQuery;
|
||||
|
||||
expect(actualTranslatedQuery).toMatchObject({
|
||||
pgTerm: '("testTerm" | "testTerm":*)',
|
||||
fields: { kind: 'testKind' },
|
||||
types: ['my-filter'],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('insert', () => {
|
||||
it('should insert documents', async () => {
|
||||
database.transaction.mockImplementation(fn => fn(tx));
|
||||
|
||||
const documents = [
|
||||
{ title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' },
|
||||
{
|
||||
location: 'location-2',
|
||||
text: 'Hello World',
|
||||
title: 'Dolor sit amet',
|
||||
},
|
||||
];
|
||||
|
||||
await searchEngine.index('my-type', documents);
|
||||
|
||||
expect(database.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toHaveBeenCalledWith(
|
||||
tx,
|
||||
'my-type',
|
||||
documents,
|
||||
);
|
||||
expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
|
||||
});
|
||||
|
||||
it('should batch insert documents', async () => {
|
||||
database.transaction.mockImplementation(fn => fn(tx));
|
||||
|
||||
const documents = range(350).map(i => ({
|
||||
title: `Hello World ${i}`,
|
||||
text: 'Lorem Ipsum',
|
||||
location: `location-${i}`,
|
||||
}));
|
||||
|
||||
await searchEngine.index('my-type', documents);
|
||||
|
||||
expect(database.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.prepareInsert).toHaveBeenCalledTimes(1);
|
||||
expect(database.insertDocuments).toBeCalledTimes(4);
|
||||
expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
|
||||
});
|
||||
});
|
||||
|
||||
describe('query', () => {
|
||||
it('should perform query', async () => {
|
||||
database.transaction.mockImplementation(fn => fn(tx));
|
||||
database.query.mockResolvedValue([
|
||||
{
|
||||
document: {
|
||||
title: 'Hello World',
|
||||
text: 'Lorem Ipsum',
|
||||
location: 'location-1',
|
||||
},
|
||||
type: 'my-type',
|
||||
},
|
||||
]);
|
||||
|
||||
const results = await searchEngine.query({
|
||||
term: 'Hello World',
|
||||
pageCursor: '',
|
||||
});
|
||||
|
||||
expect(results).toEqual({
|
||||
results: [
|
||||
{
|
||||
document: {
|
||||
title: 'Hello World',
|
||||
text: 'Lorem Ipsum',
|
||||
location: 'location-1',
|
||||
},
|
||||
type: 'my-type',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(database.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(database.query).toHaveBeenCalledWith(tx, {
|
||||
pgTerm: '("Hello" | "Hello":*)&("World" | "World":*)',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { PluginDatabaseManager } from '@backstage/backend-common';
|
||||
import { SearchEngine } from '@backstage/plugin-search-backend-node';
|
||||
import {
|
||||
IndexableDocument,
|
||||
SearchQuery,
|
||||
SearchResultSet,
|
||||
} from '@backstage/search-common';
|
||||
import { chunk } from 'lodash';
|
||||
import {
|
||||
DatabaseDocumentStore,
|
||||
DatabaseStore,
|
||||
PgSearchQuery,
|
||||
} from '../database';
|
||||
|
||||
// TODO: Support paging using page cursor (return cursor and parse cursor)
|
||||
|
||||
export class PgSearchEngine implements SearchEngine {
|
||||
constructor(private readonly databaseStore: DatabaseStore) {}
|
||||
|
||||
static async from({
|
||||
database,
|
||||
}: {
|
||||
database: PluginDatabaseManager;
|
||||
}): Promise<PgSearchEngine> {
|
||||
return new PgSearchEngine(
|
||||
await DatabaseDocumentStore.create(await database.getClient()),
|
||||
);
|
||||
}
|
||||
|
||||
static async supported(database: PluginDatabaseManager): Promise<boolean> {
|
||||
return await DatabaseDocumentStore.supported(await database.getClient());
|
||||
}
|
||||
|
||||
translator(query: SearchQuery): PgSearchQuery {
|
||||
return {
|
||||
pgTerm: query.term
|
||||
.split(/\s/)
|
||||
.map(p => p.trim())
|
||||
.filter(p => p !== '')
|
||||
.map(p => `(${JSON.stringify(p)} | ${JSON.stringify(p)}:*)`)
|
||||
.join('&'),
|
||||
fields: query.filters as Record<string, string | string[]>,
|
||||
types: query.types,
|
||||
};
|
||||
}
|
||||
|
||||
setTranslator(translator: (query: SearchQuery) => PgSearchQuery): void {
|
||||
this.translator = translator;
|
||||
}
|
||||
|
||||
async index(type: string, documents: IndexableDocument[]): Promise<void> {
|
||||
await this.databaseStore.transaction(async tx => {
|
||||
await this.databaseStore.prepareInsert(tx);
|
||||
|
||||
const batchSize = 100;
|
||||
for (const documentBatch of chunk(documents, batchSize)) {
|
||||
await this.databaseStore.insertDocuments(tx, type, documentBatch);
|
||||
}
|
||||
|
||||
await this.databaseStore.completeInsert(tx, type);
|
||||
});
|
||||
}
|
||||
|
||||
async query(query: SearchQuery): Promise<SearchResultSet> {
|
||||
const pgQuery = this.translator(query);
|
||||
|
||||
const rows = await this.databaseStore.transaction(async tx =>
|
||||
this.databaseStore.query(tx, pgQuery),
|
||||
);
|
||||
const results = rows.map(({ type, document }) => ({
|
||||
type,
|
||||
document,
|
||||
}));
|
||||
|
||||
return { results };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { PgSearchEngine } from './PgSearchEngine';
|
||||
@@ -0,0 +1,512 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { DatabaseDocumentStore } from './DatabaseDocumentStore';
|
||||
|
||||
describe('DatabaseDocumentStore', () => {
|
||||
describe('unsupported', () => {
|
||||
const databases = TestDatabases.create({
|
||||
ids: ['MYSQL_8', 'POSTGRES_9', 'SQLITE_3'],
|
||||
});
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should return support state, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
const supported = await DatabaseDocumentStore.supported(knex);
|
||||
|
||||
expect(supported).toBe(false);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should fail to create, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
|
||||
await expect(
|
||||
async () => await DatabaseDocumentStore.create(knex),
|
||||
).rejects.toThrow();
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
|
||||
describe('supported', () => {
|
||||
const databases = TestDatabases.create({
|
||||
ids: ['POSTGRES_13'],
|
||||
});
|
||||
|
||||
async function createStore(databaseId: TestDatabaseId) {
|
||||
const knex = await databases.init(databaseId);
|
||||
const store = await DatabaseDocumentStore.create(knex);
|
||||
return { store, knex };
|
||||
}
|
||||
|
||||
if (databases.eachSupportedId().length < 1) {
|
||||
// Only execute tests if at least on database engine is available, e.g. if
|
||||
// not in CI=1. it.each doesn't support an empty array.
|
||||
return;
|
||||
}
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should return support state, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
const supported = await DatabaseDocumentStore.supported(knex);
|
||||
|
||||
expect(supported).toBe(true);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should insert documents, %p',
|
||||
async databaseId => {
|
||||
const { store, knex } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
{
|
||||
title: 'TITLE 1',
|
||||
text: 'TEXT 1',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
{
|
||||
title: 'TITLE 2',
|
||||
text: 'TEXT 2',
|
||||
location: 'LOCATION-2',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
expect(
|
||||
await knex.count('*').where('type', 'my-type').from('documents'),
|
||||
).toEqual([{ count: '2' }]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should insert documents in batches, %p',
|
||||
async databaseId => {
|
||||
const { store, knex } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
{
|
||||
title: 'TITLE 1',
|
||||
text: 'TEXT 1',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
{
|
||||
title: 'TITLE 2',
|
||||
text: 'TEXT 2',
|
||||
location: 'LOCATION-2',
|
||||
},
|
||||
]);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
{
|
||||
title: 'TITLE 3',
|
||||
text: 'TEXT 3',
|
||||
location: 'LOCATION-3',
|
||||
},
|
||||
{
|
||||
title: 'TITLE 4',
|
||||
text: 'TEXT 4',
|
||||
location: 'LOCATION-4',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
expect(
|
||||
await knex.count('*').where('type', 'my-type').from('documents'),
|
||||
).toEqual([{ count: '4' }]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should clear index for type, %p',
|
||||
async databaseId => {
|
||||
const { store, knex } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'test', [
|
||||
{
|
||||
title: 'TITLE 1',
|
||||
text: 'TEXT 1',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'test');
|
||||
});
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
{
|
||||
title: 'TITLE 1',
|
||||
text: 'TEXT 1',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
{
|
||||
title: 'TITLE 2',
|
||||
text: 'TEXT 2',
|
||||
location: 'LOCATION-2',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
expect(
|
||||
await knex.count('*').where('type', 'test').from('documents'),
|
||||
).toEqual([{ count: '1' }]);
|
||||
expect(
|
||||
await knex.count('*').where('type', 'my-type').from('documents'),
|
||||
).toEqual([{ count: '0' }]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'query by term, %p',
|
||||
async databaseId => {
|
||||
const { store } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'test', [
|
||||
{
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
{
|
||||
title: 'Hello World',
|
||||
text: 'Around the world',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'test');
|
||||
});
|
||||
|
||||
const rows = await store.transaction(tx =>
|
||||
store.query(tx, { pgTerm: 'Hello & World' }),
|
||||
);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Around the world',
|
||||
title: 'Hello World',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'test',
|
||||
},
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Hello World',
|
||||
title: 'Lorem Ipsum',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'test',
|
||||
},
|
||||
]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'query by term for specific type, %p',
|
||||
async databaseId => {
|
||||
const { store } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
{
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'test', [
|
||||
{
|
||||
title: 'Hello World',
|
||||
text: 'Around the world',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'test');
|
||||
});
|
||||
|
||||
const rows = await store.transaction(tx =>
|
||||
store.query(tx, { pgTerm: 'Hello & World', types: ['my-type'] }),
|
||||
);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Hello World',
|
||||
title: 'Lorem Ipsum',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'query by term and filter by field, %p',
|
||||
async databaseId => {
|
||||
const { store } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
({
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
({
|
||||
title: 'Dolor sit amet',
|
||||
text: 'Hello World',
|
||||
myField: 'that',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
{
|
||||
title: 'Hello World',
|
||||
text: 'Around the world',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
const rows = await store.transaction(tx =>
|
||||
store.query(tx, {
|
||||
pgTerm: 'Hello & World',
|
||||
fields: { myField: 'this' },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Hello World',
|
||||
title: 'Lorem Ipsum',
|
||||
myField: 'this',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'query by term and filter by field (any of), %p',
|
||||
async databaseId => {
|
||||
const { store } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
({
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
({
|
||||
title: 'Dolor sit amet',
|
||||
text: 'Hello World',
|
||||
myField: 'that',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
{
|
||||
title: 'Hello World',
|
||||
text: 'Around the world',
|
||||
location: 'LOCATION-1',
|
||||
},
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
const rows = await store.transaction(tx =>
|
||||
store.query(tx, {
|
||||
pgTerm: 'Hello & World',
|
||||
fields: { myField: ['this', 'that'] },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Hello World',
|
||||
title: 'Lorem Ipsum',
|
||||
myField: 'this',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Hello World',
|
||||
title: 'Dolor sit amet',
|
||||
myField: 'that',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'query by term and filter by fields, %p',
|
||||
async databaseId => {
|
||||
const { store } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
({
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
otherField: 'another',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
({
|
||||
title: 'Dolor sit amet',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
otherField: 'unknown',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
const rows = await store.transaction(tx =>
|
||||
store.query(tx, {
|
||||
pgTerm: 'Hello & World',
|
||||
fields: { myField: 'this', otherField: 'another' },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
document: {
|
||||
location: 'LOCATION-1',
|
||||
text: 'Hello World',
|
||||
title: 'Lorem Ipsum',
|
||||
myField: 'this',
|
||||
otherField: 'another',
|
||||
},
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'query without term and filter by field, %p',
|
||||
async databaseId => {
|
||||
const { store } = await createStore(databaseId);
|
||||
|
||||
await store.transaction(async tx => {
|
||||
await store.prepareInsert(tx);
|
||||
await store.insertDocuments(tx, 'my-type', [
|
||||
({
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
({
|
||||
title: 'Dolor sit amet',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
]);
|
||||
await store.completeInsert(tx, 'my-type');
|
||||
});
|
||||
|
||||
const rows = await store.transaction(tx =>
|
||||
store.query(tx, {
|
||||
fields: { myField: 'this' },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
document: ({
|
||||
title: 'Lorem Ipsum',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
{
|
||||
document: ({
|
||||
title: 'Dolor sit amet',
|
||||
text: 'Hello World',
|
||||
myField: 'this',
|
||||
location: 'LOCATION-1',
|
||||
} as unknown) as IndexableDocument,
|
||||
rank: expect.any(Number),
|
||||
type: 'my-type',
|
||||
},
|
||||
]);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { resolvePackagePath } from '@backstage/backend-common';
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Knex } from 'knex';
|
||||
import {
|
||||
DatabaseStore,
|
||||
DocumentResultRow,
|
||||
PgSearchQuery,
|
||||
RawDocumentRow,
|
||||
} from './types';
|
||||
import { queryPostgresMajorVersion } from './util';
|
||||
|
||||
const migrationsDir = resolvePackagePath(
|
||||
'@backstage/plugin-search-backend-module-pg',
|
||||
'migrations',
|
||||
);
|
||||
|
||||
export class DatabaseDocumentStore implements DatabaseStore {
|
||||
static async create(knex: Knex): Promise<DatabaseDocumentStore> {
|
||||
try {
|
||||
const majorVersion = await queryPostgresMajorVersion(knex);
|
||||
|
||||
if (majorVersion < 11) {
|
||||
// We are using some features (like generated columns) that aren't
|
||||
// available in older postgres versions.
|
||||
throw new Error(
|
||||
`The PgSearchEngine requires at least postgres version 11 (but is running on ${majorVersion})`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Actually both mysql and sqlite have a full text search, too. We could
|
||||
// implement them separately or add them here.
|
||||
throw new Error(
|
||||
'The PgSearchEngine is only supported when using a postgres database (>=11.x)',
|
||||
);
|
||||
}
|
||||
|
||||
await knex.migrate.latest({
|
||||
directory: migrationsDir,
|
||||
});
|
||||
return new DatabaseDocumentStore(knex);
|
||||
}
|
||||
|
||||
static async supported(knex: Knex): Promise<boolean> {
|
||||
try {
|
||||
const majorVersion = await queryPostgresMajorVersion(knex);
|
||||
|
||||
return majorVersion >= 11;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
constructor(private readonly db: Knex) {}
|
||||
|
||||
async transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T> {
|
||||
return await this.db.transaction(fn);
|
||||
}
|
||||
|
||||
async prepareInsert(tx: Knex.Transaction): Promise<void> {
|
||||
// We create a temporary table to collect the hashes of the documents that
|
||||
// we expect to be in the documents table at the end. The table is deleted
|
||||
// at the end of the transaction.
|
||||
// The hash makes sure that we generate a new row for every change.
|
||||
await tx.raw(
|
||||
'CREATE TEMP TABLE documents_to_insert (' +
|
||||
'type text NOT NULL, ' +
|
||||
'document jsonb NOT NULL, ' +
|
||||
// Generating the hash requires a trick, as the text to bytea
|
||||
// conversation runs into errors in case the text contains a backslash.
|
||||
// Therefore we have to escape them.
|
||||
"hash bytea NOT NULL GENERATED ALWAYS AS (sha256(replace(document::text || type, '\\', '\\\\')::bytea)) STORED" +
|
||||
') ON COMMIT DROP',
|
||||
);
|
||||
}
|
||||
|
||||
async completeInsert(tx: Knex.Transaction, type: string): Promise<void> {
|
||||
// Copy all new rows into the documents table
|
||||
await tx
|
||||
.insert(
|
||||
tx<RawDocumentRow>('documents_to_insert').select(
|
||||
'type',
|
||||
'document',
|
||||
'hash',
|
||||
),
|
||||
)
|
||||
.into(tx.raw('documents (type, document, hash)'))
|
||||
.onConflict('hash')
|
||||
.ignore();
|
||||
|
||||
// Delete all documents that we don't expect (deleted and changed)
|
||||
await tx<RawDocumentRow>('documents')
|
||||
.where({ type })
|
||||
.whereNotIn(
|
||||
'hash',
|
||||
tx<RawDocumentRow>('documents_to_insert').select('hash'),
|
||||
)
|
||||
.delete();
|
||||
}
|
||||
|
||||
async insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
type: string,
|
||||
documents: IndexableDocument[],
|
||||
): Promise<void> {
|
||||
// Insert all documents into the temporary table to process them later
|
||||
await tx<DocumentResultRow>('documents_to_insert').insert(
|
||||
documents.map(document => ({
|
||||
type,
|
||||
document,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
async query(
|
||||
tx: Knex.Transaction,
|
||||
{ types, pgTerm, fields }: PgSearchQuery,
|
||||
): Promise<DocumentResultRow[]> {
|
||||
// Builds a query like:
|
||||
// SELECT ts_rank_cd(body, query) AS rank, type, document
|
||||
// FROM documents, to_tsquery('english', 'consent') query
|
||||
// WHERE query @@ body AND (document @> '{"kind": "API"}')
|
||||
// ORDER BY rank DESC
|
||||
// LIMIT 10;
|
||||
const query = tx<DocumentResultRow>('documents');
|
||||
|
||||
if (pgTerm) {
|
||||
query
|
||||
.from(tx.raw("documents, to_tsquery('english', ?) query", pgTerm))
|
||||
.whereRaw('query @@ body');
|
||||
} else {
|
||||
query.from('documents');
|
||||
}
|
||||
|
||||
if (types) {
|
||||
query.whereIn('type', types);
|
||||
}
|
||||
|
||||
if (fields) {
|
||||
Object.keys(fields).forEach(key => {
|
||||
const value = fields[key];
|
||||
const valueArray = Array.isArray(value) ? value : [value];
|
||||
const valueCompare = valueArray
|
||||
.map(v => ({ [key]: v }))
|
||||
.map(v => JSON.stringify(v));
|
||||
query.whereRaw(
|
||||
`(${valueCompare.map(() => 'document @> ?').join(' OR ')})`,
|
||||
valueCompare,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
query.select('type', 'document');
|
||||
|
||||
if (pgTerm) {
|
||||
query
|
||||
.select(tx.raw('ts_rank_cd(body, query) AS "rank"'))
|
||||
.orderBy('rank', 'desc');
|
||||
} else {
|
||||
query.select(tx.raw('1 as rank'));
|
||||
}
|
||||
|
||||
return await query.limit(100);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { DatabaseDocumentStore } from './DatabaseDocumentStore';
|
||||
export type { DatabaseStore, PgSearchQuery, RawDocumentRow } from './types';
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { IndexableDocument } from '@backstage/search-common';
|
||||
import { Knex } from 'knex';
|
||||
|
||||
export interface PgSearchQuery {
|
||||
fields?: Record<string, string | string[]>;
|
||||
types?: string[];
|
||||
pgTerm?: string;
|
||||
}
|
||||
|
||||
export interface DatabaseStore {
|
||||
transaction<T>(fn: (tx: Knex.Transaction) => Promise<T>): Promise<T>;
|
||||
prepareInsert(tx: Knex.Transaction): Promise<void>;
|
||||
insertDocuments(
|
||||
tx: Knex.Transaction,
|
||||
type: string,
|
||||
documents: IndexableDocument[],
|
||||
): Promise<void>;
|
||||
completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
|
||||
query(
|
||||
tx: Knex.Transaction,
|
||||
pgQuery: PgSearchQuery,
|
||||
): Promise<DocumentResultRow[]>;
|
||||
}
|
||||
|
||||
export interface RawDocumentRow {
|
||||
document: IndexableDocument;
|
||||
type: string;
|
||||
hash: unknown;
|
||||
}
|
||||
|
||||
export interface DocumentResultRow {
|
||||
document: IndexableDocument;
|
||||
type: string;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { TestDatabases } from '@backstage/backend-test-utils';
|
||||
import { queryPostgresMajorVersion } from './util';
|
||||
|
||||
describe('util', () => {
|
||||
describe('unsupported', () => {
|
||||
const databases = TestDatabases.create({
|
||||
ids: ['SQLITE_3', 'MYSQL_8'],
|
||||
});
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should fail on get postgres major version, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
|
||||
await expect(
|
||||
async () => await queryPostgresMajorVersion(knex),
|
||||
).rejects.toThrow();
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
|
||||
describe('supported', () => {
|
||||
const databases = TestDatabases.create({
|
||||
ids: ['POSTGRES_13', 'POSTGRES_9'],
|
||||
});
|
||||
|
||||
if (databases.eachSupportedId().length < 1) {
|
||||
// Only execute tests if at least on database engine is available, e.g. if
|
||||
// not in CI=1. it.each doesn't support an empty array.
|
||||
return;
|
||||
}
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should get postgres major version, %p',
|
||||
async databaseId => {
|
||||
const knex = await databases.init(databaseId);
|
||||
const expectedVersion = +databaseId.substr(9);
|
||||
|
||||
await expect(queryPostgresMajorVersion(knex)).resolves.toBe(
|
||||
expectedVersion,
|
||||
);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Knex } from 'knex';
|
||||
|
||||
export async function queryPostgresMajorVersion(knex: Knex): Promise<number> {
|
||||
if (knex.client.config.client !== 'pg') {
|
||||
throw new Error("Can't resolve version, not a postgres database");
|
||||
}
|
||||
|
||||
const { rows } = await knex.raw('SHOW server_version_num');
|
||||
const [result] = rows;
|
||||
const version = +result.server_version_num;
|
||||
const majorVersion = Math.floor(version / 10000);
|
||||
return majorVersion;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export * from './database';
|
||||
export * from './PgSearchEngine';
|
||||
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export {};
|
||||
Reference in New Issue
Block a user