diff --git a/.changeset/brown-falcons-own.md b/.changeset/brown-falcons-own.md new file mode 100644 index 0000000000..e38f3bbfbc --- /dev/null +++ b/.changeset/brown-falcons-own.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Truncate long docs to fit PG index size limit diff --git a/plugins/search-backend-module-pg/report.api.md b/plugins/search-backend-module-pg/report.api.md index ff2129b4bb..67e60b7822 100644 --- a/plugins/search-backend-module-pg/report.api.md +++ b/plugins/search-backend-module-pg/report.api.md @@ -24,7 +24,11 @@ export type ConcretePgSearchQuery = { export class DatabaseDocumentStore implements DatabaseStore { constructor(db: Knex); // (undocumented) - completeInsert(tx: Knex.Transaction, type: string): Promise; + completeInsert( + tx: Knex.Transaction, + type: string, + truncate?: boolean, + ): Promise; // (undocumented) static create(database: DatabaseService): Promise; // (undocumented) @@ -51,7 +55,11 @@ export class DatabaseDocumentStore implements DatabaseStore { // @public (undocumented) export interface DatabaseStore { // (undocumented) - completeInsert(tx: Knex.Transaction, type: string): Promise; + completeInsert( + tx: Knex.Transaction, + type: string, + truncate?: boolean, + ): Promise; // (undocumented) getTransaction(): Promise; // (undocumented) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts index 37af4f00f0..7554cebfd7 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts @@ -65,11 +65,50 @@ describe('PgSearchEngineIndexer', () => { 'my-type', documents, ); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type', false); expect(tx.commit).toHaveBeenCalled(); expect(tx.rollback).not.toHaveBeenCalled(); }); + it('should insert documents that are too long for tsvector', async () => { + const documents = [ + { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, + { + location: 'location-2', + text: 'Hello World', + title: 'Dolor sit amet', + }, + ]; + + const tsvectorError = new Error('string is too long for tsvector'); + database.completeInsert.mockRejectedValueOnce(tsvectorError); + + await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledWith( + tx, + 'my-type', + documents, + ); + expect(database.completeInsert).toHaveBeenCalledTimes(2); + expect(database.completeInsert).toHaveBeenNthCalledWith( + 1, + tx, + 'my-type', + false, + ); + expect(database.completeInsert).toHaveBeenNthCalledWith( + 2, + tx, + 'my-type', + true, + ); + expect(tx.commit).toHaveBeenCalled(); + expect(tx.rollback).toHaveBeenCalledTimes(1); + }); + it('should batch insert documents', async () => { const documents = range(350).map(i => ({ title: `Hello World ${i}`, @@ -82,7 +121,7 @@ describe('PgSearchEngineIndexer', () => { expect(database.getTransaction).toHaveBeenCalledTimes(1); expect(database.prepareInsert).toHaveBeenCalledTimes(1); expect(database.insertDocuments).toHaveBeenCalledTimes(4); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type', false); }); it('should rollback transaction if no documents indexed', async () => { diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts index 2e1997c42c..ad8c4f093f 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -85,16 +85,33 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { return; } - // Attempt to complete and commit the transaction. - try { - await this.store.completeInsert(this.tx!, this.type); - this.tx!.commit(); - } catch (e) { - // Otherwise, rollback the transaction and re-throw the error so that the - // stream can be closed and destroyed properly. - this.tx!.rollback!(e); - throw e; - } + // Do not truncate text by default + let retryTruncated = false; + + do { + // Attempt to complete and commit the transaction. + try { + await this.store.completeInsert(this.tx!, this.type, retryTruncated); + this.tx!.commit(); + retryTruncated = false; + } catch (e) { + // Otherwise, rollback the transaction and re-throw the error so that the + // stream can be closed and destroyed properly. + this.tx!.rollback!(e); + + if ( + e instanceof Error && + e.message.includes('string is too long for tsvector') && + !retryTruncated + ) { + // If the error is due to a string being too long for tsvector, we + // retry the operation with truncated text. + retryTruncated = true; + } else { + throw e; + } + } + } while (retryTruncated); } /** diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts index 0029b67bd6..f25351b3a5 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts @@ -22,6 +22,7 @@ import { import { IndexableDocument } from '@backstage/plugin-search-common'; import { PgSearchHighlightOptions } from '../PgSearchEngine'; import { DatabaseDocumentStore } from './DatabaseDocumentStore'; +import { v4 as uuidv4 } from 'uuid'; const highlightOptions: PgSearchHighlightOptions = { preTag: '', @@ -122,6 +123,32 @@ describe('DatabaseDocumentStore', () => { }, ); + it.each(databases.eachSupportedId())( + 'should insert truncated documents, %p', + async databaseId => { + const { store, knex } = await createStore(databaseId); + + await store.transaction(async tx => { + await store.prepareInsert(tx); + + await store.insertDocuments(tx, 'my-type', [ + { + title: 'TITLE 1', + text: Array.from({ length: 100000 }) + .map(() => uuidv4()) // text tokens should be unique to overflow the tsvector indexing and trigger truncation + .join(' '), + location: 'LOCATION-1', + }, + ]); + await store.completeInsert(tx, 'my-type', true); + }); + + expect( + await knex.count('*').where('type', 'my-type').from('documents'), + ).toEqual([{ count: '1' }]); + }, + ); + it.each(databases.eachSupportedId())( 'should insert documents in batches, %p', async databaseId => { diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 8bff8c0a60..0c0e838f0d 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -102,13 +102,23 @@ export class DatabaseDocumentStore implements DatabaseStore { ); } - async completeInsert(tx: Knex.Transaction, type: string): Promise { + async completeInsert( + tx: Knex.Transaction, + type: string, + truncate = false, + ): Promise { + const documentSelect = truncate + ? tx.raw( + `jsonb_set(document, '{text}', to_jsonb(LEFT(document->>'text', 50000))) as document`, + ) + : 'document'; + // Copy all new rows into the documents table await tx .insert( tx('documents_to_insert').select( 'type', - 'document', + documentSelect, 'hash', ), ) @@ -139,7 +149,10 @@ export class DatabaseDocumentStore implements DatabaseStore { await tx('documents_to_insert').insert( documents.map(document => ({ type, - document, + document: { + ...document, + // text: truncateDocsText(document.text), + }, })), ); } diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 5093a170ac..f8529fbebe 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -38,7 +38,11 @@ export interface DatabaseStore { type: string, documents: IndexableDocument[], ): Promise; - completeInsert(tx: Knex.Transaction, type: string): Promise; + completeInsert( + tx: Knex.Transaction, + type: string, + truncate?: boolean, + ): Promise; query( tx: Knex.Transaction, pgQuery: PgSearchQuery,