feat: truncate document text to fit PostgreSQL index size limit (#30943)

Signed-off-by: Stanislav C <150145013+stanislav-c@users.noreply.github.com>

.changeset/brown-falcons-own.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-pg': patch
+---
+
+Truncate long docs to fit PG index size limit

plugins/search-backend-module-pg/report.api.md

@@ -24,7 +24,11 @@ export type ConcretePgSearchQuery = {
 export class DatabaseDocumentStore implements DatabaseStore {
   constructor(db: Knex);
   // (undocumented)
-  completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
+  completeInsert(
+    tx: Knex.Transaction,
+    type: string,
+    truncate?: boolean,
+  ): Promise<void>;
   // (undocumented)
   static create(database: DatabaseService): Promise<DatabaseDocumentStore>;
   // (undocumented)
@@ -51,7 +55,11 @@ export class DatabaseDocumentStore implements DatabaseStore {
 // @public (undocumented)
 export interface DatabaseStore {
   // (undocumented)
-  completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
+  completeInsert(
+    tx: Knex.Transaction,
+    type: string,
+    truncate?: boolean,
+  ): Promise<void>;
   // (undocumented)
   getTransaction(): Promise<Knex.Transaction>;
   // (undocumented)

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts

@@ -65,11 +65,50 @@ describe('PgSearchEngineIndexer', () => {
       'my-type',
       documents,
     );
-    expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
+    expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type', false);
     expect(tx.commit).toHaveBeenCalled();
     expect(tx.rollback).not.toHaveBeenCalled();
   });
 
+  it('should insert documents that are too long for tsvector', async () => {
+    const documents = [
+      { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' },
+      {
+        location: 'location-2',
+        text: 'Hello World',
+        title: 'Dolor sit amet',
+      },
+    ];
+
+    const tsvectorError = new Error('string is too long for tsvector');
+    database.completeInsert.mockRejectedValueOnce(tsvectorError);
+
+    await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute();
+
+    expect(database.getTransaction).toHaveBeenCalledTimes(1);
+    expect(database.prepareInsert).toHaveBeenCalledTimes(1);
+    expect(database.insertDocuments).toHaveBeenCalledWith(
+      tx,
+      'my-type',
+      documents,
+    );
+    expect(database.completeInsert).toHaveBeenCalledTimes(2);
+    expect(database.completeInsert).toHaveBeenNthCalledWith(
+      1,
+      tx,
+      'my-type',
+      false,
+    );
+    expect(database.completeInsert).toHaveBeenNthCalledWith(
+      2,
+      tx,
+      'my-type',
+      true,
+    );
+    expect(tx.commit).toHaveBeenCalled();
+    expect(tx.rollback).toHaveBeenCalledTimes(1);
+  });
+
   it('should batch insert documents', async () => {
     const documents = range(350).map(i => ({
       title: `Hello World ${i}`,
@@ -82,7 +121,7 @@ describe('PgSearchEngineIndexer', () => {
     expect(database.getTransaction).toHaveBeenCalledTimes(1);
     expect(database.prepareInsert).toHaveBeenCalledTimes(1);
     expect(database.insertDocuments).toHaveBeenCalledTimes(4);
-    expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type');
+    expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type', false);
   });
 
   it('should rollback transaction if no documents indexed', async () => {

plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts

@@ -85,16 +85,33 @@ export class PgSearchEngineIndexer extends BatchSearchEngineIndexer {
       return;
     }
 
-    // Attempt to complete and commit the transaction.
-    try {
-      await this.store.completeInsert(this.tx!, this.type);
-      this.tx!.commit();
-    } catch (e) {
-      // Otherwise, rollback the transaction and re-throw the error so that the
-      // stream can be closed and destroyed properly.
-      this.tx!.rollback!(e);
-      throw e;
-    }
+    // Do not truncate text by default
+    let retryTruncated = false;
+
+    do {
+      // Attempt to complete and commit the transaction.
+      try {
+        await this.store.completeInsert(this.tx!, this.type, retryTruncated);
+        this.tx!.commit();
+        retryTruncated = false;
+      } catch (e) {
+        // Otherwise, rollback the transaction and re-throw the error so that the
+        // stream can be closed and destroyed properly.
+        this.tx!.rollback!(e);
+
+        if (
+          e instanceof Error &&
+          e.message.includes('string is too long for tsvector') &&
+          !retryTruncated
+        ) {
+          // If the error is due to a string being too long for tsvector, we
+          // retry the operation with truncated text.
+          retryTruncated = true;
+        } else {
+          throw e;
+        }
+      }
+    } while (retryTruncated);
   }
 
   /**

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.test.ts

@@ -22,6 +22,7 @@ import {
 import { IndexableDocument } from '@backstage/plugin-search-common';
 import { PgSearchHighlightOptions } from '../PgSearchEngine';
 import { DatabaseDocumentStore } from './DatabaseDocumentStore';
+import { v4 as uuidv4 } from 'uuid';
 
 const highlightOptions: PgSearchHighlightOptions = {
   preTag: '<tag>',
@@ -122,6 +123,32 @@ describe('DatabaseDocumentStore', () => {
       },
     );
 
+    it.each(databases.eachSupportedId())(
+      'should insert truncated documents, %p',
+      async databaseId => {
+        const { store, knex } = await createStore(databaseId);
+
+        await store.transaction(async tx => {
+          await store.prepareInsert(tx);
+
+          await store.insertDocuments(tx, 'my-type', [
+            {
+              title: 'TITLE 1',
+              text: Array.from({ length: 100000 })
+                .map(() => uuidv4()) // text tokens should be unique to overflow the tsvector indexing and trigger truncation
+                .join(' '),
+              location: 'LOCATION-1',
+            },
+          ]);
+          await store.completeInsert(tx, 'my-type', true);
+        });
+
+        expect(
+          await knex.count('*').where('type', 'my-type').from('documents'),
+        ).toEqual([{ count: '1' }]);
+      },
+    );
+
     it.each(databases.eachSupportedId())(
       'should insert documents in batches, %p',
       async databaseId => {

plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts

@@ -102,13 +102,23 @@ export class DatabaseDocumentStore implements DatabaseStore {
     );
   }
 
-  async completeInsert(tx: Knex.Transaction, type: string): Promise<void> {
+  async completeInsert(
+    tx: Knex.Transaction,
+    type: string,
+    truncate = false,
+  ): Promise<void> {
+    const documentSelect = truncate
+      ? tx.raw(
+          `jsonb_set(document, '{text}', to_jsonb(LEFT(document->>'text', 50000))) as document`,
+        )
+      : 'document';
+
     // Copy all new rows into the documents table
     await tx
       .insert(
         tx<RawDocumentRow>('documents_to_insert').select(
           'type',
-          'document',
+          documentSelect,
           'hash',
         ),
       )
@@ -139,7 +149,10 @@ export class DatabaseDocumentStore implements DatabaseStore {
     await tx<DocumentResultRow>('documents_to_insert').insert(
       documents.map(document => ({
         type,
-        document,
+        document: {
+          ...document,
+          // text: truncateDocsText(document.text),
+        },
       })),
     );
   }

plugins/search-backend-module-pg/src/database/types.ts

@@ -38,7 +38,11 @@ export interface DatabaseStore {
     type: string,
     documents: IndexableDocument[],
   ): Promise<void>;
-  completeInsert(tx: Knex.Transaction, type: string): Promise<void>;
+  completeInsert(
+    tx: Knex.Transaction,
+    type: string,
+    truncate?: boolean,
+  ): Promise<void>;
   query(
     tx: Knex.Transaction,
     pgQuery: PgSearchQuery,