feat: handle deletion of many indices in chunks

Signed-off-by: Thomas Cardonne <thomas.cardonne@adevinta.com>

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts

@@ -114,7 +114,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
     mock.add(
       {
         method: 'DELETE',
-        path: '/some-type-index__123tobedeleted%2Csome-type-index__456tobedeleted',
+        path: '/*',
       },
       deleteSpy,
     );
@@ -161,7 +161,11 @@ describe('ElasticSearchSearchEngineIndexer', () => {
     });
 
     // Old index should be cleaned up.
-    expect(deleteSpy).toHaveBeenCalled();
+    expect(deleteSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        path: '/some-type-index__123tobedeleted%2Csome-type-index__456tobedeleted',
+      }),
+    );
   });
 
   it('handles when no documents are received', async () => {
@@ -185,7 +189,11 @@ describe('ElasticSearchSearchEngineIndexer', () => {
     expect(aliasesSpy).not.toHaveBeenCalled();
 
     // Old index should not be cleaned up.
-    expect(deleteSpy).not.toHaveBeenCalled();
+    expect(deleteSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        path: expect.not.stringContaining('tobedeleted'),
+      }),
+    );
   });
 
   it('handles bulk and batching during indexing', async () => {
@@ -240,6 +248,47 @@ describe('ElasticSearchSearchEngineIndexer', () => {
     expect(deleteSpy).not.toHaveBeenCalled();
   });
 
+  it('split index deletion in chunks', async () => {
+    // Generate 200 existing indices
+    const currentIndices = Array.from(
+      new Array(200),
+      (_, i) => `some-type-index__old${i}`,
+    );
+
+    const indicesResponse = currentIndices.reduce((acc, curr) => {
+      return {
+        ...acc,
+        [curr]: { mappings: {}, aliases: {} },
+      };
+    }, {});
+
+    getSpy = jest.fn().mockReturnValue(indicesResponse);
+    mock.clear({
+      method: 'GET',
+      path: '/some-type-index__*',
+    });
+    mock.add(
+      {
+        method: 'GET',
+        path: '/some-type-index__*',
+      },
+      getSpy,
+    );
+
+    await TestPipeline.fromIndexer(indexer)
+      .withDocuments([
+        {
+          title: 'testTerm',
+          text: 'testText',
+          location: 'test/location',
+        },
+      ])
+      .execute();
+
+    // Delete endpoint should have been called for 4 chunks of 50 (200 indices)
+    expect(deleteSpy).toHaveBeenCalledTimes(4);
+  });
+
   it('handles bulk client rejection', async () => {
     // Given an ES client wrapper that rejects an error
     const expectedError = new Error('HTTP Timeout');

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts

@@ -181,12 +181,33 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
       this.logger.info('Removing stale search indices', {
         removableIndices: this.removableIndices,
       });
-      try {
-        await this.elasticSearchClientWrapper.deleteIndex({
-          index: this.removableIndices,
-        });
-      } catch (e) {
-        this.logger.warn(`Failed to remove stale search indices: ${e}`);
+
+      // Split the array into chunks of up to 50 indices to handle the case
+      // where we need to delete a lot of stalled indices
+      const chunks = this.removableIndices.reduce(
+        (resultArray, item, index) => {
+          const chunkIndex = Math.floor(index / 50);
+
+          if (!resultArray[chunkIndex]) {
+            resultArray[chunkIndex] = []; // start a new chunk
+          }
+
+          resultArray[chunkIndex].push(item);
+
+          return resultArray;
+        },
+        [] as string[][],
+      );
+
+      // Call deleteIndex for each chunk
+      for (const chunk of chunks) {
+        try {
+          await this.elasticSearchClientWrapper.deleteIndex({
+            index: chunk,
+          });
+        } catch (e) {
+          this.logger.warn(`Failed to remove stale search indices: ${e}`);
+        }
       }
     }
   }