diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 75b86cc868..1590d5fb2a 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -114,7 +114,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { mock.add( { method: 'DELETE', - path: '/some-type-index__123tobedeleted%2Csome-type-index__456tobedeleted', + path: '/*', }, deleteSpy, ); @@ -161,7 +161,11 @@ describe('ElasticSearchSearchEngineIndexer', () => { }); // Old index should be cleaned up. - expect(deleteSpy).toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalledWith( + expect.objectContaining({ + path: '/some-type-index__123tobedeleted%2Csome-type-index__456tobedeleted', + }), + ); }); it('handles when no documents are received', async () => { @@ -185,7 +189,11 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(aliasesSpy).not.toHaveBeenCalled(); // Old index should not be cleaned up. - expect(deleteSpy).not.toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalledWith( + expect.objectContaining({ + path: expect.not.stringContaining('tobedeleted'), + }), + ); }); it('handles bulk and batching during indexing', async () => { @@ -240,6 +248,47 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(deleteSpy).not.toHaveBeenCalled(); }); + it('split index deletion in chunks', async () => { + // Generate 200 existing indices + const currentIndices = Array.from( + new Array(200), + (_, i) => `some-type-index__old${i}`, + ); + + const indicesResponse = currentIndices.reduce((acc, curr) => { + return { + ...acc, + [curr]: { mappings: {}, aliases: {} }, + }; + }, {}); + + getSpy = jest.fn().mockReturnValue(indicesResponse); + mock.clear({ + method: 'GET', + path: '/some-type-index__*', + }); + mock.add( + { + method: 'GET', + path: '/some-type-index__*', + }, + getSpy, + ); + + await TestPipeline.fromIndexer(indexer) + .withDocuments([ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]) + .execute(); + + // Delete endpoint should have been called for 4 chunks of 50 (200 indices) + expect(deleteSpy).toHaveBeenCalledTimes(4); + }); + it('handles bulk client rejection', async () => { // Given an ES client wrapper that rejects an error const expectedError = new Error('HTTP Timeout'); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 58773600a7..616c1ca349 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -181,12 +181,33 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { this.logger.info('Removing stale search indices', { removableIndices: this.removableIndices, }); - try { - await this.elasticSearchClientWrapper.deleteIndex({ - index: this.removableIndices, - }); - } catch (e) { - this.logger.warn(`Failed to remove stale search indices: ${e}`); + + // Split the array into chunks of up to 50 indices to handle the case + // where we need to delete a lot of stalled indices + const chunks = this.removableIndices.reduce( + (resultArray, item, index) => { + const chunkIndex = Math.floor(index / 50); + + if (!resultArray[chunkIndex]) { + resultArray[chunkIndex] = []; // start a new chunk + } + + resultArray[chunkIndex].push(item); + + return resultArray; + }, + [] as string[][], + ); + + // Call deleteIndex for each chunk + for (const chunk of chunks) { + try { + await this.elasticSearchClientWrapper.deleteIndex({ + index: chunk, + }); + } catch (e) { + this.logger.warn(`Failed to remove stale search indices: ${e}`); + } } } }