From b18670184f2bd06240eec0afcbdf8b5ab6416a9a Mon Sep 17 00:00:00 2001 From: Thomas Cardonne Date: Wed, 8 May 2024 00:43:22 +0200 Subject: [PATCH 1/5] feat(search-backend-module-elasticsearch): ensure all stale indices are deleted Signed-off-by: Thomas Cardonne --- .changeset/wild-eyes-grab.md | 10 ++++ .../ElasticSearchClientWrapper.test.ts | 60 +++++++------------ .../src/engines/ElasticSearchClientWrapper.ts | 32 ++++------ .../ElasticSearchSearchEngineIndexer.test.ts | 53 ++++++---------- .../ElasticSearchSearchEngineIndexer.ts | 20 ++----- 5 files changed, 68 insertions(+), 107 deletions(-) create mode 100644 .changeset/wild-eyes-grab.md diff --git a/.changeset/wild-eyes-grab.md b/.changeset/wild-eyes-grab.md new file mode 100644 index 0000000000..d97c92d99e --- /dev/null +++ b/.changeset/wild-eyes-grab.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +--- + +**BREAKING** The ElasticSearch indexer will now delete stale indices matching the indexer's pattern. + +An indexer using the `some-type-index__*` pattern will remove indices matching this pattern after indexation +to prevent stale indices leading to shards exhaustion. + +Note: The ElasticSearch indexer already uses wildcards patterns to remove aliases on these indices. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts index 75f2e36b2b..e888aa4506 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts @@ -28,17 +28,13 @@ jest.mock('@elastic/elasticsearch', () => ({ search: jest .fn() .mockImplementation(async args => ({ client: 'es', args })), - cat: { - aliases: jest - .fn() - .mockImplementation(async args => ({ client: 'es', args })), - }, helpers: { bulk: jest .fn() .mockImplementation(async args => ({ client: 'es', args })), }, indices: { + get: jest.fn().mockImplementation(async args => ({ client: 'es', args })), create: jest .fn() .mockImplementation(async args => ({ client: 'es', args })), @@ -64,17 +60,13 @@ jest.mock('@opensearch-project/opensearch', () => ({ search: jest .fn() .mockImplementation(async args => ({ client: 'os', args })), - cat: { - aliases: jest - .fn() - .mockImplementation(async args => ({ client: 'os', args })), - }, helpers: { bulk: jest .fn() .mockImplementation(async args => ({ client: 'os', args })), }, indices: { + get: jest.fn().mockImplementation(async args => ({ client: 'os', args })), create: jest .fn() .mockImplementation(async args => ({ client: 'os', args })), @@ -154,6 +146,16 @@ describe('ElasticSearchClientWrapper', () => { expect(result.args).toStrictEqual(indexTemplate); }); + it('indexList', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { index: 'xyz-*' }; + const result = (await wrapper.listIndices(input)) as any; + + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(input); + }); + it('indexExists', async () => { const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); @@ -187,20 +189,6 @@ describe('ElasticSearchClientWrapper', () => { expect(result.args).toStrictEqual(input); }); - it('getAliases', async () => { - const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); - - const input = { aliases: ['xyz'] }; - const result = (await wrapper.getAliases(input)) as any; - - // Should call the OpenSearch client with expected input. - expect(result.client).toBe('es'); - expect(result.args).toStrictEqual({ - format: 'json', - name: input.aliases, - }); - }); - it('updateAliases', async () => { const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); @@ -282,6 +270,16 @@ describe('ElasticSearchClientWrapper', () => { expect(result.args).toStrictEqual(indexTemplate); }); + it('indexList', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { index: 'xyz-*' }; + const result = (await wrapper.listIndices(input)) as any; + + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(input); + }); + it('indexExists', async () => { const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); @@ -315,20 +313,6 @@ describe('ElasticSearchClientWrapper', () => { expect(result.args).toStrictEqual(input); }); - it('getAliases', async () => { - const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); - - const input = { aliases: ['xyz'] }; - const result = (await wrapper.getAliases(input)) as any; - - // Should call the OpenSearch client with expected input. - expect(result.client).toBe('os'); - expect(result.args).toStrictEqual({ - format: 'json', - name: input.aliases, - }); - }); - it('updateAliases', async () => { const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts index a7aca3d03f..454e1a6d50 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts @@ -141,6 +141,18 @@ export class ElasticSearchClientWrapper { throw new Error('No client defined'); } + listIndices(options: { index: string }) { + if (this.openSearchClient) { + return this.openSearchClient.indices.get(options); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.indices.get(options); + } + + throw new Error('No client defined'); + } + indexExists(options: { index: string | string[] }) { if (this.openSearchClient) { return this.openSearchClient.indices.exists(options); @@ -177,26 +189,6 @@ export class ElasticSearchClientWrapper { throw new Error('No client defined'); } - getAliases(options: { aliases: string[] }) { - const { aliases } = options; - - if (this.openSearchClient) { - return this.openSearchClient.cat.aliases({ - format: 'json', - name: aliases, - }); - } - - if (this.elasticSearchClient) { - return this.elasticSearchClient.cat.aliases({ - format: 'json', - name: aliases, - }); - } - - throw new Error('No client defined'); - } - updateAliases(options: { actions: ElasticSearchAliasAction[] }) { const filteredActions = options.actions.filter(Boolean); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 44d33c27a5..75b86cc868 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -30,7 +30,7 @@ const clientWrapper = ElasticSearchClientWrapper.fromClientOptions({ describe('ElasticSearchSearchEngineIndexer', () => { let indexer: ElasticSearchSearchEngineIndexer; let bulkSpy: jest.Mock; - let catSpy: jest.Mock; + let getSpy: jest.Mock; let createSpy: jest.Mock; let aliasesSpy: jest.Mock; let deleteSpy: jest.Mock; @@ -68,30 +68,24 @@ describe('ElasticSearchSearchEngineIndexer', () => { refreshSpy, ); - catSpy = jest.fn().mockReturnValue([ - { - alias: 'some-type-index__search', - index: 'some-type-index__123tobedeleted', - filter: '-', - 'routing.index': '-', - 'routing.search': '-', - is_write_index: '-', + getSpy = jest.fn().mockReturnValue({ + 'some-type-index__123tobedeleted': { + aliases: {}, + mappings: {}, + settings: {}, }, - { - alias: 'some-type-index__search_removable', - index: 'some-type-index__456tobedeleted', - filter: '-', - 'routing.index': '-', - 'routing.search': '-', - is_write_index: '-', + 'some-type-index__456tobedeleted': { + aliases: {}, + mappings: {}, + settings: {}, }, - ]); + }); mock.add( { method: 'GET', - path: '/_cat/aliases/some-type-index__search%2Csome-type-index__search_removable', + path: '/some-type-index__*', }, - catSpy, + getSpy, ); createSpy = jest.fn().mockReturnValue({ @@ -143,7 +137,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute(); // Older indices should have been queried for. - expect(catSpy).toHaveBeenCalled(); + expect(getSpy).toHaveBeenCalled(); // A new index should have been created. const createdIndex = createSpy.mock.calls[0][0].path.slice(1); @@ -163,15 +157,6 @@ describe('ElasticSearchSearchEngineIndexer', () => { remove: { index: 'some-type-index__*', alias: 'some-type-index__search' }, }); expect(aliasActions[1]).toStrictEqual({ - add: { - indices: [ - 'some-type-index__123tobedeleted', - 'some-type-index__456tobedeleted', - ], - alias: 'some-type-index__search_removable', - }, - }); - expect(aliasActions[2]).toStrictEqual({ add: { index: createdIndex, alias: 'some-type-index__search' }, }); @@ -183,7 +168,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { await TestPipeline.fromIndexer(indexer).withDocuments([]).execute(); // Older indices should have been queried for. - expect(catSpy).toHaveBeenCalled(); + expect(getSpy).toHaveBeenCalled(); // A new index should have been created. expect(createSpy).toHaveBeenNthCalledWith( @@ -236,17 +221,17 @@ describe('ElasticSearchSearchEngineIndexer', () => { ]; // Update initial alias cat to return nothing. - catSpy = jest.fn().mockReturnValue([]); + getSpy = jest.fn().mockReturnValue({}); mock.clear({ method: 'GET', - path: '/_cat/aliases/some-type-index__search%2Csome-type-index__search_removable', + path: '/some-type-index__*', }); mock.add( { method: 'GET', - path: '/_cat/aliases/some-type-index__search%2Csome-type-index__search_removable', + path: '/some-type-index__*', }, - catSpy, + getSpy, ); await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute(); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 4b1307114a..58773600a7 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -56,7 +56,6 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private readonly indexPrefix: string; private readonly indexSeparator: string; private readonly alias: string; - private readonly removableAlias: string; private readonly logger: Logger | LoggerService; private readonly sourceStream: Readable; private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper; @@ -74,7 +73,6 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { this.indexSeparator = options.indexSeparator; this.indexName = this.constructIndexName(`${Date.now()}`); this.alias = options.alias; - this.removableAlias = `${this.alias}_removable`; this.elasticSearchClientWrapper = options.elasticSearchClientWrapper; // The ES client bulk helper supports stream-based indexing, but we have to @@ -108,13 +106,13 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { async initialize(): Promise { this.logger.info(`Started indexing documents for index ${this.type}`); - const aliases = await this.elasticSearchClientWrapper.getAliases({ - aliases: [this.alias, this.removableAlias], + const indices = await this.elasticSearchClientWrapper.listIndices({ + index: this.constructIndexName('*'), }); - this.removableIndices = [ - ...new Set(aliases.body.map((r: Record) => r.index)), - ] as string[]; + for (const key of Object.keys(indices.body)) { + this.removableIndices.push(key); + } await this.elasticSearchClientWrapper.createIndex({ index: this.indexName, @@ -171,14 +169,6 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { { remove: { index: this.constructIndexName('*'), alias: this.alias }, }, - this.removableIndices.length - ? { - add: { - indices: this.removableIndices, - alias: this.removableAlias, - }, - } - : undefined, { add: { index: this.indexName, alias: this.alias }, }, From 7019b5328e589b04582968b78ccb45dc536bca3f Mon Sep 17 00:00:00 2001 From: Thomas Cardonne Date: Wed, 8 May 2024 01:05:36 +0200 Subject: [PATCH 2/5] fix: regen api-reports Signed-off-by: Thomas Cardonne --- .../api-report.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 053c267f29..2c23c82f76 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -154,18 +154,18 @@ export class ElasticSearchClientWrapper { options: ElasticSearchClientOptions, ): ElasticSearchClientWrapper; // (undocumented) - getAliases(options: { - aliases: string[]; - }): - | TransportRequestPromise, unknown>> - | TransportRequestPromise_2, unknown>>; - // (undocumented) indexExists(options: { index: string | string[]; }): | TransportRequestPromise> | TransportRequestPromise_2>; // (undocumented) + listIndices(options: { + index: string; + }): + | TransportRequestPromise, unknown>> + | TransportRequestPromise_2, unknown>>; + // (undocumented) putIndexTemplate( template: ElasticSearchCustomIndexTemplate, ): From 59a4bea757b3b4170f495b549d9b3b728b589602 Mon Sep 17 00:00:00 2001 From: Thomas Cardonne Date: Wed, 8 May 2024 12:30:16 +0200 Subject: [PATCH 3/5] fix: deprecate getAliases instead of removing it Signed-off-by: Thomas Cardonne --- .../api-report.md | 6 +++ .../ElasticSearchClientWrapper.test.ts | 38 +++++++++++++++++++ .../src/engines/ElasticSearchClientWrapper.ts | 23 +++++++++++ 3 files changed, 67 insertions(+) diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 2c23c82f76..4c55dad91e 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -153,6 +153,12 @@ export class ElasticSearchClientWrapper { static fromClientOptions( options: ElasticSearchClientOptions, ): ElasticSearchClientWrapper; + // @deprecated (undocumented) + getAliases(options: { + aliases: string[]; + }): + | TransportRequestPromise, unknown>> + | TransportRequestPromise_2, unknown>>; // (undocumented) indexExists(options: { index: string | string[]; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts index e888aa4506..dd5a37e5f0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts @@ -28,6 +28,11 @@ jest.mock('@elastic/elasticsearch', () => ({ search: jest .fn() .mockImplementation(async args => ({ client: 'es', args })), + cat: { + aliases: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + }, helpers: { bulk: jest .fn() @@ -60,6 +65,11 @@ jest.mock('@opensearch-project/opensearch', () => ({ search: jest .fn() .mockImplementation(async args => ({ client: 'os', args })), + cat: { + aliases: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + }, helpers: { bulk: jest .fn() @@ -189,6 +199,20 @@ describe('ElasticSearchClientWrapper', () => { expect(result.args).toStrictEqual(input); }); + it('getAliases', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { aliases: ['xyz'] }; + const result = (await wrapper.getAliases(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual({ + format: 'json', + name: input.aliases, + }); + }); + it('updateAliases', async () => { const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); @@ -313,6 +337,20 @@ describe('ElasticSearchClientWrapper', () => { expect(result.args).toStrictEqual(input); }); + it('getAliases', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { aliases: ['xyz'] }; + const result = (await wrapper.getAliases(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual({ + format: 'json', + name: input.aliases, + }); + }); + it('updateAliases', async () => { const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts index 454e1a6d50..4bf8a3d998 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts @@ -177,6 +177,29 @@ export class ElasticSearchClientWrapper { throw new Error('No client defined'); } + /** + * @deprecated unused by the ElasticSearch Engine, will be removed in the future + */ + getAliases(options: { aliases: string[] }) { + const { aliases } = options; + + if (this.openSearchClient) { + return this.openSearchClient.cat.aliases({ + format: 'json', + name: aliases, + }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.cat.aliases({ + format: 'json', + name: aliases, + }); + } + + throw new Error('No client defined'); + } + createIndex(options: { index: string }) { if (this.openSearchClient) { return this.openSearchClient.indices.create(options); From 3a5c6e6d8b44a8b6ac0976a13c57228ea56656b9 Mon Sep 17 00:00:00 2001 From: Thomas Cardonne Date: Fri, 17 May 2024 10:48:42 +0200 Subject: [PATCH 4/5] chore(changeset): add deprecation notice and upgrade task Signed-off-by: Thomas Cardonne --- .changeset/wild-eyes-grab.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.changeset/wild-eyes-grab.md b/.changeset/wild-eyes-grab.md index d97c92d99e..f382c406f3 100644 --- a/.changeset/wild-eyes-grab.md +++ b/.changeset/wild-eyes-grab.md @@ -2,9 +2,13 @@ '@backstage/plugin-search-backend-module-elasticsearch': minor --- -**BREAKING** The ElasticSearch indexer will now delete stale indices matching the indexer's pattern. +**BREAKING**: The ElasticSearch indexer will now delete stale indices matching the indexer's pattern. +The method `getAliases` of `ElasticSearchClientWrapper` has been deprecated and might be removed in future releases. An indexer using the `some-type-index__*` pattern will remove indices matching this pattern after indexation to prevent stale indices leading to shards exhaustion. +Before upgrading ensure that the index pattern doesn't match indices that are not managed by Backstage +and thus shouldn't be deleted. + Note: The ElasticSearch indexer already uses wildcards patterns to remove aliases on these indices. From 5eba6fcd79a286b561cb9b7e09c184bd36b903bd Mon Sep 17 00:00:00 2001 From: Thomas Cardonne Date: Thu, 23 May 2024 14:10:04 +0200 Subject: [PATCH 5/5] feat: handle deletion of many indices in chunks Signed-off-by: Thomas Cardonne --- .../ElasticSearchSearchEngineIndexer.test.ts | 55 ++++++++++++++++++- .../ElasticSearchSearchEngineIndexer.ts | 33 +++++++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 75b86cc868..1590d5fb2a 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -114,7 +114,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { mock.add( { method: 'DELETE', - path: '/some-type-index__123tobedeleted%2Csome-type-index__456tobedeleted', + path: '/*', }, deleteSpy, ); @@ -161,7 +161,11 @@ describe('ElasticSearchSearchEngineIndexer', () => { }); // Old index should be cleaned up. - expect(deleteSpy).toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalledWith( + expect.objectContaining({ + path: '/some-type-index__123tobedeleted%2Csome-type-index__456tobedeleted', + }), + ); }); it('handles when no documents are received', async () => { @@ -185,7 +189,11 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(aliasesSpy).not.toHaveBeenCalled(); // Old index should not be cleaned up. - expect(deleteSpy).not.toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalledWith( + expect.objectContaining({ + path: expect.not.stringContaining('tobedeleted'), + }), + ); }); it('handles bulk and batching during indexing', async () => { @@ -240,6 +248,47 @@ describe('ElasticSearchSearchEngineIndexer', () => { expect(deleteSpy).not.toHaveBeenCalled(); }); + it('split index deletion in chunks', async () => { + // Generate 200 existing indices + const currentIndices = Array.from( + new Array(200), + (_, i) => `some-type-index__old${i}`, + ); + + const indicesResponse = currentIndices.reduce((acc, curr) => { + return { + ...acc, + [curr]: { mappings: {}, aliases: {} }, + }; + }, {}); + + getSpy = jest.fn().mockReturnValue(indicesResponse); + mock.clear({ + method: 'GET', + path: '/some-type-index__*', + }); + mock.add( + { + method: 'GET', + path: '/some-type-index__*', + }, + getSpy, + ); + + await TestPipeline.fromIndexer(indexer) + .withDocuments([ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]) + .execute(); + + // Delete endpoint should have been called for 4 chunks of 50 (200 indices) + expect(deleteSpy).toHaveBeenCalledTimes(4); + }); + it('handles bulk client rejection', async () => { // Given an ES client wrapper that rejects an error const expectedError = new Error('HTTP Timeout'); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 58773600a7..616c1ca349 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -181,12 +181,33 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { this.logger.info('Removing stale search indices', { removableIndices: this.removableIndices, }); - try { - await this.elasticSearchClientWrapper.deleteIndex({ - index: this.removableIndices, - }); - } catch (e) { - this.logger.warn(`Failed to remove stale search indices: ${e}`); + + // Split the array into chunks of up to 50 indices to handle the case + // where we need to delete a lot of stalled indices + const chunks = this.removableIndices.reduce( + (resultArray, item, index) => { + const chunkIndex = Math.floor(index / 50); + + if (!resultArray[chunkIndex]) { + resultArray[chunkIndex] = []; // start a new chunk + } + + resultArray[chunkIndex].push(item); + + return resultArray; + }, + [] as string[][], + ); + + // Call deleteIndex for each chunk + for (const chunk of chunks) { + try { + await this.elasticSearchClientWrapper.deleteIndex({ + index: chunk, + }); + } catch (e) { + this.logger.warn(`Failed to remove stale search indices: ${e}`); + } } } }