From 1e1a9fe979802f96160e54e0b1bd1271dae156df Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 22 Dec 2022 14:09:49 +0100 Subject: [PATCH] Prevent the readiness check from looping endlessly Signed-off-by: Eric Peterson --- .changeset/search-break-loop.md | 5 +++++ .../ElasticSearchSearchEngineIndexer.ts | 20 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 .changeset/search-break-loop.md diff --git a/.changeset/search-break-loop.md b/.changeset/search-break-loop.md new file mode 100644 index 0000000000..ce65645900 --- /dev/null +++ b/.changeset/search-break-loop.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Fixed a bug that could cause an indexing process to silently fail, timeout, and accumulate stale indices. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index d1534c53a7..a9d15c421e 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -217,11 +217,27 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // Otherwise, continue periodically checking the stream queue to see if // ES has consumed the documents and continue when it's ready for more. - return new Promise(resolve => { + return new Promise((isReady, abort) => { + let streamLengthChecks = 0; const interval = setInterval(() => { + streamLengthChecks++; + if (this.sourceStream.readableLength < this.configuredBatchSize) { clearInterval(interval); - resolve(); + isReady(); + } + + // Do not allow this interval to loop endlessly; anything longer than 5 + // minutes likely indicates an unrecoverable error in ES; direct the + // user to inspect ES logs for more clues and abort in order to allow + // the index to be cleaned up. + if (streamLengthChecks >= 6000) { + clearInterval(interval); + abort( + new Error( + 'Exceeded 5 minutes waiting for elastic to be ready to accept more documents. Check the elastic logs for possible problems.', + ), + ); } }, 50); });