From 95ad2b00648ad81188d68d7375b4c79de5e1d443 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 31 Jan 2025 15:36:19 +0100 Subject: [PATCH 1/4] enable configuration of indexPrefix Signed-off-by: Kiss Miklos --- .../search-backend-module-elasticsearch/config.d.ts | 4 ++++ .../src/engines/ElasticSearchSearchEngineIndexer.ts | 4 ++-- .../search-backend-module-elasticsearch/src/module.ts | 10 +++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index df9852c922..e0717e33cc 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -21,6 +21,10 @@ export interface Config { * Options for ElasticSearch */ elasticsearch?: { + /** + * Prefix to be used for all index creation. Value will be put in front of the index as is. + */ + indexPrefix?: string; /** * Batch size for elastic search indexing tasks. Defaults to 1000. */ diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 3af195bd55..4faa2f86dd 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -140,7 +140,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // does not occur, and clean up the empty index we just created. if (this.processed === 0) { this.logger.warn( - `Index for ${this.type} was not ${ + `Index for ${this.indexName} was not ${ this.removableIndices.length ? 'replaced' : 'created' }: indexer received 0 documents`, ); @@ -158,7 +158,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // stale indices can be referenced for deletion in case initial attempt // fails. Allow errors to bubble up so that we can clean up the created index. this.logger.info( - `Indexing completed for index ${this.type} in ${duration( + `Indexing completed for index ${this.indexName} in ${duration( this.startTimestamp, )}`, result, diff --git a/plugins/search-backend-module-elasticsearch/src/module.ts b/plugins/search-backend-module-elasticsearch/src/module.ts index 827c8d48b8..3d45d19d0c 100644 --- a/plugins/search-backend-module-elasticsearch/src/module.ts +++ b/plugins/search-backend-module-elasticsearch/src/module.ts @@ -76,12 +76,20 @@ export default createBackendModule({ ); return; } - + const indexPrefix = config.getOptionalString( + 'search.elasticsearch.indexPrefix', + ); + if (indexPrefix) { + logger.info( + `Preceeding all indices with index prefix: ${indexPrefix}`, + ); + } searchEngineRegistry.setSearchEngine( await ElasticSearchSearchEngine.fromConfig({ logger, config, translator, + indexPrefix, }), ); }, From 514525df6c31dffccda7d816894fc0ea20171971 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 31 Jan 2025 16:50:29 +0100 Subject: [PATCH 2/4] better log message Signed-off-by: Kiss Miklos --- plugins/search-backend-module-elasticsearch/src/module.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/module.ts b/plugins/search-backend-module-elasticsearch/src/module.ts index 3d45d19d0c..1ed3c3f87c 100644 --- a/plugins/search-backend-module-elasticsearch/src/module.ts +++ b/plugins/search-backend-module-elasticsearch/src/module.ts @@ -80,9 +80,7 @@ export default createBackendModule({ 'search.elasticsearch.indexPrefix', ); if (indexPrefix) { - logger.info( - `Preceeding all indices with index prefix: ${indexPrefix}`, - ); + logger.info(`Index prefix will be used for indices: ${indexPrefix}`); } searchEngineRegistry.setSearchEngine( await ElasticSearchSearchEngine.fromConfig({ From 37e53868a8439cf2b2b9f895c9d4374f4275c237 Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Fri, 31 Jan 2025 16:52:12 +0100 Subject: [PATCH 3/4] add changeset Signed-off-by: Kiss Miklos --- .changeset/calm-files-hunt.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/calm-files-hunt.md diff --git a/.changeset/calm-files-hunt.md b/.changeset/calm-files-hunt.md new file mode 100644 index 0000000000..afe0348420 --- /dev/null +++ b/.changeset/calm-files-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Allow indexPrefix configuration through the app-config.yaml From f84d2583c88cce3dd52b3200be40c2093db1e13e Mon Sep 17 00:00:00 2001 From: Kiss Miklos Date: Wed, 19 Feb 2025 20:27:10 +0100 Subject: [PATCH 4/4] add proper docs Signed-off-by: Kiss Miklos --- .changeset/calm-files-hunt.md | 2 +- docs/features/search/search-engines.md | 17 +++++++++++++++++ .../engines/ElasticSearchSearchEngineIndexer.ts | 8 ++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.changeset/calm-files-hunt.md b/.changeset/calm-files-hunt.md index afe0348420..c7f1ae5ad5 100644 --- a/.changeset/calm-files-hunt.md +++ b/.changeset/calm-files-hunt.md @@ -2,4 +2,4 @@ '@backstage/plugin-search-backend-module-elasticsearch': patch --- -Allow indexPrefix configuration through the app-config.yaml +Allow `indexPrefix` configuration through the `app-config.yaml` diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 8209ae951d..55cab198a0 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -238,3 +238,20 @@ search: ``` > You can also increase the batch size if you are using a large ES instance. + +### Elasticsearch Index Name Customization + +By default, the Elasticsearch indexer creates index names based on their type, a separator, and the current date as a postfix. You can configure a custom prefix for all indices by adding the following section to your app configuration. + +An example of a default index name would look like this: +`software-catalog-index__20250219` + +To prefix all indices with a custom string (e.g., `custom-prefix`), use the following configuration: + +```yaml +search: + elasticsearch: + indexPrefix: custom-prefix- +``` + +After applying this setting, an index name would look like this: `custom-prefix-software-catalog-index__20250219` diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 4faa2f86dd..75b880af67 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -140,7 +140,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // does not occur, and clean up the empty index we just created. if (this.processed === 0) { this.logger.warn( - `Index for ${this.indexName} was not ${ + `Index for ${this.indexName} of ${this.type} was not ${ this.removableIndices.length ? 'replaced' : 'created' }: indexer received 0 documents`, ); @@ -158,9 +158,9 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // stale indices can be referenced for deletion in case initial attempt // fails. Allow errors to bubble up so that we can clean up the created index. this.logger.info( - `Indexing completed for index ${this.indexName} in ${duration( - this.startTimestamp, - )}`, + `Indexing completed for index ${this.indexName} of ${ + this.type + } in ${duration(this.startTimestamp)}`, result, ); await this.elasticSearchClientWrapper.updateAliases({