Merge pull request #11962 from backstage/emmaindal/elasticsearch-search-engine-custom-index-template

[Search - ES module] possibility to set custom index template

.changeset/search-wet-icons-shout.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-elasticsearch': patch
+---
+
+Now possible to set a custom index template on the elasticsearch search engine.

docs/features/search/search-engines.md

@@ -102,6 +102,31 @@ import { Client } from '@elastic/elastic-search';
 const client = searchEngine.newClient(options => new Client(options));
 ```
 
+#### Set custom index template
+
+The elasticsearch engine gives you the ability to set a custom index template if needed.
+
+> Index templates define settings, mappings, and aliases that can be applied automatically to new indices.
+
+```typescript
+// app/backend/src/plugins/search.ts
+const searchEngine = await ElasticSearchSearchEngine.initialize({
+  logger: env.logger,
+  config: env.config,
+});
+
+searchEngine.setIndexTemplate({
+  name: '<name-of-your-custom-template>',
+  body: {
+    index_patterns: ['<your-index-pattern>'],
+    template: {
+      mappings: {},
+      settings: {},
+    },
+  },
+});
+```
+
 ## Example configurations
 
 ### AWS

plugins/search-backend-module-elasticsearch/api-report.md

@@ -137,6 +137,19 @@ export interface ElasticSearchConnectionConstructor {
   };
 }
 
+// @public
+export type ElasticSearchCustomIndexTemplate = {
+  name: string;
+  body: ElasticSearchCustomIndexTemplateBody;
+};
+
+// @public
+export type ElasticSearchCustomIndexTemplateBody = {
+  index_patterns: string[];
+  composed_of?: string[];
+  template?: Record<string, any>;
+};
+
 // @public (undocumented)
 export type ElasticSearchHighlightConfig = {
   fragmentDelimiter: string;
@@ -215,6 +228,8 @@ export class ElasticSearchSearchEngine implements SearchEngine {
   // (undocumented)
   query(query: SearchQuery): Promise<IndexableResultSet>;
   // (undocumented)
+  setIndexTemplate(template: ElasticSearchCustomIndexTemplate): Promise<void>;
+  // (undocumented)
   setTranslator(translator: ElasticSearchQueryTranslator): void;
   // (undocumented)
   protected translator(

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts

@@ -50,6 +50,23 @@ jest.mock('./ElasticSearchSearchEngineIndexer', () => ({
     .mockImplementation(() => indexerMock),
 }));
 
+const customIndexTemplate = {
+  name: 'custom-index-template',
+  body: {
+    index_patterns: ['*'],
+    template: {
+      settings: {
+        number_of_shards: 1,
+      },
+      mappings: {
+        _source: {
+          enabled: false,
+        },
+      },
+    },
+  },
+};
+
 describe('ElasticSearchSearchEngine', () => {
   let testSearchEngine: ElasticSearchSearchEngine;
   let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests;
@@ -72,6 +89,20 @@ describe('ElasticSearchSearchEngine', () => {
     client = testSearchEngine['elasticSearchClient'];
   });
 
+  describe('custom index template', () => {
+    it('should set custom index template', async () => {
+      const indexTemplateSpy = jest.fn().mockReturnValue(customIndexTemplate);
+      mock.add(
+        { method: 'PUT', path: '/_index_template/custom-index-template' },
+        indexTemplateSpy,
+      );
+      await inspectableSearchEngine.setIndexTemplate(customIndexTemplate);
+
+      expect(indexTemplateSpy).toHaveBeenCalled();
+      expect(indexTemplateSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
   describe('queryTranslator', () => {
     beforeAll(() => {
       mock.clearAll();
@@ -760,6 +791,10 @@ describe('ElasticSearchSearchEngine', () => {
   });
 
   describe('indexer', () => {
+    beforeEach(async () => {
+      await testSearchEngine.setIndexTemplate(customIndexTemplate);
+    });
+
     it('should get indexer', async () => {
       const indexer = await testSearchEngine.getIndexer('test-index');
 

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts

@@ -36,6 +36,37 @@ import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineInd
 
 export type { ElasticSearchClientOptions };
 
+/**
+ * Elasticsearch specific index template
+ * @public
+ */
+export type ElasticSearchCustomIndexTemplate = {
+  name: string;
+  body: ElasticSearchCustomIndexTemplateBody;
+};
+
+/**
+ * Elasticsearch specific index template body
+ * @public
+ */
+export type ElasticSearchCustomIndexTemplateBody = {
+  /**
+   * Array of wildcard (*) expressions used to match the names of data streams and indices during creation.
+   */
+  index_patterns: string[];
+  /**
+   * An ordered list of component template names.
+   * Component templates are merged in the order specified,
+   * meaning that the last component template specified has the highest precedence.
+   */
+  composed_of?: string[];
+  /**
+   * See available properties of template
+   * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body
+   */
+  template?: Record<string, any>;
+};
+
 /**
  * Search query that the elasticsearch engine understands.
  * @public
@@ -236,8 +267,18 @@ export class ElasticSearchSearchEngine implements SearchEngine {
     this.translator = translator;
   }
 
+  async setIndexTemplate(template: ElasticSearchCustomIndexTemplate) {
+    try {
+      await this.elasticSearchClient.indices.putIndexTemplate(template);
+      this.logger.info('Custom index template set');
+    } catch (error) {
+      this.logger.error(`Unable to set custom index template: ${error}`);
+    }
+  }
+
   async getIndexer(type: string) {
     const alias = this.constructSearchAlias(type);
+
     const indexer = new ElasticSearchSearchEngineIndexer({
       type,
       indexPrefix: this.indexPrefix,

plugins/search-backend-module-elasticsearch/src/engines/index.ts

@@ -30,6 +30,8 @@ export type {
   ElasticSearchQueryTranslator,
   ElasticSearchQueryTranslatorOptions,
   ElasticSearchOptions,
+  ElasticSearchCustomIndexTemplate,
+  ElasticSearchCustomIndexTemplateBody,
 } from './ElasticSearchSearchEngine';
 export type {
   ElasticSearchSearchEngineIndexer,

plugins/search-backend-module-elasticsearch/src/index.ts

@@ -36,4 +36,6 @@ export type {
   ElasticSearchAuth,
   ElasticSearchSearchEngineIndexer,
   ElasticSearchSearchEngineIndexerOptions,
+  ElasticSearchCustomIndexTemplate,
+  ElasticSearchCustomIndexTemplateBody,
 } from './engines';