From 1b59892019bf4073cbe8251450742f6e33a64d27 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 10:03:25 +0200 Subject: [PATCH 1/7] implement possibilities to set custom template on elastic search engine Signed-off-by: Emma Indal --- .../engines/ElasticSearchSearchEngine.test.ts | 64 +++++++++++++++++++ .../src/engines/ElasticSearchSearchEngine.ts | 38 +++++++++++ 2 files changed, 102 insertions(+) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 9867920c6e..c1c8ed9c86 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -32,6 +32,9 @@ class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEng getTranslator() { return this.translator; } + getIndexTemplate() { + return this.indexTemplate; + } } const mock = new Mock(); @@ -50,6 +53,23 @@ jest.mock('./ElasticSearchSearchEngineIndexer', () => ({ .mockImplementation(() => indexerMock), })); +const customIndexTemplate = { + name: 'custom-index-template', + body: { + index_patterns: ['*'], + template: { + settings: { + number_of_shards: 1, + }, + mappings: { + _source: { + enabled: false, + }, + }, + }, + }, +}; + describe('ElasticSearchSearchEngine', () => { let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; @@ -72,6 +92,20 @@ describe('ElasticSearchSearchEngine', () => { client = testSearchEngine['elasticSearchClient']; }); + describe('custom index template', () => { + it('should not have custom template set as default', async () => { + expect(inspectableSearchEngine.getIndexTemplate()).toBeUndefined(); + }); + + it('should set custom index template', async () => { + await inspectableSearchEngine.setIndexTemplate(customIndexTemplate); + + expect(inspectableSearchEngine.getIndexTemplate()).toMatchObject( + customIndexTemplate, + ); + }); + }); + describe('queryTranslator', () => { beforeAll(() => { mock.clearAll(); @@ -760,6 +794,10 @@ describe('ElasticSearchSearchEngine', () => { }); describe('indexer', () => { + beforeEach(async () => { + await testSearchEngine.setIndexTemplate(customIndexTemplate); + }); + it('should get indexer', async () => { const indexer = await testSearchEngine.getIndexer('test-index'); @@ -773,12 +811,38 @@ describe('ElasticSearchSearchEngine', () => { elasticSearchClient: client, }), ); + expect(indexerMock.on).toHaveBeenCalledWith( + 'finish', + expect.any(Function), + ); expect(indexerMock.on).toHaveBeenCalledWith( 'error', expect.any(Function), ); }); + describe('onFinish', () => { + let callback: Function; + + beforeEach(async () => { + mock.clearAll(); + await testSearchEngine.getIndexer('test-index'); + callback = indexerMock.on.mock.calls[1][1]; + }); + + it('should set provided index template on finish', async () => { + const indexTemplateSpy = jest.fn().mockReturnValue(customIndexTemplate); + mock.add( + { method: 'PUT', path: '/_index_template/custom-index-template' }, + indexTemplateSpy, + ); + + await callback(); + + expect(indexTemplateSpy).toHaveBeenCalled(); + }); + }); + describe('onError', () => { let errorHandler: Function; const error = new Error('some error'); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 6541d2ed80..025a9fe1fc 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -36,6 +36,24 @@ import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineInd export type { ElasticSearchClientOptions }; +/** + * Elasticsearch specific index template + * @public + */ +export type ElasticSearchCustomIndexTemplate = { + name: string; + body: ElasticSearchIndexTemplateBody; +}; + +/** + * Elasticsearch specific index template body + * @public + */ +export type ElasticSearchIndexTemplateBody = { + index_patterns: string[]; + template: Record; +}; + /** * Search query that the elasticsearch engine understands. * @public @@ -114,6 +132,7 @@ function isBlank(str: string) { export class ElasticSearchSearchEngine implements SearchEngine { private readonly elasticSearchClient: Client; private readonly highlightOptions: ElasticSearchHighlightConfig; + protected indexTemplate?: ElasticSearchCustomIndexTemplate; constructor( private readonly elasticSearchClientOptions: ElasticSearchClientOptions, @@ -236,8 +255,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { this.translator = translator; } + setIndexTemplate(template: ElasticSearchCustomIndexTemplate) { + this.indexTemplate = template; + } + async getIndexer(type: string) { const alias = this.constructSearchAlias(type); + const indexer = new ElasticSearchSearchEngineIndexer({ type, indexPrefix: this.indexPrefix, @@ -266,6 +290,20 @@ export class ElasticSearchSearchEngine implements SearchEngine { } }); + indexer.on('finish', async () => { + // Set custom index template if set + if (this.indexTemplate) { + try { + await this.elasticSearchClient.indices.putIndexTemplate( + this.indexTemplate, + ); + this.logger.info('Custom index template set'); + } catch (error) { + this.logger.error(`Unable to set custom index template: ${error}`); + } + } + }); + return indexer; } From 535847cf14d521e694177ae01464c39fa2548243 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 10:08:48 +0200 Subject: [PATCH 2/7] changeset and api report updates Signed-off-by: Emma Indal --- .changeset/wet-icons-shout.md | 5 +++++ .../api-report.md | 16 ++++++++++++++++ .../src/engines/ElasticSearchSearchEngine.ts | 6 ++++-- .../src/engines/index.ts | 2 ++ .../src/index.ts | 2 ++ 5 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 .changeset/wet-icons-shout.md diff --git a/.changeset/wet-icons-shout.md b/.changeset/wet-icons-shout.md new file mode 100644 index 0000000000..de105cbde9 --- /dev/null +++ b/.changeset/wet-icons-shout.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Now possible to set a custom index template on the elasticsearch search engine. diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 143d3ab3e8..a6be34dcb1 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -137,6 +137,18 @@ export interface ElasticSearchConnectionConstructor { }; } +// @public +export type ElasticSearchCustomIndexTemplate = { + name: string; + body: ElasticSearchCustomIndexTemplateBody; +}; + +// @public +export type ElasticSearchCustomIndexTemplateBody = { + index_patterns: string[]; + template: Record; +}; + // @public (undocumented) export type ElasticSearchHighlightConfig = { fragmentDelimiter: string; @@ -211,10 +223,14 @@ export class ElasticSearchSearchEngine implements SearchEngine { }: ElasticSearchOptions): Promise; // (undocumented) getIndexer(type: string): Promise; + // (undocumented) + protected indexTemplate?: ElasticSearchCustomIndexTemplate; newClient(create: (options: ElasticSearchClientOptions) => T): T; // (undocumented) query(query: SearchQuery): Promise; // (undocumented) + setIndexTemplate(template: ElasticSearchCustomIndexTemplate): void; + // (undocumented) setTranslator(translator: ElasticSearchQueryTranslator): void; // (undocumented) protected translator( diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 025a9fe1fc..a4b3be6eec 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -42,15 +42,17 @@ export type { ElasticSearchClientOptions }; */ export type ElasticSearchCustomIndexTemplate = { name: string; - body: ElasticSearchIndexTemplateBody; + body: ElasticSearchCustomIndexTemplateBody; }; /** * Elasticsearch specific index template body * @public */ -export type ElasticSearchIndexTemplateBody = { +export type ElasticSearchCustomIndexTemplateBody = { index_patterns: string[]; + // See available properties of template + // https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body template: Record; }; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index d99f96ef72..8ac86970a1 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -30,6 +30,8 @@ export type { ElasticSearchQueryTranslator, ElasticSearchQueryTranslatorOptions, ElasticSearchOptions, + ElasticSearchCustomIndexTemplate, + ElasticSearchCustomIndexTemplateBody, } from './ElasticSearchSearchEngine'; export type { ElasticSearchSearchEngineIndexer, diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index 772c7d3689..adf7fa43f2 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -36,4 +36,6 @@ export type { ElasticSearchAuth, ElasticSearchSearchEngineIndexer, ElasticSearchSearchEngineIndexerOptions, + ElasticSearchCustomIndexTemplate, + ElasticSearchCustomIndexTemplateBody, } from './engines'; From 7c26e81120cd6918116496f30664e0cc87d6d6ac Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 10:33:29 +0200 Subject: [PATCH 3/7] add some more docs to the template body type as well as optional composed_of Signed-off-by: Emma Indal --- .../api-report.md | 3 ++- .../src/engines/ElasticSearchSearchEngine.ts | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index a6be34dcb1..dfbb7e493f 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -146,7 +146,8 @@ export type ElasticSearchCustomIndexTemplate = { // @public export type ElasticSearchCustomIndexTemplateBody = { index_patterns: string[]; - template: Record; + composed_of?: string[]; + template?: Record; }; // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index a4b3be6eec..01bffdfff2 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -50,10 +50,21 @@ export type ElasticSearchCustomIndexTemplate = { * @public */ export type ElasticSearchCustomIndexTemplateBody = { + /** + * Array of wildcard (*) expressions used to match the names of data streams and indices during creation. + */ index_patterns: string[]; - // See available properties of template - // https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body - template: Record; + /** + * An ordered list of component template names. + * Component templates are merged in the order specified, + * meaning that the last component template specified has the highest precedence. + */ + composed_of?: string[]; + /** + * See available properties of template + * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body + */ + template?: Record; }; /** From ddce23d080673bb9d5b715e3467e29bb2586e2b4 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 11:38:46 +0200 Subject: [PATCH 4/7] prefix changeset with search-* Signed-off-by: Emma Indal --- .changeset/{wet-icons-shout.md => search-wet-icons-shout.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changeset/{wet-icons-shout.md => search-wet-icons-shout.md} (100%) diff --git a/.changeset/wet-icons-shout.md b/.changeset/search-wet-icons-shout.md similarity index 100% rename from .changeset/wet-icons-shout.md rename to .changeset/search-wet-icons-shout.md From 0b8c1b93e25e633701cdbb479db18fd27b86b6fd Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 13:31:46 +0200 Subject: [PATCH 5/7] setIndexTemplate to be directly responsible for calling putIndexTemplate Signed-off-by: Emma Indal --- .../engines/ElasticSearchSearchEngine.test.ts | 43 +++---------------- .../src/engines/ElasticSearchSearchEngine.ts | 24 +++-------- 2 files changed, 14 insertions(+), 53 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index c1c8ed9c86..c114406d99 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -32,9 +32,6 @@ class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEng getTranslator() { return this.translator; } - getIndexTemplate() { - return this.indexTemplate; - } } const mock = new Mock(); @@ -93,16 +90,16 @@ describe('ElasticSearchSearchEngine', () => { }); describe('custom index template', () => { - it('should not have custom template set as default', async () => { - expect(inspectableSearchEngine.getIndexTemplate()).toBeUndefined(); - }); - it('should set custom index template', async () => { + const indexTemplateSpy = jest.fn().mockReturnValue(customIndexTemplate); + mock.add( + { method: 'PUT', path: '/_index_template/custom-index-template' }, + indexTemplateSpy, + ); await inspectableSearchEngine.setIndexTemplate(customIndexTemplate); - expect(inspectableSearchEngine.getIndexTemplate()).toMatchObject( - customIndexTemplate, - ); + expect(indexTemplateSpy).toHaveBeenCalled(); + expect(indexTemplateSpy).toHaveBeenCalledTimes(1); }); }); @@ -811,38 +808,12 @@ describe('ElasticSearchSearchEngine', () => { elasticSearchClient: client, }), ); - expect(indexerMock.on).toHaveBeenCalledWith( - 'finish', - expect.any(Function), - ); expect(indexerMock.on).toHaveBeenCalledWith( 'error', expect.any(Function), ); }); - describe('onFinish', () => { - let callback: Function; - - beforeEach(async () => { - mock.clearAll(); - await testSearchEngine.getIndexer('test-index'); - callback = indexerMock.on.mock.calls[1][1]; - }); - - it('should set provided index template on finish', async () => { - const indexTemplateSpy = jest.fn().mockReturnValue(customIndexTemplate); - mock.add( - { method: 'PUT', path: '/_index_template/custom-index-template' }, - indexTemplateSpy, - ); - - await callback(); - - expect(indexTemplateSpy).toHaveBeenCalled(); - }); - }); - describe('onError', () => { let errorHandler: Function; const error = new Error('some error'); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 01bffdfff2..d11d8a1f90 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -145,7 +145,6 @@ function isBlank(str: string) { export class ElasticSearchSearchEngine implements SearchEngine { private readonly elasticSearchClient: Client; private readonly highlightOptions: ElasticSearchHighlightConfig; - protected indexTemplate?: ElasticSearchCustomIndexTemplate; constructor( private readonly elasticSearchClientOptions: ElasticSearchClientOptions, @@ -268,8 +267,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { this.translator = translator; } - setIndexTemplate(template: ElasticSearchCustomIndexTemplate) { - this.indexTemplate = template; + async setIndexTemplate(template: ElasticSearchCustomIndexTemplate) { + try { + await this.elasticSearchClient.indices.putIndexTemplate(template); + this.logger.info('Custom index template set'); + } catch (error) { + this.logger.error(`Unable to set custom index template: ${error}`); + } } async getIndexer(type: string) { @@ -303,20 +307,6 @@ export class ElasticSearchSearchEngine implements SearchEngine { } }); - indexer.on('finish', async () => { - // Set custom index template if set - if (this.indexTemplate) { - try { - await this.elasticSearchClient.indices.putIndexTemplate( - this.indexTemplate, - ); - this.logger.info('Custom index template set'); - } catch (error) { - this.logger.error(`Unable to set custom index template: ${error}`); - } - } - }); - return indexer; } From d3edcd078476e460db7a0e97e901d58c4170fde2 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 13:41:56 +0200 Subject: [PATCH 6/7] add instructions for how to set a custom index template Signed-off-by: Emma Indal --- docs/features/search/search-engines.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index f1fd4bd62b..9bb8b8c343 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -102,6 +102,31 @@ import { Client } from '@elastic/elastic-search'; const client = searchEngine.newClient(options => new Client(options)); ``` +#### Set custom index template + +The elasticsearch engine gives you the ability to set a custom index template if needed. + +> Index templates define settings, mappings, and aliases that can be applied automatically to new indices. + +```typescript +// app/backend/src/plugins/search.ts +const searchEngine = await ElasticSearchSearchEngine.initialize({ + logger: env.logger, + config: env.config, +}); + +searchEngine.setIndexTemplate({ + name: '', + body: { + index_patterns: [''], + template: { + mappings: {}, + settings: {}, + }, + }, +}); +``` + ## Example configurations ### AWS From 539c7bb4f75b62938acb6315b2f1e0730380b5d4 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 10 Jun 2022 13:43:37 +0200 Subject: [PATCH 7/7] update api report Signed-off-by: Emma Indal --- plugins/search-backend-module-elasticsearch/api-report.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index dfbb7e493f..04b7f96f93 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -224,13 +224,11 @@ export class ElasticSearchSearchEngine implements SearchEngine { }: ElasticSearchOptions): Promise; // (undocumented) getIndexer(type: string): Promise; - // (undocumented) - protected indexTemplate?: ElasticSearchCustomIndexTemplate; newClient(create: (options: ElasticSearchClientOptions) => T): T; // (undocumented) query(query: SearchQuery): Promise; // (undocumented) - setIndexTemplate(template: ElasticSearchCustomIndexTemplate): void; + setIndexTemplate(template: ElasticSearchCustomIndexTemplate): Promise; // (undocumented) setTranslator(translator: ElasticSearchQueryTranslator): void; // (undocumented)