From b67f4138823d497ae44a8f98a2bbb2c0cbee4b03 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Aug 2023 11:29:39 +0200 Subject: [PATCH 1/7] search-backend-module-elasticsearch: add support for reading index templates from options Signed-off-by: Patrik Oldsberg --- .../config.d.ts | 29 ++++++++++++++++++- .../src/engines/ElasticSearchSearchEngine.ts | 28 +++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 16190e7ed0..d7fdd0cd1b 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -43,6 +43,33 @@ export interface Config { */ fragmentDelimiter?: string; }; + + /** Elasticsearch specific index template bodies */ + indexTemplates?: Array<{ + name: string; + body: { + /** + * Array of wildcard (*) expressions used to match the names of data streams and indices during creation. + */ + index_patterns: string[]; + + /** + * An ordered list of component template names. + * Component templates are merged in the order specified, + * meaning that the last component template specified has the highest precedence. + */ + composed_of?: string[]; + + /** + * See available properties of template + * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body + */ + template?: { + [key: string]: unknown; + }; + }; + }>; + /** Miscellaneous options for the client */ clientOptions?: { ssl?: { @@ -128,7 +155,7 @@ export interface Config { }; /* TODO(kuangp): unsupported until @elastic/elasticsearch@7.14 is released | { - + /** * Bearer authentication token to connect to the cluster. * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 90f6f407d9..51492699a8 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -165,7 +165,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { logger.info('Initializing ElasticSearch search engine.'); } - return new ElasticSearchSearchEngine( + const engine = new ElasticSearchSearchEngine( clientOptions, aliasPostfix, indexPrefix, @@ -176,6 +176,14 @@ export class ElasticSearchSearchEngine implements SearchEngine { 'search.elasticsearch.highlightOptions', ), ); + + for (const indexTemplate of this.readIndexTemplateConfig( + config.getConfig('search.elasticsearch'), + )) { + await engine.setIndexTemplate(indexTemplate); + } + + return engine; } /** @@ -518,6 +526,24 @@ export class ElasticSearchSearchEngine implements SearchEngine { : {}), }; } + + private static readIndexTemplateConfig( + config: Config, + ): ElasticSearchCustomIndexTemplate[] { + return ( + config.getOptionalConfigArray('indexTemplates')?.map(templateConfig => { + const bodyConfig = templateConfig.getConfig('body'); + return { + name: templateConfig.getString('name'), + body: { + index_patterns: bodyConfig.getStringArray('index_patterns'), + composed_of: bodyConfig.getOptionalStringArray('composed_of'), + template: bodyConfig.getOptionalConfig('template')?.get(), + }, + }; + }) ?? [] + ); + } } /** From 7ccf1f90901307d6ccb6e0920f948bf4a9be659b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Aug 2023 11:34:15 +0200 Subject: [PATCH 2/7] search-backend-module-elasticsearch: allow LoggerService to be used instead of Logger Signed-off-by: Patrik Oldsberg --- .../src/engines/ElasticSearchSearchEngine.ts | 10 +++++----- .../src/engines/ElasticSearchSearchEngineIndexer.ts | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 51492699a8..6eb94fbd5b 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -38,6 +38,7 @@ import { AwsCredentialProvider, DefaultAwsCredentialsManager, } from '@backstage/integration-aws-node'; +import { LoggerService } from '@backstage/backend-plugin-api'; export type { ElasticSearchClientOptions }; @@ -73,7 +74,7 @@ export type ElasticSearchQueryTranslator = ( * @public */ export type ElasticSearchOptions = { - logger: Logger; + logger: Logger | LoggerService; config: Config; aliasPostfix?: string; indexPrefix?: string; @@ -126,7 +127,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly elasticSearchClientOptions: ElasticSearchClientOptions, private readonly aliasPostfix: string, private readonly indexPrefix: string, - private readonly logger: Logger, + private readonly logger: Logger | LoggerService, private readonly batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ) { @@ -229,11 +230,10 @@ export class ElasticSearchSearchEngine implements SearchEngine { .boolQuery() .should(value.map(it => esb.matchQuery(key, it.toString()))); } - this.logger.error( - 'Failed to query, unrecognized filter type', + this.logger.error('Failed to query, unrecognized filter type', { key, value, - ); + }); throw new Error( 'Failed to add filters to query. Unrecognized filter type', ); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index a9d15c421e..b742216d52 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -19,6 +19,7 @@ import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; import { Readable } from 'stream'; +import { LoggerService } from '@backstage/backend-plugin-api'; /** * Options for instantiate ElasticSearchSearchEngineIndexer @@ -29,7 +30,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { indexPrefix: string; indexSeparator: string; alias: string; - logger: Logger; + logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; }; @@ -55,7 +56,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private readonly indexSeparator: string; private readonly alias: string; private readonly removableAlias: string; - private readonly logger: Logger; + private readonly logger: Logger | LoggerService; private readonly sourceStream: Readable; private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper; private configuredBatchSize: number; @@ -186,7 +187,9 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // If any indices are removable, remove them. Do not bubble up this error, // as doing so would delete the now aliased index. Log instead. if (this.removableIndices.length) { - this.logger.info('Removing stale search indices', this.removableIndices); + this.logger.info('Removing stale search indices', { + removableIndices: this.removableIndices, + }); try { await this.elasticSearchClientWrapper.deleteIndex({ index: this.removableIndices, From ac09a60a40316393eb043a874e0fb1ad124522ef Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Aug 2023 11:42:06 +0200 Subject: [PATCH 3/7] search-backend-module-elasticsearch: allow translator to be configured through options Signed-off-by: Patrik Oldsberg --- .../src/engines/ElasticSearchSearchEngine.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 6eb94fbd5b..9b085b52b0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -78,6 +78,7 @@ export type ElasticSearchOptions = { config: Config; aliasPostfix?: string; indexPrefix?: string; + translator?: ElasticSearchQueryTranslator; }; /** @@ -150,6 +151,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { config, aliasPostfix = `search`, indexPrefix = ``, + translator, } = options; const credentialProvider = DefaultAwsCredentialsManager.fromConfig(config); const clientOptions = await this.createElasticSearchClientOptions( @@ -184,6 +186,10 @@ export class ElasticSearchSearchEngine implements SearchEngine { await engine.setIndexTemplate(indexTemplate); } + if (translator) { + await engine.setTranslator(translator); + } + return engine; } From 783d4c1939f1c80b371be64f171334c867caa01b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Aug 2023 11:46:37 +0200 Subject: [PATCH 4/7] search-backend-module-elasticsearch: remove options from alpha module Signed-off-by: Patrik Oldsberg --- .../src/alpha.ts | 107 ++++++++++-------- 1 file changed, 57 insertions(+), 50 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts index 7e1b95279a..54f5f07777 100644 --- a/plugins/search-backend-module-elasticsearch/src/alpha.ts +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -17,58 +17,65 @@ import { coreServices, createBackendModule, } from '@backstage/backend-plugin-api'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; - -import { - ElasticSearchCustomIndexTemplate, - ElasticSearchQueryTranslator, - ElasticSearchSearchEngine, -} from '@backstage/plugin-search-backend-module-elasticsearch'; +import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; /** - * @alpha - * Options for {@link searchModuleElasticsearchEngine}. - */ -export type SearchModuleElasticsearchEngineOptions = { - translator?: ElasticSearchQueryTranslator; - indexTemplate?: ElasticSearchCustomIndexTemplate; -}; - -/** - * @alpha * Search backend module for the Elasticsearch engine. + * + * @alpha + * @remarks + * + * If you need to customize the search engine beyond what is supported by the + * static configuration, you can create a custom module using the `ElasticSearchSearchEngine`. + * + * For example, this lets you configure your own query translator: + * + * ```ts + * export const searchModuleElasticsearchEngine = createBackendModule({ + * moduleId: 'elasticsearchEngine', + * pluginId: 'search', + * register(env) { + * env.registerInit({ + * deps: { + * searchEngineRegistry: searchEngineRegistryExtensionPoint, + * logger: coreServices.logger, + * config: coreServices.rootConfig, + * }, + * async init({ searchEngineRegistry, logger, config }) { + * searchEngineRegistry.setSearchEngine( + * await ElasticSearchSearchEngine.fromConfig({ + * logger, + * config, + * translator(query) { + * return ... // translated query + * } + * }), + * ); + * }, + * }); + * }, + * }); + * ``` */ -export const searchModuleElasticsearchEngine = createBackendModule( - (options?: SearchModuleElasticsearchEngineOptions) => ({ - moduleId: 'elasticsearchEngine', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - searchEngineRegistry: searchEngineRegistryExtensionPoint, - logger: coreServices.logger, - config: coreServices.rootConfig, - }, - async init({ searchEngineRegistry, logger, config }) { - const searchEngine = await ElasticSearchSearchEngine.fromConfig({ - logger: loggerToWinstonLogger(logger), - config: config, - }); - - // set custom translator if available - if (options?.translator) { - searchEngine.setTranslator(options.translator); - } - - // set custom index template if available - if (options?.indexTemplate) { - searchEngine.setIndexTemplate(options.indexTemplate); - } - - searchEngineRegistry.setSearchEngine(searchEngine); - }, - }); - }, - }), -); +export const searchModuleElasticsearchEngine = createBackendModule({ + moduleId: 'elasticsearchEngine', + pluginId: 'search', + register(env) { + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + logger: coreServices.logger, + config: coreServices.rootConfig, + }, + async init({ searchEngineRegistry, logger, config }) { + searchEngineRegistry.setSearchEngine( + await ElasticSearchSearchEngine.fromConfig({ + logger, + config, + }), + ); + }, + }); + }, +}); From b0ca08a269728c363740d096131e45399be6d4ce Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Aug 2023 11:49:44 +0200 Subject: [PATCH 5/7] search-backend-module-elasticsearch: update API reports Signed-off-by: Patrik Oldsberg --- .../alpha-api-report.md | 12 +----------- .../api-report.md | 8 +++++--- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/alpha-api-report.md b/plugins/search-backend-module-elasticsearch/alpha-api-report.md index 91007c20de..ecbe3f2700 100644 --- a/plugins/search-backend-module-elasticsearch/alpha-api-report.md +++ b/plugins/search-backend-module-elasticsearch/alpha-api-report.md @@ -4,19 +4,9 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { ElasticSearchCustomIndexTemplate } from '@backstage/plugin-search-backend-module-elasticsearch'; -import { ElasticSearchQueryTranslator } from '@backstage/plugin-search-backend-module-elasticsearch'; // @alpha -export const searchModuleElasticsearchEngine: ( - options?: SearchModuleElasticsearchEngineOptions | undefined, -) => BackendFeature; - -// @alpha -export type SearchModuleElasticsearchEngineOptions = { - translator?: ElasticSearchQueryTranslator; - indexTemplate?: ElasticSearchCustomIndexTemplate; -}; +export const searchModuleElasticsearchEngine: () => BackendFeature; // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index b567414b22..408ec288d0 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -15,6 +15,7 @@ import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; +import { LoggerService } from '@backstage/backend-plugin-api'; import { Readable } from 'stream'; import { SearchEngine } from '@backstage/plugin-search-common'; import { SearchQuery } from '@backstage/plugin-search-common'; @@ -299,10 +300,11 @@ export interface ElasticSearchNodeOptions { // @public export type ElasticSearchOptions = { - logger: Logger; + logger: Logger | LoggerService; config: Config; aliasPostfix?: string; indexPrefix?: string; + translator?: ElasticSearchQueryTranslator; }; // @public @@ -322,7 +324,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { elasticSearchClientOptions: ElasticSearchClientOptions, aliasPostfix: string, indexPrefix: string, - logger: Logger, + logger: Logger | LoggerService, batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ); @@ -365,7 +367,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { indexPrefix: string; indexSeparator: string; alias: string; - logger: Logger; + logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; }; From 631eb3816b4817f1f696e62291329c40f47030e6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Aug 2023 12:02:41 +0200 Subject: [PATCH 6/7] search-backend-module-elasticsearch: add changeset Signed-off-by: Patrik Oldsberg --- .changeset/popular-beds-invent.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/popular-beds-invent.md diff --git a/.changeset/popular-beds-invent.md b/.changeset/popular-beds-invent.md new file mode 100644 index 0000000000..73c28d629e --- /dev/null +++ b/.changeset/popular-beds-invent.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Index templates can now be configured through configuration under the `search.elasticsearch.indexTemplates`. In addition, the `ElasticSearchSearchEngine.fromConfig` now also accepts a `LoggerService` as the `logger` option as well as a new `translator` option. + +The alpha `searchModuleElasticsearchEngine` export no longer accepts options. From 05447283d142ebdeb95cb63a00d9845a6e6a8b6a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 12 Aug 2023 12:01:45 +0200 Subject: [PATCH 7/7] search-backend-module-elasticsearch: switch translator customization to extension point Signed-off-by: Patrik Oldsberg --- .changeset/popular-beds-invent.md | 2 +- .../alpha-api-report.md | 11 +++ .../src/alpha.ts | 68 +++++++++---------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/.changeset/popular-beds-invent.md b/.changeset/popular-beds-invent.md index 73c28d629e..61d0cf0545 100644 --- a/.changeset/popular-beds-invent.md +++ b/.changeset/popular-beds-invent.md @@ -4,4 +4,4 @@ Index templates can now be configured through configuration under the `search.elasticsearch.indexTemplates`. In addition, the `ElasticSearchSearchEngine.fromConfig` now also accepts a `LoggerService` as the `logger` option as well as a new `translator` option. -The alpha `searchModuleElasticsearchEngine` export no longer accepts options. +The alpha `searchModuleElasticsearchEngine` export no longer accepts options and a new alpha `elasticsearchTranslatorExtensionPoint` export has been added which lets you customize the query translator. diff --git a/plugins/search-backend-module-elasticsearch/alpha-api-report.md b/plugins/search-backend-module-elasticsearch/alpha-api-report.md index ecbe3f2700..9e5e7c2136 100644 --- a/plugins/search-backend-module-elasticsearch/alpha-api-report.md +++ b/plugins/search-backend-module-elasticsearch/alpha-api-report.md @@ -4,6 +4,17 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; +import { ElasticSearchQueryTranslator } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; + +// @alpha (undocumented) +export interface ElasticSearchQueryTranslatorExtensionPoint { + // (undocumented) + setTranslator(translator: ElasticSearchQueryTranslator): void; +} + +// @alpha +export const elasticsearchTranslatorExtensionPoint: ExtensionPoint; // @alpha export const searchModuleElasticsearchEngine: () => BackendFeature; diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts index 54f5f07777..9801cf4643 100644 --- a/plugins/search-backend-module-elasticsearch/src/alpha.ts +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -16,52 +16,51 @@ import { coreServices, createBackendModule, + createExtensionPoint, } from '@backstage/backend-plugin-api'; import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; -import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { + ElasticSearchQueryTranslator, + ElasticSearchSearchEngine, +} from '@backstage/plugin-search-backend-module-elasticsearch'; + +/** @alpha */ +export interface ElasticSearchQueryTranslatorExtensionPoint { + setTranslator(translator: ElasticSearchQueryTranslator): void; +} + +/** + * Extension point used to customize the ElasticSearch query translator. + * + * @alpha + */ +export const elasticsearchTranslatorExtensionPoint = + createExtensionPoint({ + id: 'search.elasticsearchEngine.translator', + }); /** * Search backend module for the Elasticsearch engine. * * @alpha - * @remarks - * - * If you need to customize the search engine beyond what is supported by the - * static configuration, you can create a custom module using the `ElasticSearchSearchEngine`. - * - * For example, this lets you configure your own query translator: - * - * ```ts - * export const searchModuleElasticsearchEngine = createBackendModule({ - * moduleId: 'elasticsearchEngine', - * pluginId: 'search', - * register(env) { - * env.registerInit({ - * deps: { - * searchEngineRegistry: searchEngineRegistryExtensionPoint, - * logger: coreServices.logger, - * config: coreServices.rootConfig, - * }, - * async init({ searchEngineRegistry, logger, config }) { - * searchEngineRegistry.setSearchEngine( - * await ElasticSearchSearchEngine.fromConfig({ - * logger, - * config, - * translator(query) { - * return ... // translated query - * } - * }), - * ); - * }, - * }); - * }, - * }); - * ``` */ export const searchModuleElasticsearchEngine = createBackendModule({ moduleId: 'elasticsearchEngine', pluginId: 'search', register(env) { + let translator: ElasticSearchQueryTranslator | undefined; + + env.registerExtensionPoint(elasticsearchTranslatorExtensionPoint, { + setTranslator(newTranslator) { + if (translator) { + throw new Error( + 'ElasticSearch query translator may only be set once', + ); + } + translator = newTranslator; + }, + }); + env.registerInit({ deps: { searchEngineRegistry: searchEngineRegistryExtensionPoint, @@ -73,6 +72,7 @@ export const searchModuleElasticsearchEngine = createBackendModule({ await ElasticSearchSearchEngine.fromConfig({ logger, config, + translator, }), ); },