diff --git a/.changeset/popular-beds-invent.md b/.changeset/popular-beds-invent.md new file mode 100644 index 0000000000..61d0cf0545 --- /dev/null +++ b/.changeset/popular-beds-invent.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Index templates can now be configured through configuration under the `search.elasticsearch.indexTemplates`. In addition, the `ElasticSearchSearchEngine.fromConfig` now also accepts a `LoggerService` as the `logger` option as well as a new `translator` option. + +The alpha `searchModuleElasticsearchEngine` export no longer accepts options and a new alpha `elasticsearchTranslatorExtensionPoint` export has been added which lets you customize the query translator. diff --git a/plugins/search-backend-module-elasticsearch/alpha-api-report.md b/plugins/search-backend-module-elasticsearch/alpha-api-report.md index 91007c20de..9e5e7c2136 100644 --- a/plugins/search-backend-module-elasticsearch/alpha-api-report.md +++ b/plugins/search-backend-module-elasticsearch/alpha-api-report.md @@ -4,19 +4,20 @@ ```ts import { BackendFeature } from '@backstage/backend-plugin-api'; -import { ElasticSearchCustomIndexTemplate } from '@backstage/plugin-search-backend-module-elasticsearch'; import { ElasticSearchQueryTranslator } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; + +// @alpha (undocumented) +export interface ElasticSearchQueryTranslatorExtensionPoint { + // (undocumented) + setTranslator(translator: ElasticSearchQueryTranslator): void; +} // @alpha -export const searchModuleElasticsearchEngine: ( - options?: SearchModuleElasticsearchEngineOptions | undefined, -) => BackendFeature; +export const elasticsearchTranslatorExtensionPoint: ExtensionPoint; // @alpha -export type SearchModuleElasticsearchEngineOptions = { - translator?: ElasticSearchQueryTranslator; - indexTemplate?: ElasticSearchCustomIndexTemplate; -}; +export const searchModuleElasticsearchEngine: () => BackendFeature; // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index b567414b22..408ec288d0 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -15,6 +15,7 @@ import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; +import { LoggerService } from '@backstage/backend-plugin-api'; import { Readable } from 'stream'; import { SearchEngine } from '@backstage/plugin-search-common'; import { SearchQuery } from '@backstage/plugin-search-common'; @@ -299,10 +300,11 @@ export interface ElasticSearchNodeOptions { // @public export type ElasticSearchOptions = { - logger: Logger; + logger: Logger | LoggerService; config: Config; aliasPostfix?: string; indexPrefix?: string; + translator?: ElasticSearchQueryTranslator; }; // @public @@ -322,7 +324,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { elasticSearchClientOptions: ElasticSearchClientOptions, aliasPostfix: string, indexPrefix: string, - logger: Logger, + logger: Logger | LoggerService, batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ); @@ -365,7 +367,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { indexPrefix: string; indexSeparator: string; alias: string; - logger: Logger; + logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; }; diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 16190e7ed0..d7fdd0cd1b 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -43,6 +43,33 @@ export interface Config { */ fragmentDelimiter?: string; }; + + /** Elasticsearch specific index template bodies */ + indexTemplates?: Array<{ + name: string; + body: { + /** + * Array of wildcard (*) expressions used to match the names of data streams and indices during creation. + */ + index_patterns: string[]; + + /** + * An ordered list of component template names. + * Component templates are merged in the order specified, + * meaning that the last component template specified has the highest precedence. + */ + composed_of?: string[]; + + /** + * See available properties of template + * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body + */ + template?: { + [key: string]: unknown; + }; + }; + }>; + /** Miscellaneous options for the client */ clientOptions?: { ssl?: { @@ -128,7 +155,7 @@ export interface Config { }; /* TODO(kuangp): unsupported until @elastic/elasticsearch@7.14 is released | { - + /** * Bearer authentication token to connect to the cluster. * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html diff --git a/plugins/search-backend-module-elasticsearch/src/alpha.ts b/plugins/search-backend-module-elasticsearch/src/alpha.ts index 7e1b95279a..9801cf4643 100644 --- a/plugins/search-backend-module-elasticsearch/src/alpha.ts +++ b/plugins/search-backend-module-elasticsearch/src/alpha.ts @@ -16,59 +16,66 @@ import { coreServices, createBackendModule, + createExtensionPoint, } from '@backstage/backend-plugin-api'; -import { loggerToWinstonLogger } from '@backstage/backend-common'; import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha'; - import { - ElasticSearchCustomIndexTemplate, ElasticSearchQueryTranslator, ElasticSearchSearchEngine, } from '@backstage/plugin-search-backend-module-elasticsearch'; -/** - * @alpha - * Options for {@link searchModuleElasticsearchEngine}. - */ -export type SearchModuleElasticsearchEngineOptions = { - translator?: ElasticSearchQueryTranslator; - indexTemplate?: ElasticSearchCustomIndexTemplate; -}; +/** @alpha */ +export interface ElasticSearchQueryTranslatorExtensionPoint { + setTranslator(translator: ElasticSearchQueryTranslator): void; +} /** + * Extension point used to customize the ElasticSearch query translator. + * * @alpha + */ +export const elasticsearchTranslatorExtensionPoint = + createExtensionPoint({ + id: 'search.elasticsearchEngine.translator', + }); + +/** * Search backend module for the Elasticsearch engine. + * + * @alpha */ -export const searchModuleElasticsearchEngine = createBackendModule( - (options?: SearchModuleElasticsearchEngineOptions) => ({ - moduleId: 'elasticsearchEngine', - pluginId: 'search', - register(env) { - env.registerInit({ - deps: { - searchEngineRegistry: searchEngineRegistryExtensionPoint, - logger: coreServices.logger, - config: coreServices.rootConfig, - }, - async init({ searchEngineRegistry, logger, config }) { - const searchEngine = await ElasticSearchSearchEngine.fromConfig({ - logger: loggerToWinstonLogger(logger), - config: config, - }); +export const searchModuleElasticsearchEngine = createBackendModule({ + moduleId: 'elasticsearchEngine', + pluginId: 'search', + register(env) { + let translator: ElasticSearchQueryTranslator | undefined; - // set custom translator if available - if (options?.translator) { - searchEngine.setTranslator(options.translator); - } + env.registerExtensionPoint(elasticsearchTranslatorExtensionPoint, { + setTranslator(newTranslator) { + if (translator) { + throw new Error( + 'ElasticSearch query translator may only be set once', + ); + } + translator = newTranslator; + }, + }); - // set custom index template if available - if (options?.indexTemplate) { - searchEngine.setIndexTemplate(options.indexTemplate); - } - - searchEngineRegistry.setSearchEngine(searchEngine); - }, - }); - }, - }), -); + env.registerInit({ + deps: { + searchEngineRegistry: searchEngineRegistryExtensionPoint, + logger: coreServices.logger, + config: coreServices.rootConfig, + }, + async init({ searchEngineRegistry, logger, config }) { + searchEngineRegistry.setSearchEngine( + await ElasticSearchSearchEngine.fromConfig({ + logger, + config, + translator, + }), + ); + }, + }); + }, +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 90f6f407d9..9b085b52b0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -38,6 +38,7 @@ import { AwsCredentialProvider, DefaultAwsCredentialsManager, } from '@backstage/integration-aws-node'; +import { LoggerService } from '@backstage/backend-plugin-api'; export type { ElasticSearchClientOptions }; @@ -73,10 +74,11 @@ export type ElasticSearchQueryTranslator = ( * @public */ export type ElasticSearchOptions = { - logger: Logger; + logger: Logger | LoggerService; config: Config; aliasPostfix?: string; indexPrefix?: string; + translator?: ElasticSearchQueryTranslator; }; /** @@ -126,7 +128,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly elasticSearchClientOptions: ElasticSearchClientOptions, private readonly aliasPostfix: string, private readonly indexPrefix: string, - private readonly logger: Logger, + private readonly logger: Logger | LoggerService, private readonly batchSize: number, highlightOptions?: ElasticSearchHighlightOptions, ) { @@ -149,6 +151,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { config, aliasPostfix = `search`, indexPrefix = ``, + translator, } = options; const credentialProvider = DefaultAwsCredentialsManager.fromConfig(config); const clientOptions = await this.createElasticSearchClientOptions( @@ -165,7 +168,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { logger.info('Initializing ElasticSearch search engine.'); } - return new ElasticSearchSearchEngine( + const engine = new ElasticSearchSearchEngine( clientOptions, aliasPostfix, indexPrefix, @@ -176,6 +179,18 @@ export class ElasticSearchSearchEngine implements SearchEngine { 'search.elasticsearch.highlightOptions', ), ); + + for (const indexTemplate of this.readIndexTemplateConfig( + config.getConfig('search.elasticsearch'), + )) { + await engine.setIndexTemplate(indexTemplate); + } + + if (translator) { + await engine.setTranslator(translator); + } + + return engine; } /** @@ -221,11 +236,10 @@ export class ElasticSearchSearchEngine implements SearchEngine { .boolQuery() .should(value.map(it => esb.matchQuery(key, it.toString()))); } - this.logger.error( - 'Failed to query, unrecognized filter type', + this.logger.error('Failed to query, unrecognized filter type', { key, value, - ); + }); throw new Error( 'Failed to add filters to query. Unrecognized filter type', ); @@ -518,6 +532,24 @@ export class ElasticSearchSearchEngine implements SearchEngine { : {}), }; } + + private static readIndexTemplateConfig( + config: Config, + ): ElasticSearchCustomIndexTemplate[] { + return ( + config.getOptionalConfigArray('indexTemplates')?.map(templateConfig => { + const bodyConfig = templateConfig.getConfig('body'); + return { + name: templateConfig.getString('name'), + body: { + index_patterns: bodyConfig.getStringArray('index_patterns'), + composed_of: bodyConfig.getOptionalStringArray('composed_of'), + template: bodyConfig.getOptionalConfig('template')?.get(), + }, + }; + }) ?? [] + ); + } } /** diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index a9d15c421e..b742216d52 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -19,6 +19,7 @@ import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; import { Readable } from 'stream'; +import { LoggerService } from '@backstage/backend-plugin-api'; /** * Options for instantiate ElasticSearchSearchEngineIndexer @@ -29,7 +30,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { indexPrefix: string; indexSeparator: string; alias: string; - logger: Logger; + logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; }; @@ -55,7 +56,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private readonly indexSeparator: string; private readonly alias: string; private readonly removableAlias: string; - private readonly logger: Logger; + private readonly logger: Logger | LoggerService; private readonly sourceStream: Readable; private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper; private configuredBatchSize: number; @@ -186,7 +187,9 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // If any indices are removable, remove them. Do not bubble up this error, // as doing so would delete the now aliased index. Log instead. if (this.removableIndices.length) { - this.logger.info('Removing stale search indices', this.removableIndices); + this.logger.info('Removing stale search indices', { + removableIndices: this.removableIndices, + }); try { await this.elasticSearchClientWrapper.deleteIndex({ index: this.removableIndices,