paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Merge pull request #19288 from backstage/rugvip/no-options-es

Browse Source 
search-backend-module-elasticsearch: refactor + remove options
This commit is contained in:
Patrik Oldsberg
2023-08-15 15:23:26 +02:00
committed by GitHub
parent a3c5095f1c 05447283d1
commit 5c55972e5a
7 changed files with 142 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/popular-beds-invent.md
+7
View File
@@ -0,0 +1,7 @@
---
'@backstage/plugin-search-backend-module-elasticsearch': patch
---
Index templates can now be configured through configuration under the `search.elasticsearch.indexTemplates`. In addition, the `ElasticSearchSearchEngine.fromConfig` now also accepts a `LoggerService` as the `logger` option as well as a new `translator` option.
The alpha `searchModuleElasticsearchEngine` export no longer accepts options and a new alpha `elasticsearchTranslatorExtensionPoint` export has been added which lets you customize the query translator.
plugins/search-backend-module-elasticsearch/alpha-api-report.md
+9 -8
View File
@@ -4,19 +4,20 @@
```ts
import { BackendFeature } from '@backstage/backend-plugin-api';
import { ElasticSearchCustomIndexTemplate } from '@backstage/plugin-search-backend-module-elasticsearch';
import { ElasticSearchQueryTranslator } from '@backstage/plugin-search-backend-module-elasticsearch';
import { ExtensionPoint } from '@backstage/backend-plugin-api';
// @alpha (undocumented)
export interface ElasticSearchQueryTranslatorExtensionPoint {
// (undocumented)
setTranslator(translator: ElasticSearchQueryTranslator): void;
}
// @alpha
export const searchModuleElasticsearchEngine: (
options?: SearchModuleElasticsearchEngineOptions | undefined,
) => BackendFeature;
export const elasticsearchTranslatorExtensionPoint: ExtensionPoint<ElasticSearchQueryTranslatorExtensionPoint>;
// @alpha
export type SearchModuleElasticsearchEngineOptions = {
translator?: ElasticSearchQueryTranslator;
indexTemplate?: ElasticSearchCustomIndexTemplate;
};
export const searchModuleElasticsearchEngine: () => BackendFeature;
// (No @packageDocumentation comment for this package)
```
plugins/search-backend-module-elasticsearch/api-report.md
+5 -3
View File
@@ -15,6 +15,7 @@ import type { ConnectionOptions } from 'tls';
import { IndexableDocument } from '@backstage/plugin-search-common';
import { IndexableResultSet } from '@backstage/plugin-search-common';
import { Logger } from 'winston';
import { LoggerService } from '@backstage/backend-plugin-api';
import { Readable } from 'stream';
import { SearchEngine } from '@backstage/plugin-search-common';
import { SearchQuery } from '@backstage/plugin-search-common';
@@ -299,10 +300,11 @@ export interface ElasticSearchNodeOptions {
// @public
export type ElasticSearchOptions = {
logger: Logger;
logger: Logger | LoggerService;
config: Config;
aliasPostfix?: string;
indexPrefix?: string;
translator?: ElasticSearchQueryTranslator;
};
// @public
@@ -322,7 +324,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
elasticSearchClientOptions: ElasticSearchClientOptions,
aliasPostfix: string,
indexPrefix: string,
logger: Logger,
logger: Logger | LoggerService,
batchSize: number,
highlightOptions?: ElasticSearchHighlightOptions,
);
@@ -365,7 +367,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
indexPrefix: string;
indexSeparator: string;
alias: string;
logger: Logger;
logger: Logger | LoggerService;
elasticSearchClientWrapper: ElasticSearchClientWrapper;
batchSize: number;
};
plugins/search-backend-module-elasticsearch/config.d.ts
Vendored
+28 -1
View File
@@ -43,6 +43,33 @@ export interface Config {
*/
fragmentDelimiter?: string;
};
/** Elasticsearch specific index template bodies */
indexTemplates?: Array<{
name: string;
body: {
/**
* Array of wildcard (*) expressions used to match the names of data streams and indices during creation.
*/
index_patterns: string[];
/**
* An ordered list of component template names.
* Component templates are merged in the order specified,
* meaning that the last component template specified has the highest precedence.
*/
composed_of?: string[];
/**
* See available properties of template
* https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body
*/
template?: {
[key: string]: unknown;
};
};
}>;
/** Miscellaneous options for the client */
clientOptions?: {
ssl?: {
@@ -128,7 +155,7 @@ export interface Config {
};
/* TODO(kuangp): unsupported until @elastic/elasticsearch@7.14 is released
| {
/**
* Bearer authentication token to connect to the cluster.
* See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html
plugins/search-backend-module-elasticsearch/src/alpha.ts
+49 -42
View File
@@ -16,59 +16,66 @@
import {
coreServices,
createBackendModule,
createExtensionPoint,
} from '@backstage/backend-plugin-api';
import { loggerToWinstonLogger } from '@backstage/backend-common';
import { searchEngineRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
import {
ElasticSearchCustomIndexTemplate,
ElasticSearchQueryTranslator,
ElasticSearchSearchEngine,
} from '@backstage/plugin-search-backend-module-elasticsearch';
/**
* @alpha
* Options for {@link searchModuleElasticsearchEngine}.
*/
export type SearchModuleElasticsearchEngineOptions = {
translator?: ElasticSearchQueryTranslator;
indexTemplate?: ElasticSearchCustomIndexTemplate;
};
/** @alpha */
export interface ElasticSearchQueryTranslatorExtensionPoint {
setTranslator(translator: ElasticSearchQueryTranslator): void;
}
/**
* Extension point used to customize the ElasticSearch query translator.
*
* @alpha
*/
export const elasticsearchTranslatorExtensionPoint =
createExtensionPoint<ElasticSearchQueryTranslatorExtensionPoint>({
id: 'search.elasticsearchEngine.translator',
});
/**
* Search backend module for the Elasticsearch engine.
*
* @alpha
*/
export const searchModuleElasticsearchEngine = createBackendModule(
(options?: SearchModuleElasticsearchEngineOptions) => ({
moduleId: 'elasticsearchEngine',
pluginId: 'search',
register(env) {
env.registerInit({
deps: {
searchEngineRegistry: searchEngineRegistryExtensionPoint,
logger: coreServices.logger,
config: coreServices.rootConfig,
},
async init({ searchEngineRegistry, logger, config }) {
const searchEngine = await ElasticSearchSearchEngine.fromConfig({
logger: loggerToWinstonLogger(logger),
config: config,
});
export const searchModuleElasticsearchEngine = createBackendModule({
moduleId: 'elasticsearchEngine',
pluginId: 'search',
register(env) {
let translator: ElasticSearchQueryTranslator | undefined;
// set custom translator if available
if (options?.translator) {
searchEngine.setTranslator(options.translator);
}
env.registerExtensionPoint(elasticsearchTranslatorExtensionPoint, {
setTranslator(newTranslator) {
if (translator) {
throw new Error(
'ElasticSearch query translator may only be set once',
);
}
translator = newTranslator;
},
});
// set custom index template if available
if (options?.indexTemplate) {
searchEngine.setIndexTemplate(options.indexTemplate);
}
searchEngineRegistry.setSearchEngine(searchEngine);
},
});
},
}),
);
env.registerInit({
deps: {
searchEngineRegistry: searchEngineRegistryExtensionPoint,
logger: coreServices.logger,
config: coreServices.rootConfig,
},
async init({ searchEngineRegistry, logger, config }) {
searchEngineRegistry.setSearchEngine(
await ElasticSearchSearchEngine.fromConfig({
logger,
config,
translator,
}),
);
},
});
},
});
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
+38 -6
View File
@@ -38,6 +38,7 @@ import {
AwsCredentialProvider,
DefaultAwsCredentialsManager,
} from '@backstage/integration-aws-node';
import { LoggerService } from '@backstage/backend-plugin-api';
export type { ElasticSearchClientOptions };
@@ -73,10 +74,11 @@ export type ElasticSearchQueryTranslator = (
* @public
*/
export type ElasticSearchOptions = {
logger: Logger;
logger: Logger | LoggerService;
config: Config;
aliasPostfix?: string;
indexPrefix?: string;
translator?: ElasticSearchQueryTranslator;
};
/**
@@ -126,7 +128,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
private readonly elasticSearchClientOptions: ElasticSearchClientOptions,
private readonly aliasPostfix: string,
private readonly indexPrefix: string,
private readonly logger: Logger,
private readonly logger: Logger | LoggerService,
private readonly batchSize: number,
highlightOptions?: ElasticSearchHighlightOptions,
) {
@@ -149,6 +151,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
config,
aliasPostfix = `search`,
indexPrefix = ``,
translator,
} = options;
const credentialProvider = DefaultAwsCredentialsManager.fromConfig(config);
const clientOptions = await this.createElasticSearchClientOptions(
@@ -165,7 +168,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
logger.info('Initializing ElasticSearch search engine.');
}
return new ElasticSearchSearchEngine(
const engine = new ElasticSearchSearchEngine(
clientOptions,
aliasPostfix,
indexPrefix,
@@ -176,6 +179,18 @@ export class ElasticSearchSearchEngine implements SearchEngine {
'search.elasticsearch.highlightOptions',
),
);
for (const indexTemplate of this.readIndexTemplateConfig(
config.getConfig('search.elasticsearch'),
)) {
await engine.setIndexTemplate(indexTemplate);
}
if (translator) {
await engine.setTranslator(translator);
}
return engine;
}
/**
@@ -221,11 +236,10 @@ export class ElasticSearchSearchEngine implements SearchEngine {
.boolQuery()
.should(value.map(it => esb.matchQuery(key, it.toString())));
}
this.logger.error(
'Failed to query, unrecognized filter type',
this.logger.error('Failed to query, unrecognized filter type', {
key,
value,
);
});
throw new Error(
'Failed to add filters to query. Unrecognized filter type',
);
@@ -518,6 +532,24 @@ export class ElasticSearchSearchEngine implements SearchEngine {
: {}),
};
}
private static readIndexTemplateConfig(
config: Config,
): ElasticSearchCustomIndexTemplate[] {
return (
config.getOptionalConfigArray('indexTemplates')?.map(templateConfig => {
const bodyConfig = templateConfig.getConfig('body');
return {
name: templateConfig.getString('name'),
body: {
index_patterns: bodyConfig.getStringArray('index_patterns'),
composed_of: bodyConfig.getOptionalStringArray('composed_of'),
template: bodyConfig.getOptionalConfig('template')?.get(),
},
};
}) ?? []
);
}
}
/**
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts
+6 -3
View File
@@ -19,6 +19,7 @@ import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
import { IndexableDocument } from '@backstage/plugin-search-common';
import { Logger } from 'winston';
import { Readable } from 'stream';
import { LoggerService } from '@backstage/backend-plugin-api';
/**
* Options for instantiate ElasticSearchSearchEngineIndexer
@@ -29,7 +30,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
indexPrefix: string;
indexSeparator: string;
alias: string;
logger: Logger;
logger: Logger | LoggerService;
elasticSearchClientWrapper: ElasticSearchClientWrapper;
batchSize: number;
};
@@ -55,7 +56,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
private readonly indexSeparator: string;
private readonly alias: string;
private readonly removableAlias: string;
private readonly logger: Logger;
private readonly logger: Logger | LoggerService;
private readonly sourceStream: Readable;
private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper;
private configuredBatchSize: number;
@@ -186,7 +187,9 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
// If any indices are removable, remove them. Do not bubble up this error,
// as doing so would delete the now aliased index. Log instead.
if (this.removableIndices.length) {
this.logger.info('Removing stale search indices', this.removableIndices);
this.logger.info('Removing stale search indices', {
removableIndices: this.removableIndices,
});
try {
await this.elasticSearchClientWrapper.deleteIndex({
index: this.removableIndices,