diff --git a/.changeset/add-new-client-method-to-elastic-search-egine.md b/.changeset/add-new-client-method-to-elastic-search-egine.md new file mode 100644 index 0000000000..9e5c363522 --- /dev/null +++ b/.changeset/add-new-client-method-to-elastic-search-egine.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Add `newClient()` method to re-use the configuration of the elastic search +engine with custom clients diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 1331c73b72..8342c74096 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -90,6 +90,18 @@ within your instance. The configuration options are documented in the The underlying functionality is using official ElasticSearch client version 7.x, meaning that ElasticSearch version 7 is the only one confirmed to be supported. +Should you need to create your own bespoke search experiences that require more +than just a query translator (such as faceted search or Relay pagination), you +can access the configuration of the search engine in order to create new elastic +search clients. The version of the client need not be the same as one used +internally by the elastic search engine plugin. For example: + +```typescript +import { Client } from '@elastic/elastic-search'; + +const client = searchEngine.newClient(options => new Client(options)); +``` + ## Example configurations ### AWS diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 1e798119c3..6dc0256586 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -3,18 +3,103 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { Client } from '@elastic/elasticsearch'; +/// + import { Config } from '@backstage/config'; +import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; import { SearchEngine } from '@backstage/search-common'; import { SearchQuery } from '@backstage/search-common'; import { SearchResultSet } from '@backstage/search-common'; +// Warning: (ae-missing-release-tag) "ElasticSearchClientOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "@backstage/plugin-search-backend-module-elasticsearch" does not have an export "ElasticSearchEngine" +// +// @public +export interface ElasticSearchClientOptions { + // Warning: (ae-forgotten-export) The symbol "ElasticSearchAgentOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false; + // Warning: (ae-forgotten-export) The symbol "ElasticSearchAuth" needs to be exported by the entry point index.d.ts + // + // (undocumented) + auth?: ElasticSearchAuth; + // (undocumented) + cloud?: { + id: string; + username?: string; + password?: string; + }; + // (undocumented) + compression?: 'gzip'; + // Warning: (ae-forgotten-export) The symbol "ElasticSearchConnectionConstructor" needs to be exported by the entry point index.d.ts + // + // (undocumented) + Connection?: ElasticSearchConnectionConstructor; + // (undocumented) + disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor'; + // (undocumented) + enableMetaHeader?: boolean; + // (undocumented) + headers?: Record; + // (undocumented) + maxRetries?: number; + // (undocumented) + name?: string | symbol; + // Warning: (ae-forgotten-export) The symbol "ElasticSearchNodeOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + node?: + | string + | string[] + | ElasticSearchNodeOptions + | ElasticSearchNodeOptions[]; + // (undocumented) + nodeFilter?: (connection: any) => boolean; + // (undocumented) + nodes?: + | string + | string[] + | ElasticSearchNodeOptions + | ElasticSearchNodeOptions[]; + // (undocumented) + nodeSelector?: ((connections: any[]) => any) | string; + // (undocumented) + opaqueIdPrefix?: string; + // (undocumented) + pingTimeout?: number; + // (undocumented) + provider?: 'aws' | 'elastic'; + // (undocumented) + proxy?: string | URL; + // (undocumented) + requestTimeout?: number; + // (undocumented) + resurrectStrategy?: 'ping' | 'optimistic' | 'none'; + // (undocumented) + sniffEndpoint?: string; + // (undocumented) + sniffInterval?: number | boolean; + // (undocumented) + sniffOnConnectionFault?: boolean; + // (undocumented) + sniffOnStart?: boolean; + // (undocumented) + ssl?: ConnectionOptions; + // (undocumented) + suggestCompression?: boolean; + // Warning: (ae-forgotten-export) The symbol "ElasticSearchTransportConstructor" needs to be exported by the entry point index.d.ts + // + // (undocumented) + Transport?: ElasticSearchTransportConstructor; +} + // @public (undocumented) export class ElasticSearchSearchEngine implements SearchEngine { constructor( - elasticSearchClient: Client, + elasticSearchClientOptions: ElasticSearchClientOptions, aliasPostfix: string, indexPrefix: string, logger: Logger_2, @@ -22,11 +107,15 @@ export class ElasticSearchSearchEngine implements SearchEngine { // Warning: (ae-forgotten-export) The symbol "ElasticSearchOptions" needs to be exported by the entry point index.d.ts // // (undocumented) - static fromConfig( - options: ElasticSearchOptions, - ): Promise; + static fromConfig({ + logger, + config, + aliasPostfix, + indexPrefix, + }: ElasticSearchOptions): Promise; // (undocumented) index(type: string, documents: IndexableDocument[]): Promise; + newClient(create: (options: ElasticSearchClientOptions) => T): T; // (undocumented) query(query: SearchQuery): Promise; // Warning: (ae-forgotten-export) The symbol "ElasticSearchQueryTranslator" needs to be exported by the entry point index.d.ts diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts new file mode 100644 index 0000000000..73cb149c0a --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { ConnectionOptions as TLSConnectionOptions } from 'tls'; + +/** + * Options used to configure the `@elastic/elasticsearch` client and + * are what will be passed as an argument to the + * {@link ElasticSearchEngine.newClient} method + * + * They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`, + * but are maintained separately so that this interface is not coupled to + */ +export interface ElasticSearchClientOptions { + provider?: 'aws' | 'elastic'; + node?: + | string + | string[] + | ElasticSearchNodeOptions + | ElasticSearchNodeOptions[]; + nodes?: + | string + | string[] + | ElasticSearchNodeOptions + | ElasticSearchNodeOptions[]; + Transport?: ElasticSearchTransportConstructor; + Connection?: ElasticSearchConnectionConstructor; + maxRetries?: number; + requestTimeout?: number; + pingTimeout?: number; + sniffInterval?: number | boolean; + sniffOnStart?: boolean; + sniffEndpoint?: string; + sniffOnConnectionFault?: boolean; + resurrectStrategy?: 'ping' | 'optimistic' | 'none'; + suggestCompression?: boolean; + compression?: 'gzip'; + ssl?: TLSConnectionOptions; + agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false; + nodeFilter?: (connection: any) => boolean; + nodeSelector?: ((connections: any[]) => any) | string; + headers?: Record; + opaqueIdPrefix?: string; + name?: string | symbol; + auth?: ElasticSearchAuth; + proxy?: string | URL; + enableMetaHeader?: boolean; + cloud?: { + id: string; + username?: string; + password?: string; + }; + disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor'; +} + +export type ElasticSearchAuth = + | { + username: string; + password: string; + } + | { + apiKey: + | string + | { + id: string; + api_key: string; + }; + }; + +export interface ElasticSearchNodeOptions { + url: URL; + id?: string; + agent?: ElasticSearchAgentOptions; + ssl?: TLSConnectionOptions; + headers?: Record; + roles?: { + master: boolean; + data: boolean; + ingest: boolean; + ml: boolean; + }; +} + +export interface ElasticSearchAgentOptions { + keepAlive?: boolean; + keepAliveMsecs?: number; + maxSockets?: number; + maxFreeSockets?: number; +} + +export interface ElasticSearchConnectionConstructor { + new (opts?: any): any; + statuses: { + ALIVE: string; + DEAD: string; + }; + roles: { + MASTER: string; + DATA: string; + INGEST: string; + ML: string; + }; +} + +export interface ElasticSearchTransportConstructor { + new (opts?: any): any; + sniffReasons: { + SNIFF_ON_START: string; + SNIFF_INTERVAL: string; + SNIFF_ON_CONNECTION_FAULT: string; + DEFAULT: string; + }; +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 2877632013..1566a35b57 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -15,7 +15,6 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { SearchEngine } from '@backstage/search-common'; import { Client } from '@elastic/elasticsearch'; import Mock from '@elastic/elasticsearch-mock'; import { @@ -33,28 +32,31 @@ class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEng } const mock = new Mock(); -const client = new Client({ +const options = { node: 'http://localhost:9200', Connection: mock.getConnection(), -}); +}; describe('ElasticSearchSearchEngine', () => { - let testSearchEngine: SearchEngine; + let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; + let client: Client; beforeEach(() => { testSearchEngine = new ElasticSearchSearchEngine( - client, + options, 'search', '', getVoidLogger(), ); inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests( - client, + options, 'search', '', getVoidLogger(), ); + // eslint-disable-next-line dot-notation + client = testSearchEngine['elasticSearchClient']; }); describe('queryTranslator', () => { diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 7912c691b3..48caf0d4f1 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -30,6 +30,10 @@ import esb from 'elastic-builder'; import { isEmpty, isNaN as nan, isNumber } from 'lodash'; import { Logger } from 'winston'; +import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; + +export type { ElasticSearchClientOptions }; + export type ConcreteElasticSearchQuery = { documentTypes?: string[]; elasticSearchQuery: Object; @@ -68,105 +72,49 @@ function isBlank(str: string) { * @public */ export class ElasticSearchSearchEngine implements SearchEngine { + private readonly elasticSearchClient: Client; + constructor( - private readonly elasticSearchClient: Client, + private readonly elasticSearchClientOptions: ElasticSearchClientOptions, private readonly aliasPostfix: string, private readonly indexPrefix: string, private readonly logger: Logger, - ) {} + ) { + this.elasticSearchClient = this.newClient(options => new Client(options)); + } - static async fromConfig(options: ElasticSearchOptions) { - const { - logger, - config, - aliasPostfix = `search`, - indexPrefix = ``, - } = options; + static async fromConfig({ + logger, + config, + aliasPostfix = `search`, + indexPrefix = ``, + }: ElasticSearchOptions) { + const options = await createElasticSearchClientOptions( + config.getConfig('search.elasticsearch'), + ); + if (options.provider === 'elastic') { + logger.info('Initializing Elastic.co ElasticSearch search engine.'); + } else if (options.provider === 'aws') { + logger.info('Initializing AWS ElasticSearch search engine.'); + } else { + logger.info('Initializing ElasticSearch search engine.'); + } return new ElasticSearchSearchEngine( - await ElasticSearchSearchEngine.constructElasticSearchClient( - logger, - config.getConfig('search.elasticsearch'), - ), + options, aliasPostfix, indexPrefix, logger, ); } - private static async constructElasticSearchClient( - logger: Logger, - config?: Config, - ) { - if (!config) { - throw new Error('No elastic search config found'); - } - - const clientOptionsConfig = config.getOptionalConfig('clientOptions'); - const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl'); - - if (config.getOptionalString('provider') === 'elastic') { - logger.info('Initializing Elastic.co ElasticSearch search engine.'); - const authConfig = config.getConfig('auth'); - return new Client({ - cloud: { - id: config.getString('cloudId'), - }, - auth: { - username: authConfig.getString('username'), - password: authConfig.getString('password'), - }, - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - }); - } - if (config.getOptionalString('provider') === 'aws') { - logger.info('Initializing AWS ElasticSearch search engine.'); - const awsCredentials = await awsGetCredentials(); - const AWSConnection = createAWSConnection(awsCredentials); - return new Client({ - node: config.getString('node'), - ...AWSConnection, - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - }); - } - logger.info('Initializing ElasticSearch search engine.'); - const authConfig = config.getOptionalConfig('auth'); - const auth = - authConfig && - (authConfig.has('apiKey') - ? { - apiKey: authConfig.getString('apiKey'), - } - : { - username: authConfig.getString('username'), - password: authConfig.getString('password'), - }); - return new Client({ - node: config.getString('node'), - auth, - ...(sslConfig - ? { - ssl: { - rejectUnauthorized: - sslConfig?.getOptionalBoolean('rejectUnauthorized'), - }, - } - : {}), - }); + /** + * Create a custom search client from the derived elastic search + * configuration. This need not be the same client that the engine uses + * internally. + */ + newClient(create: (options: ElasticSearchClientOptions) => T): T { + return create(this.elasticSearchClientOptions); } protected translator(query: SearchQuery): ConcreteElasticSearchQuery { @@ -349,3 +297,75 @@ export function decodePageCursor(pageCursor?: string): { page: number } { export function encodePageCursor({ page }: { page: number }): string { return Buffer.from(`${page}`, 'utf-8').toString('base64'); } + +async function createElasticSearchClientOptions( + config?: Config, +): Promise { + if (!config) { + throw new Error('No elastic search config found'); + } + const clientOptionsConfig = config.getOptionalConfig('clientOptions'); + const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl'); + + if (config.getOptionalString('provider') === 'elastic') { + const authConfig = config.getConfig('auth'); + return { + provider: 'elastic', + cloud: { + id: config.getString('cloudId'), + }, + auth: { + username: authConfig.getString('username'), + password: authConfig.getString('password'), + }, + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + }; + } + if (config.getOptionalString('provider') === 'aws') { + const awsCredentials = await awsGetCredentials(); + const AWSConnection = createAWSConnection(awsCredentials); + return { + provider: 'aws', + node: config.getString('node'), + ...AWSConnection, + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + }; + } + const authConfig = config.getOptionalConfig('auth'); + const auth = + authConfig && + (authConfig.has('apiKey') + ? { + apiKey: authConfig.getString('apiKey'), + } + : { + username: authConfig.getString('username'), + password: authConfig.getString('password'), + }); + return { + node: config.getString('node'), + auth, + ...(sslConfig + ? { + ssl: { + rejectUnauthorized: + sslConfig?.getOptionalBoolean('rejectUnauthorized'), + }, + } + : {}), + }; +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index 2f3641c114..d5eee37803 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -15,4 +15,7 @@ */ export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine'; -export type { ConcreteElasticSearchQuery } from './ElasticSearchSearchEngine'; +export type { + ConcreteElasticSearchQuery, + ElasticSearchClientOptions, +} from './ElasticSearchSearchEngine'; diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index b4dab93b76..8cf96de858 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -21,3 +21,4 @@ */ export { ElasticSearchSearchEngine } from './engines'; +export type { ElasticSearchClientOptions } from './engines';