diff --git a/.changeset/search-lightning-cult.md b/.changeset/search-lightning-cult.md new file mode 100644 index 0000000000..79cd256e77 --- /dev/null +++ b/.changeset/search-lightning-cult.md @@ -0,0 +1,24 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +--- + +**BREAKING**: In order to remain interoperable with all currently supported +deployments of Elasticsearch, this package will now conditionally use either +the `@elastic/elasticsearch` or `@opensearch-project/opensearch` client when +communicating with your deployed cluster. + +If you do not rely on types exported from this package, or if you do not make +use of the `createElasticSearchClientOptions` or `newClient` methods on the +`ElasticSearchSearchEngine` class, then upgrading to this version should not +require any further action on your part. Everything will continue to work as it +always has. + +If you _do_ rely on either of the above methods or any underlying types, some +changes may be needed to your custom code. A type guard is now exported by this +package (`isOpenSearchCompatible(options)`), which you may use to help clarify +which client options are compatible with which client constructors. + +If you are using this package with the `search.elasticsearch.provider` set to +`aws`, and you are making use of the `newClient` method in particular, you may +wish to start using the `@opensearch-project/opensearch` client in your custom +code. diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 9bb8b8c343..a5ff780c8a 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -80,15 +80,16 @@ const searchEngine = await ElasticSearchSearchEngine.initialize({ const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine }); ``` -For the engine to be available, your backend package needs a dependency into +For the engine to be available, your backend package needs a dependency on package `@backstage/plugin-search-backend-module-elasticsearch`. ElasticSearch needs some additional configuration before it is ready to use within your instance. The configuration options are documented in the [configuration schema definition file.](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-elasticsearch/config.d.ts) -The underlying functionality is using official ElasticSearch client version 7.x, -meaning that ElasticSearch version 7 is the only one confirmed to be supported. +The underlying functionality uses either the official ElasticSearch client +version 7.x (meaning that ElasticSearch version 7 is the only one confirmed to +be supported), or the OpenSearch client, when the `aws` provider is configured. Should you need to create your own bespoke search experiences that require more than just a query translator (such as faceted search or Relay pagination), you @@ -97,9 +98,19 @@ search clients. The version of the client need not be the same as one used internally by the elastic search engine plugin. For example: ```typescript -import { Client } from '@elastic/elastic-search'; +import { isOpenSearchCompatible } from '@backstage/plugin-search-backend-module-elasticsearch'; +import { Client as ElasticClient } from '@elastic/elastic-search'; +import { Client as OpenSearchClient } from '@opensearch-project/opensearch'; -const client = searchEngine.newClient(options => new Client(options)); +const client = searchEngine.newClient(options => { + // In reality, you would only import / instantiate one of the following, but + // for illustrative purposes, here are both: + if (isOpenSearchCompatible(options)) { + return new OpenSearchClient(options); + } else { + return new ElasticClient(options); + } +}); ``` #### Set custom index template diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 04b7f96f93..eed2b1b10a 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -5,60 +5,27 @@ ```ts /// +import { ApiResponse } from '@opensearch-project/opensearch'; import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; -import { Client } from '@elastic/elasticsearch'; +import { BulkHelper } from '@opensearch-project/opensearch/lib/Helpers'; +import { BulkStats } from '@opensearch-project/opensearch/lib/Helpers'; import { Config } from '@backstage/config'; import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/plugin-search-common'; import { IndexableResultSet } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; +import { Readable } from 'stream'; import { SearchEngine } from '@backstage/plugin-search-common'; import { SearchQuery } from '@backstage/plugin-search-common'; - -// @public (undocumented) -export interface ElasticSearchAgentOptions { - // (undocumented) - keepAlive?: boolean; - // (undocumented) - keepAliveMsecs?: number; - // (undocumented) - maxFreeSockets?: number; - // (undocumented) - maxSockets?: number; -} - -// @public (undocumented) -export type ElasticSearchAuth = - | { - username: string; - password: string; - } - | { - apiKey: - | string - | { - id: string; - api_key: string; - }; - }; +import { TransportRequestPromise } from '@opensearch-project/opensearch/lib/Transport'; // @public -export interface ElasticSearchClientOptions { +export interface BaseElasticSearchClientOptions { // (undocumented) agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false; // (undocumented) - auth?: ElasticSearchAuth; - // (undocumented) - cloud?: { - id: string; - username?: string; - password?: string; - }; - // (undocumented) compression?: 'gzip'; // (undocumented) - Connection?: ElasticSearchConnectionConstructor; - // (undocumented) disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor'; // (undocumented) enableMetaHeader?: boolean; @@ -69,28 +36,14 @@ export interface ElasticSearchClientOptions { // (undocumented) name?: string | symbol; // (undocumented) - node?: - | string - | string[] - | ElasticSearchNodeOptions - | ElasticSearchNodeOptions[]; - // (undocumented) nodeFilter?: (connection: any) => boolean; // (undocumented) - nodes?: - | string - | string[] - | ElasticSearchNodeOptions - | ElasticSearchNodeOptions[]; - // (undocumented) nodeSelector?: ((connections: any[]) => any) | string; // (undocumented) opaqueIdPrefix?: string; // (undocumented) pingTimeout?: number; // (undocumented) - provider?: 'aws' | 'elastic'; - // (undocumented) proxy?: string | URL; // (undocumented) requestTimeout?: number; @@ -112,6 +65,118 @@ export interface ElasticSearchClientOptions { Transport?: ElasticSearchTransportConstructor; } +// @public (undocumented) +export interface ElasticSearchAgentOptions { + // (undocumented) + keepAlive?: boolean; + // (undocumented) + keepAliveMsecs?: number; + // (undocumented) + maxFreeSockets?: number; + // (undocumented) + maxSockets?: number; +} + +// @public (undocumented) +export type ElasticSearchAliasAction = + | { + remove: { + index: any; + alias: any; + }; + add?: undefined; + } + | { + add: { + indices: any; + alias: any; + index?: undefined; + }; + remove?: undefined; + } + | { + add: { + index: any; + alias: any; + indices?: undefined; + }; + remove?: undefined; + } + | undefined; + +// @public (undocumented) +export type ElasticSearchAuth = + | OpenSearchAuth + | { + apiKey: + | string + | { + id: string; + api_key: string; + }; + }; + +// @public +export type ElasticSearchClientOptions = + | ElasticSearchElasticSearchClientOptions + | OpenSearchElasticSearchClientOptions; + +// @public +export class ElasticSearchClientWrapper { + // (undocumented) + bulk(bulkOptions: { + datasource: Readable; + onDocument: () => ElasticSearchIndexAction; + refreshOnCompletion?: string | boolean; + }): BulkHelper; + // (undocumented) + createIndex({ + index, + }: { + index: string; + }): TransportRequestPromise, unknown>>; + // (undocumented) + deleteIndex({ + index, + }: { + index: string | string[]; + }): TransportRequestPromise, unknown>>; + // (undocumented) + static fromClientOptions( + options: ElasticSearchClientOptions, + ): ElasticSearchClientWrapper; + // (undocumented) + getAliases({ + aliases, + }: { + aliases: string[]; + }): TransportRequestPromise, unknown>>; + // (undocumented) + indexExists({ + index, + }: { + index: string | string[]; + }): TransportRequestPromise>; + // (undocumented) + putIndexTemplate( + template: ElasticSearchCustomIndexTemplate, + ): TransportRequestPromise, unknown>>; + // (undocumented) + search({ + index, + body, + }: { + index: string | string[]; + body: Object; + }): TransportRequestPromise, unknown>>; + // (undocumented) + updateAliases({ + actions, + }: { + actions: ElasticSearchAliasAction[]; + }): TransportRequestPromise, unknown>>; +} + // @public export type ElasticSearchConcreteQuery = { documentTypes?: string[]; @@ -150,6 +215,35 @@ export type ElasticSearchCustomIndexTemplateBody = { template?: Record; }; +// @public +export interface ElasticSearchElasticSearchClientOptions + extends BaseElasticSearchClientOptions { + // (undocumented) + auth?: ElasticSearchAuth; + // (undocumented) + cloud?: { + id: string; + username?: string; + password?: string; + }; + // (undocumented) + Connection?: ElasticSearchConnectionConstructor; + // (undocumented) + node?: + | string + | string[] + | ElasticSearchNodeOptions + | ElasticSearchNodeOptions[]; + // (undocumented) + nodes?: + | string + | string[] + | ElasticSearchNodeOptions + | ElasticSearchNodeOptions[]; + // (undocumented) + provider?: 'elastic'; +} + // @public (undocumented) export type ElasticSearchHighlightConfig = { fragmentDelimiter: string; @@ -166,6 +260,14 @@ export type ElasticSearchHighlightOptions = { numFragments?: number; }; +// @public (undocumented) +export type ElasticSearchIndexAction = { + index: { + _index: string; + [key: string]: any; + }; +}; + // @public (undocumented) export interface ElasticSearchNodeOptions { // (undocumented) @@ -258,7 +360,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { indexSeparator: string; alias: string; logger: Logger; - elasticSearchClient: Client; + elasticSearchClientWrapper: ElasticSearchClientWrapper; }; // @public (undocumented) @@ -273,4 +375,67 @@ export interface ElasticSearchTransportConstructor { DEFAULT: string; }; } + +// @public +export const isOpenSearchCompatible: ( + opts: ElasticSearchClientOptions, +) => opts is OpenSearchElasticSearchClientOptions; + +// @public (undocumented) +export type OpenSearchAuth = { + username: string; + password: string; +}; + +// @public (undocumented) +export interface OpenSearchConnectionConstructor { + // (undocumented) + new (opts?: any): any; + // (undocumented) + roles: { + MASTER: string; + DATA: string; + INGEST: string; + }; + // (undocumented) + statuses: { + ALIVE: string; + DEAD: string; + }; +} + +// @public +export interface OpenSearchElasticSearchClientOptions + extends BaseElasticSearchClientOptions { + // (undocumented) + auth?: OpenSearchAuth; + // (undocumented) + connection?: OpenSearchConnectionConstructor; + // (undocumented) + node?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[]; + // (undocumented) + nodes?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[]; + // (undocumented) + provider?: 'aws'; +} + +// @public (undocumented) +export interface OpenSearchNodeOptions { + // (undocumented) + agent?: ElasticSearchAgentOptions; + // (undocumented) + headers?: Record; + // (undocumented) + id?: string; + // (undocumented) + roles?: { + master: boolean; + data: boolean; + ingest: boolean; + }; + // (undocumented) + ssl?: ConnectionOptions; + // (undocumented) + url: URL; +} ``` diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index 99339ef99c..0bfc0902de 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -23,11 +23,12 @@ "clean": "backstage-cli package clean" }, "dependencies": { - "@acuris/aws-es-connection": "^2.2.0", "@backstage/config": "^1.0.1", "@backstage/plugin-search-backend-node": "^0.6.3-next.0", "@backstage/plugin-search-common": "^0.3.5", - "@elastic/elasticsearch": "7.13.0", + "@elastic/elasticsearch": "^7.13.0", + "@opensearch-project/opensearch": "^1.1.0", + "aws-os-connection": "^0.1.0", "aws-sdk": "^2.948.0", "elastic-builder": "^2.16.0", "lodash": "^4.17.21", @@ -37,7 +38,8 @@ "devDependencies": { "@backstage/backend-common": "^0.14.1-next.0", "@backstage/cli": "^0.17.3-next.0", - "@elastic/elasticsearch-mock": "^1.0.0" + "@elastic/elasticsearch-mock": "^1.0.0", + "@short.io/opensearch-mock": "^0.3.1" }, "files": [ "dist", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts index c5d1f11b0c..74523020db 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts @@ -16,17 +16,60 @@ import type { ConnectionOptions as TLSConnectionOptions } from 'tls'; /** - * Options used to configure the `@elastic/elasticsearch` client and - * are what will be passed as an argument to the - * {@link ElasticSearchSearchEngine.newClient} method - * - * They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`, - * but are maintained separately so that this interface is not coupled to + * Typeguard to differentiate ElasticSearch client options which are compatible + * with OpenSearch vs. ElasticSearch clients. Useful when calling the + * {@link ElasticSearchSearchEngine.newClient} method. * * @public */ -export interface ElasticSearchClientOptions { - provider?: 'aws' | 'elastic'; +export const isOpenSearchCompatible = ( + opts: ElasticSearchClientOptions, +): opts is OpenSearchElasticSearchClientOptions => { + return opts?.provider === 'aws'; +}; + +/** + * Options used to configure the `@elastic/elasticsearch` client or the + * `@opensearch-project/opensearch` client, depending on the given config. It + * will be passed as an argument to the + * {@link ElasticSearchSearchEngine.newClient} method. + * + * @public + */ +export type ElasticSearchClientOptions = + | ElasticSearchElasticSearchClientOptions + | OpenSearchElasticSearchClientOptions; + +/** + * Options used to configure the `@opensearch-project/opensearch` client. + * + * They are drawn from the `ClientOptions` class of `@opensearch-project/opensearch`, + * but are maintained separately so that this interface is not coupled to it. + * + * @public + */ +export interface OpenSearchElasticSearchClientOptions + extends BaseElasticSearchClientOptions { + provider?: 'aws'; + auth?: OpenSearchAuth; + connection?: OpenSearchConnectionConstructor; + node?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[]; + nodes?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[]; +} + +/** + * Options used to configure the `@elastic/elasticsearch` client. + * + * They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`, + * but are maintained separately so that this interface is not coupled to it. + * + * @public + */ +export interface ElasticSearchElasticSearchClientOptions + extends BaseElasticSearchClientOptions { + provider?: 'elastic'; + auth?: ElasticSearchAuth; + Connection?: ElasticSearchConnectionConstructor; node?: | string | string[] @@ -37,8 +80,21 @@ export interface ElasticSearchClientOptions { | string[] | ElasticSearchNodeOptions | ElasticSearchNodeOptions[]; + cloud?: { + id: string; + username?: string; + password?: string; + }; +} + +/** + * Base client options that are shared across `@opensearch-project/opensearch` + * and `@elastic/elasticsearch` clients. + * + * @public + */ +export interface BaseElasticSearchClientOptions { Transport?: ElasticSearchTransportConstructor; - Connection?: ElasticSearchConnectionConstructor; maxRetries?: number; requestTimeout?: number; pingTimeout?: number; @@ -56,25 +112,24 @@ export interface ElasticSearchClientOptions { headers?: Record; opaqueIdPrefix?: string; name?: string | symbol; - auth?: ElasticSearchAuth; proxy?: string | URL; enableMetaHeader?: boolean; - cloud?: { - id: string; - username?: string; - password?: string; - }; disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor'; } +/** + * @public + */ +export type OpenSearchAuth = { + username: string; + password: string; +}; + /** * @public */ export type ElasticSearchAuth = - | { - username: string; - password: string; - } + | OpenSearchAuth | { apiKey: | string @@ -101,6 +156,22 @@ export interface ElasticSearchNodeOptions { }; } +/** + * @public + */ +export interface OpenSearchNodeOptions { + url: URL; + id?: string; + agent?: ElasticSearchAgentOptions; + ssl?: TLSConnectionOptions; + headers?: Record; + roles?: { + master: boolean; + data: boolean; + ingest: boolean; + }; +} + /** * @public */ @@ -127,6 +198,23 @@ export interface ElasticSearchConnectionConstructor { ML: string; }; } + +/** + * @public + */ +export interface OpenSearchConnectionConstructor { + new (opts?: any): any; + statuses: { + ALIVE: string; + DEAD: string; + }; + roles: { + MASTER: string; + DATA: string; + INGEST: string; + }; +} + /** * @public */ @@ -139,3 +227,5 @@ export interface ElasticSearchTransportConstructor { DEFAULT: string; }; } + +// todo(iamEAP) implement canary types to ensure we remain compatible through upgrades. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts new file mode 100644 index 0000000000..5bd8bb30b0 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts @@ -0,0 +1,337 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { Client as ElasticSearchClient } from '@elastic/elasticsearch'; +import { Client as OpenSearchClient } from '@opensearch-project/opensearch'; +import Mock from '@short.io/opensearch-mock'; +import { Readable } from 'stream'; + +import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; +import { + createElasticSearchClientOptions, + ElasticSearchClientOptions, +} from './ElasticSearchSearchEngine'; + +jest.mock('@elastic/elasticsearch', () => ({ + ...jest.requireActual('@elastic/elasticsearch'), + Client: jest.fn().mockReturnValue({ + search: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + cat: { + aliases: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + }, + helpers: { + bulk: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + }, + indices: { + create: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + delete: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + exists: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + putIndexTemplate: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + updateAliases: jest + .fn() + .mockImplementation(async args => ({ client: 'es', args })), + }, + }), +})); + +jest.mock('@opensearch-project/opensearch', () => ({ + ...jest.requireActual('@opensearch-project/opensearch'), + Client: jest.fn().mockReturnValue({ + search: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + cat: { + aliases: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + }, + helpers: { + bulk: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + }, + indices: { + create: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + delete: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + exists: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + putIndexTemplate: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + updateAliases: jest + .fn() + .mockImplementation(async args => ({ client: 'os', args })), + }, + }), +})); + +describe('ElasticSearchClientWrapper', () => { + describe('elasticSearchClient', () => { + let esOptions: ElasticSearchClientOptions; + + beforeEach(async () => { + esOptions = await createElasticSearchClientOptions( + new ConfigReader({ + node: 'http://localhost:9200', + }), + ); + jest.clearAllMocks(); + }); + + it('instantiation', () => { + ElasticSearchClientWrapper.fromClientOptions(esOptions); + expect(ElasticSearchClient).toHaveBeenCalledWith(esOptions); + }); + + it('search', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const searchInput = { index: 'xyz', body: { eg: 'etc' } }; + const result = (await wrapper.search(searchInput)) as any; + + // Should call the ElasticSearch client's search with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(searchInput); + }); + + it('bulk', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const bulkInput = { + datasource: Readable.from([{}]), + onDocument() { + return { index: { _index: 'xyz' } }; + }, + refreshCompletion: 'xyz', + }; + const result = (await wrapper.bulk(bulkInput)) as any; + + // Should call the ElasticSearch client's bulk helper with expected input + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(bulkInput); + }); + + it('putIndexTemplate', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const indexTemplate = { name: 'xyz', body: { index_patterns: ['*'] } }; + const result = (await wrapper.putIndexTemplate(indexTemplate)) as any; + + // Should call the ElasticSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(indexTemplate); + }); + + it('indexExists', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { index: 'xyz' }; + const result = (await wrapper.indexExists(input)) as any; + + // Should call the ElasticSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(input); + }); + + it('deleteIndex', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { index: 'xyz' }; + const result = (await wrapper.deleteIndex(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(input); + }); + + it('createIndex', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { index: 'xyz' }; + const result = (await wrapper.createIndex(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual(input); + }); + + it('getAliases', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { aliases: ['xyz'] }; + const result = (await wrapper.getAliases(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual({ + format: 'json', + name: input.aliases, + }); + }); + + it('updateAliases', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions); + + const input = { actions: [{ remove: { index: 'xyz', alias: 'abc' } }] }; + const result = (await wrapper.updateAliases(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('es'); + expect(result.args).toStrictEqual({ + body: { actions: input.actions }, + }); + }); + }); + + describe('openSearchClient', () => { + const mock = new Mock(); + let osOptions: ElasticSearchClientOptions; + + beforeEach(() => { + osOptions = { + provider: 'aws', + node: 'https://my-es-cluster.eu-west-1.es.amazonaws.com', + connection: mock.getConnection(), + }; + + jest.clearAllMocks(); + }); + + it('instantiation', () => { + ElasticSearchClientWrapper.fromClientOptions(osOptions); + expect(OpenSearchClient).toHaveBeenCalledWith(osOptions); + }); + + it('search', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const searchInput = { index: 'xyz', body: { eg: 'etc' } }; + const result = (await wrapper.search(searchInput)) as any; + + // Should call the OpenSearch client's search with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(searchInput); + }); + + it('bulk', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const bulkInput = { + datasource: Readable.from([{}]), + onDocument() { + return { index: { _index: 'xyz' } }; + }, + refreshCompletion: 'xyz', + }; + const result = (await wrapper.bulk(bulkInput)) as any; + + // Should call the OpenSearch client's bulk helper with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(bulkInput); + }); + + it('putIndexTemplate', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const indexTemplate = { name: 'xyz', body: { index_patterns: ['*'] } }; + const result = (await wrapper.putIndexTemplate(indexTemplate)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(indexTemplate); + }); + + it('indexExists', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { index: 'xyz' }; + const result = (await wrapper.indexExists(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(input); + }); + + it('deleteIndex', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { index: 'xyz' }; + const result = (await wrapper.deleteIndex(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(input); + }); + + it('createIndex', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { index: 'xyz' }; + const result = (await wrapper.createIndex(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual(input); + }); + + it('getAliases', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { aliases: ['xyz'] }; + const result = (await wrapper.getAliases(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual({ + format: 'json', + name: input.aliases, + }); + }); + + it('updateAliases', async () => { + const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions); + + const input = { actions: [{ remove: { index: 'xyz', alias: 'abc' } }] }; + const result = (await wrapper.updateAliases(input)) as any; + + // Should call the OpenSearch client with expected input. + expect(result.client).toBe('os'); + expect(result.args).toStrictEqual({ + body: { actions: input.actions }, + }); + }); + }); +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts new file mode 100644 index 0000000000..bcf67e091c --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts @@ -0,0 +1,209 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Client as ElasticSearchClient } from '@elastic/elasticsearch'; +import { Client as OpenSearchClient } from '@opensearch-project/opensearch'; +import { Readable } from 'stream'; +import { + ElasticSearchClientOptions, + isOpenSearchCompatible, +} from './ElasticSearchClientOptions'; +import { ElasticSearchCustomIndexTemplate } from './types'; + +/** + * @public + */ +export type ElasticSearchAliasAction = + | { + remove: { index: any; alias: any }; + add?: undefined; + } + | { + add: { indices: any; alias: any; index?: undefined }; + remove?: undefined; + } + | { + add: { index: any; alias: any; indices?: undefined }; + remove?: undefined; + } + | undefined; + +/** + * @public + */ +export type ElasticSearchIndexAction = { + index: { + _index: string; + [key: string]: any; + }; +}; + +/** + * A wrapper class that exposes logical methods that are conditionally fired + * against either a configured Elasticsearch client or a configured Opensearch + * client. + * + * This is necessary because, despite its intention to be API-compatible, the + * opensearch client does not support API key-based authentication. This is + * also the sanest way to accomplish this while making typescript happy. + * + * In the future, if the differences between implementations become + * unmaintainably divergent, we should split out the Opensearch and + * Elasticsearch search engine implementations. + * + * @public + */ +export class ElasticSearchClientWrapper { + private readonly elasticSearchClient: ElasticSearchClient | undefined; + private readonly openSearchClient: OpenSearchClient | undefined; + + private constructor({ + openSearchClient, + elasticSearchClient, + }: { + openSearchClient?: OpenSearchClient; + elasticSearchClient?: ElasticSearchClient; + }) { + this.openSearchClient = openSearchClient; + this.elasticSearchClient = elasticSearchClient; + } + + static fromClientOptions(options: ElasticSearchClientOptions) { + if (isOpenSearchCompatible(options)) { + return new ElasticSearchClientWrapper({ + openSearchClient: new OpenSearchClient(options), + }); + } + + return new ElasticSearchClientWrapper({ + elasticSearchClient: new ElasticSearchClient(options), + }); + } + + search({ index, body }: { index: string | string[]; body: Object }) { + if (this.openSearchClient) { + return this.openSearchClient.search({ index, body }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.search({ index, body }); + } + + throw new Error('No client defined'); + } + + bulk(bulkOptions: { + datasource: Readable; + onDocument: () => ElasticSearchIndexAction; + refreshOnCompletion?: string | boolean; + }) { + if (this.openSearchClient) { + return this.openSearchClient.helpers.bulk(bulkOptions); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.helpers.bulk(bulkOptions); + } + + throw new Error('No client defined'); + } + + putIndexTemplate(template: ElasticSearchCustomIndexTemplate) { + if (this.openSearchClient) { + return this.openSearchClient.indices.putIndexTemplate(template); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.indices.putIndexTemplate(template); + } + + throw new Error('No client defined'); + } + + indexExists({ index }: { index: string | string[] }) { + if (this.openSearchClient) { + return this.openSearchClient.indices.exists({ index }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.indices.exists({ index }); + } + + throw new Error('No client defined'); + } + + deleteIndex({ index }: { index: string | string[] }) { + if (this.openSearchClient) { + return this.openSearchClient.indices.delete({ index }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.indices.delete({ index }); + } + + throw new Error('No client defined'); + } + + createIndex({ index }: { index: string }) { + if (this.openSearchClient) { + return this.openSearchClient.indices.create({ index }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.indices.create({ index }); + } + + throw new Error('No client defined'); + } + + getAliases({ aliases }: { aliases: string[] }) { + if (this.openSearchClient) { + return this.openSearchClient.cat.aliases({ + format: 'json', + name: aliases, + }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.cat.aliases({ + format: 'json', + name: aliases, + }); + } + + throw new Error('No client defined'); + } + + updateAliases({ actions }: { actions: ElasticSearchAliasAction[] }) { + const filteredActions = actions.filter(Boolean); + if (this.openSearchClient) { + return this.openSearchClient.indices.updateAliases({ + body: { + actions: filteredActions, + }, + }); + } + + if (this.elasticSearchClient) { + return this.elasticSearchClient.indices.updateAliases({ + body: { + actions: filteredActions, + }, + }); + } + + throw new Error('No client defined'); + } +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 7892b731ed..cc88f456b6 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -16,8 +16,9 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { Client, errors } from '@elastic/elasticsearch'; +import { errors } from '@elastic/elasticsearch'; import Mock from '@elastic/elasticsearch-mock'; +import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { ElasticSearchConcreteQuery, decodePageCursor, @@ -70,7 +71,7 @@ const customIndexTemplate = { describe('ElasticSearchSearchEngine', () => { let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; - let client: Client; + let clientWrapper: ElasticSearchClientWrapper; beforeEach(() => { testSearchEngine = new ElasticSearchSearchEngine( @@ -86,7 +87,7 @@ describe('ElasticSearchSearchEngine', () => { getVoidLogger(), ); // eslint-disable-next-line dot-notation - client = testSearchEngine['elasticSearchClient']; + clientWrapper = testSearchEngine['elasticSearchClientWrapper']; }); describe('custom index template', () => { @@ -686,7 +687,7 @@ describe('ElasticSearchSearchEngine', () => { }); it('should handle index/search type filtering correctly', async () => { - const elasticSearchQuerySpy = jest.spyOn(client, 'search'); + const elasticSearchQuerySpy = jest.spyOn(clientWrapper, 'search'); await testSearchEngine.query({ term: 'testTerm', filters: {}, @@ -725,7 +726,7 @@ describe('ElasticSearchSearchEngine', () => { }); it('should create matchAll query if no term defined', async () => { - const elasticSearchQuerySpy = jest.spyOn(client, 'search'); + const elasticSearchQuerySpy = jest.spyOn(clientWrapper, 'search'); await testSearchEngine.query({ term: '', filters: {}, @@ -759,7 +760,7 @@ describe('ElasticSearchSearchEngine', () => { }); it('should query only specified indices if defined', async () => { - const elasticSearchQuerySpy = jest.spyOn(client, 'search'); + const elasticSearchQuerySpy = jest.spyOn(clientWrapper, 'search'); await testSearchEngine.query({ term: '', filters: {}, @@ -809,7 +810,7 @@ describe('ElasticSearchSearchEngine', () => { type: 'test-index', indexPrefix: '', indexSeparator: '-index__', - elasticSearchClient: client, + elasticSearchClientWrapper: clientWrapper, }), ); expect(indexerMock.on).toHaveBeenCalledWith( diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 31112e3446..4fa0d4e110 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -14,10 +14,7 @@ * limitations under the License. */ -import { - awsGetCredentials, - createAWSConnection, -} from '@acuris/aws-es-connection'; +import { awsGetCredentials, createAWSConnection } from 'aws-os-connection'; import { Config } from '@backstage/config'; import { IndexableDocument, @@ -26,47 +23,17 @@ import { SearchEngine, SearchQuery, } from '@backstage/plugin-search-common'; -import { Client } from '@elastic/elasticsearch'; import esb from 'elastic-builder'; import { isEmpty, isNaN as nan, isNumber } from 'lodash'; import { v4 as uuid } from 'uuid'; import { Logger } from 'winston'; -import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; +import { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; +import { ElasticSearchCustomIndexTemplate } from './types'; +import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; export type { ElasticSearchClientOptions }; -/** - * Elasticsearch specific index template - * @public - */ -export type ElasticSearchCustomIndexTemplate = { - name: string; - body: ElasticSearchCustomIndexTemplateBody; -}; - -/** - * Elasticsearch specific index template body - * @public - */ -export type ElasticSearchCustomIndexTemplateBody = { - /** - * Array of wildcard (*) expressions used to match the names of data streams and indices during creation. - */ - index_patterns: string[]; - /** - * An ordered list of component template names. - * Component templates are merged in the order specified, - * meaning that the last component template specified has the highest precedence. - */ - composed_of?: string[]; - /** - * See available properties of template - * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body - */ - template?: Record; -}; - /** * Search query that the elasticsearch engine understands. * @public @@ -143,7 +110,7 @@ function isBlank(str: string) { * @public */ export class ElasticSearchSearchEngine implements SearchEngine { - private readonly elasticSearchClient: Client; + private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper; private readonly highlightOptions: ElasticSearchHighlightConfig; constructor( @@ -153,7 +120,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly logger: Logger, highlightOptions?: ElasticSearchHighlightOptions, ) { - this.elasticSearchClient = this.newClient(options => new Client(options)); + this.elasticSearchClientWrapper = + ElasticSearchClientWrapper.fromClientOptions(elasticSearchClientOptions); const uuidTag = uuid(); this.highlightOptions = { preTag: `<${uuidTag}>`, @@ -177,7 +145,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { if (options.provider === 'elastic') { logger.info('Initializing Elastic.co ElasticSearch search engine.'); } else if (options.provider === 'aws') { - logger.info('Initializing AWS ElasticSearch search engine.'); + logger.info('Initializing AWS OpenSearch search engine.'); } else { logger.info('Initializing ElasticSearch search engine.'); } @@ -269,7 +237,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { async setIndexTemplate(template: ElasticSearchCustomIndexTemplate) { try { - await this.elasticSearchClient.indices.putIndexTemplate(template); + await this.elasticSearchClientWrapper.putIndexTemplate(template); this.logger.info('Custom index template set'); } catch (error) { this.logger.error(`Unable to set custom index template: ${error}`); @@ -284,7 +252,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexPrefix: this.indexPrefix, indexSeparator: this.indexSeparator, alias, - elasticSearchClient: this.elasticSearchClient, + elasticSearchClientWrapper: this.elasticSearchClientWrapper, logger: this.logger, }); @@ -292,13 +260,13 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexer.on('error', async e => { this.logger.error(`Failed to index documents for type ${type}`, e); try { - const response = await this.elasticSearchClient.indices.exists({ + const response = await this.elasticSearchClientWrapper.indexExists({ index: indexer.indexName, }); const indexCreated = response.body; if (indexCreated) { this.logger.info(`Removing created index ${indexer.indexName}`); - await this.elasticSearchClient.indices.delete({ + await this.elasticSearchClientWrapper.deleteIndex({ index: indexer.indexName, }); } @@ -319,7 +287,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { ? documentTypes.map(it => this.constructSearchAlias(it)) : this.constructSearchAlias('*'); try { - const result = await this.elasticSearchClient.search({ + const result = await this.elasticSearchClientWrapper.search({ index: queryIndices, body: elasticSearchQuery, }); @@ -398,7 +366,7 @@ export function encodePageCursor({ page }: { page: number }): string { return Buffer.from(`${page}`, 'utf-8').toString('base64'); } -async function createElasticSearchClientOptions( +export async function createElasticSearchClientOptions( config?: Config, ): Promise { if (!config) { diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 4e0e8e229f..eff056c3f4 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -16,13 +16,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestPipeline } from '@backstage/plugin-search-backend-node'; -import { Client } from '@elastic/elasticsearch'; import Mock from '@elastic/elasticsearch-mock'; import { range } from 'lodash'; +import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; const mock = new Mock(); -const client = new Client({ +const clientWrapper = ElasticSearchClientWrapper.fromClientOptions({ node: 'http://localhost:9200', Connection: mock.getConnection(), }); @@ -43,7 +43,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { indexSeparator: '-index__', alias: 'some-type-index__search', logger: getVoidLogger(), - elasticSearchClient: client, + elasticSearchClientWrapper: clientWrapper, }); // Set up all requisite Elastic mocks. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index fd16d0c963..87a4472a15 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -16,9 +16,9 @@ import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; import { IndexableDocument } from '@backstage/plugin-search-common'; -import { Client } from '@elastic/elasticsearch'; import { Readable } from 'stream'; import { Logger } from 'winston'; +import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; /** * Options for instansiate ElasticSearchSearchEngineIndexer @@ -30,7 +30,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { indexSeparator: string; alias: string; logger: Logger; - elasticSearchClient: Client; + elasticSearchClientWrapper: ElasticSearchClientWrapper; }; function duration(startTimestamp: [number, number]): string { @@ -57,7 +57,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { private readonly removableAlias: string; private readonly logger: Logger; private readonly sourceStream: Readable; - private readonly elasticSearchClient: Client; + private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper; private bulkResult: Promise; constructor(options: ElasticSearchSearchEngineIndexerOptions) { @@ -70,7 +70,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { this.indexName = this.constructIndexName(`${Date.now()}`); this.alias = options.alias; this.removableAlias = `${this.alias}_removable`; - this.elasticSearchClient = options.elasticSearchClient; + this.elasticSearchClientWrapper = options.elasticSearchClientWrapper; // The ES client bulk helper supports stream-based indexing, but we have to // supply the stream directly to it at instantiation-time. We can't supply @@ -83,7 +83,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { // Keep a reference to the ES Bulk helper so that we can know when all // documents have been successfully written to ES. - this.bulkResult = this.elasticSearchClient.helpers.bulk({ + this.bulkResult = this.elasticSearchClientWrapper.bulk({ datasource: this.sourceStream, onDocument() { that.processed++; @@ -98,16 +98,15 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { async initialize(): Promise { this.logger.info(`Started indexing documents for index ${this.type}`); - const aliases = await this.elasticSearchClient.cat.aliases({ - format: 'json', - name: [this.alias, this.removableAlias], + const aliases = await this.elasticSearchClientWrapper.getAliases({ + aliases: [this.alias, this.removableAlias], }); this.removableIndices = [ ...new Set(aliases.body.map((r: Record) => r.index)), ] as string[]; - await this.elasticSearchClient.indices.create({ + await this.elasticSearchClientWrapper.createIndex({ index: this.indexName, }); } @@ -140,25 +139,23 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { )}`, result, ); - await this.elasticSearchClient.indices.updateAliases({ - body: { - actions: [ - { - remove: { index: this.constructIndexName('*'), alias: this.alias }, - }, - this.removableIndices.length - ? { - add: { - indices: this.removableIndices, - alias: this.removableAlias, - }, - } - : undefined, - { - add: { index: this.indexName, alias: this.alias }, - }, - ].filter(Boolean), - }, + await this.elasticSearchClientWrapper.updateAliases({ + actions: [ + { + remove: { index: this.constructIndexName('*'), alias: this.alias }, + }, + this.removableIndices.length + ? { + add: { + indices: this.removableIndices, + alias: this.removableAlias, + }, + } + : undefined, + { + add: { index: this.indexName, alias: this.alias }, + }, + ].filter(Boolean), }); // If any indices are removable, remove them. Do not bubble up this error, @@ -166,7 +163,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { if (this.removableIndices.length) { this.logger.info('Removing stale search indices', this.removableIndices); try { - await this.elasticSearchClient.indices.delete({ + await this.elasticSearchClientWrapper.deleteIndex({ index: this.removableIndices, }); } catch (e) { diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index 8ac86970a1..8c3b9e3c6d 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -15,13 +15,25 @@ */ export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine'; +export { isOpenSearchCompatible } from './ElasticSearchClientOptions'; export type { + BaseElasticSearchClientOptions, ElasticSearchAgentOptions, ElasticSearchConnectionConstructor, + ElasticSearchElasticSearchClientOptions, ElasticSearchTransportConstructor, ElasticSearchNodeOptions, ElasticSearchAuth, + OpenSearchAuth, + OpenSearchConnectionConstructor, + OpenSearchElasticSearchClientOptions, + OpenSearchNodeOptions, } from './ElasticSearchClientOptions'; +export type { + ElasticSearchAliasAction, + ElasticSearchClientWrapper, + ElasticSearchIndexAction, +} from './ElasticSearchClientWrapper'; export type { ElasticSearchConcreteQuery, ElasticSearchClientOptions, @@ -30,10 +42,12 @@ export type { ElasticSearchQueryTranslator, ElasticSearchQueryTranslatorOptions, ElasticSearchOptions, - ElasticSearchCustomIndexTemplate, - ElasticSearchCustomIndexTemplateBody, } from './ElasticSearchSearchEngine'; export type { ElasticSearchSearchEngineIndexer, ElasticSearchSearchEngineIndexerOptions, } from './ElasticSearchSearchEngineIndexer'; +export type { + ElasticSearchCustomIndexTemplate, + ElasticSearchCustomIndexTemplateBody, +} from './types'; diff --git a/plugins/search-backend-module-elasticsearch/src/engines/types.ts b/plugins/search-backend-module-elasticsearch/src/engines/types.ts new file mode 100644 index 0000000000..a83e8b09b6 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/types.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Elasticsearch specific index template + * @public + */ +export type ElasticSearchCustomIndexTemplate = { + name: string; + body: ElasticSearchCustomIndexTemplateBody; +}; + +/** + * Elasticsearch specific index template body + * @public + */ +export type ElasticSearchCustomIndexTemplateBody = { + /** + * Array of wildcard (*) expressions used to match the names of data streams and indices during creation. + */ + index_patterns: string[]; + /** + * An ordered list of component template names. + * Component templates are merged in the order specified, + * meaning that the last component template specified has the highest precedence. + */ + composed_of?: string[]; + /** + * See available properties of template + * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body + */ + template?: Record; +}; diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index adf7fa43f2..cb4f62098a 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -20,13 +20,18 @@ * @packageDocumentation */ -export { ElasticSearchSearchEngine } from './engines'; +export { ElasticSearchSearchEngine, isOpenSearchCompatible } from './engines'; export type { + BaseElasticSearchClientOptions, ElasticSearchAgentOptions, + ElasticSearchAliasAction, + ElasticSearchClientWrapper, ElasticSearchConcreteQuery, ElasticSearchClientOptions, + ElasticSearchElasticSearchClientOptions, ElasticSearchHighlightConfig, ElasticSearchHighlightOptions, + ElasticSearchIndexAction, ElasticSearchQueryTranslator, ElasticSearchQueryTranslatorOptions, ElasticSearchConnectionConstructor, @@ -38,4 +43,8 @@ export type { ElasticSearchSearchEngineIndexerOptions, ElasticSearchCustomIndexTemplate, ElasticSearchCustomIndexTemplateBody, + OpenSearchAuth, + OpenSearchConnectionConstructor, + OpenSearchElasticSearchClientOptions, + OpenSearchNodeOptions, } from './engines'; diff --git a/yarn.lock b/yarn.lock index 345aa3d510..015ab547a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,13 +2,6 @@ # yarn lockfile v1 -"@acuris/aws-es-connection@^2.2.0": - version "2.3.0" - resolved "https://registry.npmjs.org/@acuris/aws-es-connection/-/aws-es-connection-2.3.0.tgz#bfa736163bdfe2e7fc12f8529eead949fb4e73fe" - integrity sha512-k8jSkdNwpSPty7oautGaJXoAzzPjP7WKbmaAE9i+vJrKWg9n2UtjesHeDhOAPzV9MG8EzXofGozYjaLdeTP+Ug== - dependencies: - aws4 "^1.11.0" - "@ampproject/remapping@^2.1.0": version "2.1.2" resolved "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.1.2.tgz#4edca94973ded9630d20101cd8559cedb8d8bd34" @@ -2079,10 +2072,10 @@ find-my-way "^4.3.3" into-stream "^6.0.0" -"@elastic/elasticsearch@7.13.0": - version "7.13.0" - resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.13.0.tgz#6dcf511dfa91187e22c81e54f41f4bd0fd96b4d6" - integrity sha512-WgwLWo2p9P2tdqzBGX9fHeG8p5IOTXprXNTECQG2mJ7z9n93N5AFBJpEw4d35tWWeCWi9jI13A2wzQZH7XZ/xw== +"@elastic/elasticsearch@^7.13.0": + version "7.17.0" + resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.17.0.tgz#589fb219234cf1b0da23744e82b1d25e2fe9a797" + integrity sha512-5QLPCjd0uLmLj1lSuKSThjNpq39f6NmlTy9ROLFwG5gjyTgpwSqufDeYG/Fm43Xs05uF7WcscoO7eguI3HuuYA== dependencies: debug "^4.3.1" hpagent "^0.1.1" @@ -5127,6 +5120,16 @@ rxjs "7.5.5" tslib "2.0.3" +"@opensearch-project/opensearch@^1.1.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7" + integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ== + dependencies: + debug "^4.3.1" + hpagent "^0.1.1" + ms "^2.1.3" + secure-json-parse "^2.4.0" + "@opentelemetry/api@^1.0.1": version "1.0.4" resolved "https://registry.npmjs.org/@opentelemetry/api/-/api-1.0.4.tgz#a167e46c10d05a07ab299fc518793b0cff8f6924" @@ -5420,6 +5423,15 @@ dependencies: any-observable "^0.3.0" +"@short.io/opensearch-mock@^0.3.1": + version "0.3.1" + resolved "https://registry.npmjs.org/@short.io/opensearch-mock/-/opensearch-mock-0.3.1.tgz#41678404b94342ac60263a8df590f9b1efa80622" + integrity sha512-gwLTzxJrZNGVQNn+FHLhpw7at3jHb6tNczq2hvFPJ3C3nLaCBRkbP2IQHrIDzklLBVFV83Ug0Xgs73A5o9eLtg== + dependencies: + fast-deep-equal "^3.1.3" + find-my-way "^4.3.3" + into-stream "^6.0.0" + "@sideway/address@^4.1.0": version "4.1.1" resolved "https://registry.npmjs.org/@sideway/address/-/address-4.1.1.tgz#9e321e74310963fdf8eebfbee09c7bd69972de4d" @@ -8250,6 +8262,13 @@ available-typed-arrays@^1.0.2: dependencies: array-filter "^1.0.0" +aws-os-connection@^0.1.0: + version "0.1.0" + resolved "https://registry.npmjs.org/aws-os-connection/-/aws-os-connection-0.1.0.tgz#02c1be08cf63448c5bdc9d47b03b7ad3222b662c" + integrity sha512-WBN2S/5zgzfPpEiOTEW7IPJX+hlq+AQeAVKK4AL/Th0SfRRKu6a46i4aZKovQslBkKnmcSFvy+mHSJ299N8YWA== + dependencies: + aws4 "^1.11.0" + aws-sdk-mock@^5.2.1: version "5.7.0" resolved "https://registry.npmjs.org/aws-sdk-mock/-/aws-sdk-mock-5.7.0.tgz#b2a9454c78d008186c3f1a460b79cdb9cc9dfdc4"