diff --git a/.changeset/search-lightning-cult.md b/.changeset/search-lightning-cult.md
new file mode 100644
index 0000000000..79cd256e77
--- /dev/null
+++ b/.changeset/search-lightning-cult.md
@@ -0,0 +1,24 @@
+---
+'@backstage/plugin-search-backend-module-elasticsearch': minor
+---
+
+**BREAKING**: In order to remain interoperable with all currently supported
+deployments of Elasticsearch, this package will now conditionally use either
+the `@elastic/elasticsearch` or `@opensearch-project/opensearch` client when
+communicating with your deployed cluster.
+
+If you do not rely on types exported from this package, or if you do not make
+use of the `createElasticSearchClientOptions` or `newClient` methods on the
+`ElasticSearchSearchEngine` class, then upgrading to this version should not
+require any further action on your part. Everything will continue to work as it
+always has.
+
+If you _do_ rely on either of the above methods or any underlying types, some
+changes may be needed to your custom code. A type guard is now exported by this
+package (`isOpenSearchCompatible(options)`), which you may use to help clarify
+which client options are compatible with which client constructors.
+
+If you are using this package with the `search.elasticsearch.provider` set to
+`aws`, and you are making use of the `newClient` method in particular, you may
+wish to start using the `@opensearch-project/opensearch` client in your custom
+code.
diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md
index 9bb8b8c343..a5ff780c8a 100644
--- a/docs/features/search/search-engines.md
+++ b/docs/features/search/search-engines.md
@@ -80,15 +80,16 @@ const searchEngine = await ElasticSearchSearchEngine.initialize({
const indexBuilder = new IndexBuilder({ logger: env.logger, searchEngine });
```
-For the engine to be available, your backend package needs a dependency into
+For the engine to be available, your backend package needs a dependency on
package `@backstage/plugin-search-backend-module-elasticsearch`.
ElasticSearch needs some additional configuration before it is ready to use
within your instance. The configuration options are documented in the
[configuration schema definition file.](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-elasticsearch/config.d.ts)
-The underlying functionality is using official ElasticSearch client version 7.x,
-meaning that ElasticSearch version 7 is the only one confirmed to be supported.
+The underlying functionality uses either the official ElasticSearch client
+version 7.x (meaning that ElasticSearch version 7 is the only one confirmed to
+be supported), or the OpenSearch client, when the `aws` provider is configured.
Should you need to create your own bespoke search experiences that require more
than just a query translator (such as faceted search or Relay pagination), you
@@ -97,9 +98,19 @@ search clients. The version of the client need not be the same as one used
internally by the elastic search engine plugin. For example:
```typescript
-import { Client } from '@elastic/elastic-search';
+import { isOpenSearchCompatible } from '@backstage/plugin-search-backend-module-elasticsearch';
+import { Client as ElasticClient } from '@elastic/elastic-search';
+import { Client as OpenSearchClient } from '@opensearch-project/opensearch';
-const client = searchEngine.newClient(options => new Client(options));
+const client = searchEngine.newClient(options => {
+ // In reality, you would only import / instantiate one of the following, but
+ // for illustrative purposes, here are both:
+ if (isOpenSearchCompatible(options)) {
+ return new OpenSearchClient(options);
+ } else {
+ return new ElasticClient(options);
+ }
+});
```
#### Set custom index template
diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md
index 04b7f96f93..eed2b1b10a 100644
--- a/plugins/search-backend-module-elasticsearch/api-report.md
+++ b/plugins/search-backend-module-elasticsearch/api-report.md
@@ -5,60 +5,27 @@
```ts
///
+import { ApiResponse } from '@opensearch-project/opensearch';
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
-import { Client } from '@elastic/elasticsearch';
+import { BulkHelper } from '@opensearch-project/opensearch/lib/Helpers';
+import { BulkStats } from '@opensearch-project/opensearch/lib/Helpers';
import { Config } from '@backstage/config';
import type { ConnectionOptions } from 'tls';
import { IndexableDocument } from '@backstage/plugin-search-common';
import { IndexableResultSet } from '@backstage/plugin-search-common';
import { Logger } from 'winston';
+import { Readable } from 'stream';
import { SearchEngine } from '@backstage/plugin-search-common';
import { SearchQuery } from '@backstage/plugin-search-common';
-
-// @public (undocumented)
-export interface ElasticSearchAgentOptions {
- // (undocumented)
- keepAlive?: boolean;
- // (undocumented)
- keepAliveMsecs?: number;
- // (undocumented)
- maxFreeSockets?: number;
- // (undocumented)
- maxSockets?: number;
-}
-
-// @public (undocumented)
-export type ElasticSearchAuth =
- | {
- username: string;
- password: string;
- }
- | {
- apiKey:
- | string
- | {
- id: string;
- api_key: string;
- };
- };
+import { TransportRequestPromise } from '@opensearch-project/opensearch/lib/Transport';
// @public
-export interface ElasticSearchClientOptions {
+export interface BaseElasticSearchClientOptions {
// (undocumented)
agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false;
// (undocumented)
- auth?: ElasticSearchAuth;
- // (undocumented)
- cloud?: {
- id: string;
- username?: string;
- password?: string;
- };
- // (undocumented)
compression?: 'gzip';
// (undocumented)
- Connection?: ElasticSearchConnectionConstructor;
- // (undocumented)
disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor';
// (undocumented)
enableMetaHeader?: boolean;
@@ -69,28 +36,14 @@ export interface ElasticSearchClientOptions {
// (undocumented)
name?: string | symbol;
// (undocumented)
- node?:
- | string
- | string[]
- | ElasticSearchNodeOptions
- | ElasticSearchNodeOptions[];
- // (undocumented)
nodeFilter?: (connection: any) => boolean;
// (undocumented)
- nodes?:
- | string
- | string[]
- | ElasticSearchNodeOptions
- | ElasticSearchNodeOptions[];
- // (undocumented)
nodeSelector?: ((connections: any[]) => any) | string;
// (undocumented)
opaqueIdPrefix?: string;
// (undocumented)
pingTimeout?: number;
// (undocumented)
- provider?: 'aws' | 'elastic';
- // (undocumented)
proxy?: string | URL;
// (undocumented)
requestTimeout?: number;
@@ -112,6 +65,118 @@ export interface ElasticSearchClientOptions {
Transport?: ElasticSearchTransportConstructor;
}
+// @public (undocumented)
+export interface ElasticSearchAgentOptions {
+ // (undocumented)
+ keepAlive?: boolean;
+ // (undocumented)
+ keepAliveMsecs?: number;
+ // (undocumented)
+ maxFreeSockets?: number;
+ // (undocumented)
+ maxSockets?: number;
+}
+
+// @public (undocumented)
+export type ElasticSearchAliasAction =
+ | {
+ remove: {
+ index: any;
+ alias: any;
+ };
+ add?: undefined;
+ }
+ | {
+ add: {
+ indices: any;
+ alias: any;
+ index?: undefined;
+ };
+ remove?: undefined;
+ }
+ | {
+ add: {
+ index: any;
+ alias: any;
+ indices?: undefined;
+ };
+ remove?: undefined;
+ }
+ | undefined;
+
+// @public (undocumented)
+export type ElasticSearchAuth =
+ | OpenSearchAuth
+ | {
+ apiKey:
+ | string
+ | {
+ id: string;
+ api_key: string;
+ };
+ };
+
+// @public
+export type ElasticSearchClientOptions =
+ | ElasticSearchElasticSearchClientOptions
+ | OpenSearchElasticSearchClientOptions;
+
+// @public
+export class ElasticSearchClientWrapper {
+ // (undocumented)
+ bulk(bulkOptions: {
+ datasource: Readable;
+ onDocument: () => ElasticSearchIndexAction;
+ refreshOnCompletion?: string | boolean;
+ }): BulkHelper;
+ // (undocumented)
+ createIndex({
+ index,
+ }: {
+ index: string;
+ }): TransportRequestPromise, unknown>>;
+ // (undocumented)
+ deleteIndex({
+ index,
+ }: {
+ index: string | string[];
+ }): TransportRequestPromise, unknown>>;
+ // (undocumented)
+ static fromClientOptions(
+ options: ElasticSearchClientOptions,
+ ): ElasticSearchClientWrapper;
+ // (undocumented)
+ getAliases({
+ aliases,
+ }: {
+ aliases: string[];
+ }): TransportRequestPromise, unknown>>;
+ // (undocumented)
+ indexExists({
+ index,
+ }: {
+ index: string | string[];
+ }): TransportRequestPromise>;
+ // (undocumented)
+ putIndexTemplate(
+ template: ElasticSearchCustomIndexTemplate,
+ ): TransportRequestPromise, unknown>>;
+ // (undocumented)
+ search({
+ index,
+ body,
+ }: {
+ index: string | string[];
+ body: Object;
+ }): TransportRequestPromise, unknown>>;
+ // (undocumented)
+ updateAliases({
+ actions,
+ }: {
+ actions: ElasticSearchAliasAction[];
+ }): TransportRequestPromise, unknown>>;
+}
+
// @public
export type ElasticSearchConcreteQuery = {
documentTypes?: string[];
@@ -150,6 +215,35 @@ export type ElasticSearchCustomIndexTemplateBody = {
template?: Record;
};
+// @public
+export interface ElasticSearchElasticSearchClientOptions
+ extends BaseElasticSearchClientOptions {
+ // (undocumented)
+ auth?: ElasticSearchAuth;
+ // (undocumented)
+ cloud?: {
+ id: string;
+ username?: string;
+ password?: string;
+ };
+ // (undocumented)
+ Connection?: ElasticSearchConnectionConstructor;
+ // (undocumented)
+ node?:
+ | string
+ | string[]
+ | ElasticSearchNodeOptions
+ | ElasticSearchNodeOptions[];
+ // (undocumented)
+ nodes?:
+ | string
+ | string[]
+ | ElasticSearchNodeOptions
+ | ElasticSearchNodeOptions[];
+ // (undocumented)
+ provider?: 'elastic';
+}
+
// @public (undocumented)
export type ElasticSearchHighlightConfig = {
fragmentDelimiter: string;
@@ -166,6 +260,14 @@ export type ElasticSearchHighlightOptions = {
numFragments?: number;
};
+// @public (undocumented)
+export type ElasticSearchIndexAction = {
+ index: {
+ _index: string;
+ [key: string]: any;
+ };
+};
+
// @public (undocumented)
export interface ElasticSearchNodeOptions {
// (undocumented)
@@ -258,7 +360,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
indexSeparator: string;
alias: string;
logger: Logger;
- elasticSearchClient: Client;
+ elasticSearchClientWrapper: ElasticSearchClientWrapper;
};
// @public (undocumented)
@@ -273,4 +375,67 @@ export interface ElasticSearchTransportConstructor {
DEFAULT: string;
};
}
+
+// @public
+export const isOpenSearchCompatible: (
+ opts: ElasticSearchClientOptions,
+) => opts is OpenSearchElasticSearchClientOptions;
+
+// @public (undocumented)
+export type OpenSearchAuth = {
+ username: string;
+ password: string;
+};
+
+// @public (undocumented)
+export interface OpenSearchConnectionConstructor {
+ // (undocumented)
+ new (opts?: any): any;
+ // (undocumented)
+ roles: {
+ MASTER: string;
+ DATA: string;
+ INGEST: string;
+ };
+ // (undocumented)
+ statuses: {
+ ALIVE: string;
+ DEAD: string;
+ };
+}
+
+// @public
+export interface OpenSearchElasticSearchClientOptions
+ extends BaseElasticSearchClientOptions {
+ // (undocumented)
+ auth?: OpenSearchAuth;
+ // (undocumented)
+ connection?: OpenSearchConnectionConstructor;
+ // (undocumented)
+ node?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[];
+ // (undocumented)
+ nodes?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[];
+ // (undocumented)
+ provider?: 'aws';
+}
+
+// @public (undocumented)
+export interface OpenSearchNodeOptions {
+ // (undocumented)
+ agent?: ElasticSearchAgentOptions;
+ // (undocumented)
+ headers?: Record;
+ // (undocumented)
+ id?: string;
+ // (undocumented)
+ roles?: {
+ master: boolean;
+ data: boolean;
+ ingest: boolean;
+ };
+ // (undocumented)
+ ssl?: ConnectionOptions;
+ // (undocumented)
+ url: URL;
+}
```
diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json
index 99339ef99c..0bfc0902de 100644
--- a/plugins/search-backend-module-elasticsearch/package.json
+++ b/plugins/search-backend-module-elasticsearch/package.json
@@ -23,11 +23,12 @@
"clean": "backstage-cli package clean"
},
"dependencies": {
- "@acuris/aws-es-connection": "^2.2.0",
"@backstage/config": "^1.0.1",
"@backstage/plugin-search-backend-node": "^0.6.3-next.0",
"@backstage/plugin-search-common": "^0.3.5",
- "@elastic/elasticsearch": "7.13.0",
+ "@elastic/elasticsearch": "^7.13.0",
+ "@opensearch-project/opensearch": "^1.1.0",
+ "aws-os-connection": "^0.1.0",
"aws-sdk": "^2.948.0",
"elastic-builder": "^2.16.0",
"lodash": "^4.17.21",
@@ -37,7 +38,8 @@
"devDependencies": {
"@backstage/backend-common": "^0.14.1-next.0",
"@backstage/cli": "^0.17.3-next.0",
- "@elastic/elasticsearch-mock": "^1.0.0"
+ "@elastic/elasticsearch-mock": "^1.0.0",
+ "@short.io/opensearch-mock": "^0.3.1"
},
"files": [
"dist",
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts
index c5d1f11b0c..74523020db 100644
--- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts
@@ -16,17 +16,60 @@
import type { ConnectionOptions as TLSConnectionOptions } from 'tls';
/**
- * Options used to configure the `@elastic/elasticsearch` client and
- * are what will be passed as an argument to the
- * {@link ElasticSearchSearchEngine.newClient} method
- *
- * They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`,
- * but are maintained separately so that this interface is not coupled to
+ * Typeguard to differentiate ElasticSearch client options which are compatible
+ * with OpenSearch vs. ElasticSearch clients. Useful when calling the
+ * {@link ElasticSearchSearchEngine.newClient} method.
*
* @public
*/
-export interface ElasticSearchClientOptions {
- provider?: 'aws' | 'elastic';
+export const isOpenSearchCompatible = (
+ opts: ElasticSearchClientOptions,
+): opts is OpenSearchElasticSearchClientOptions => {
+ return opts?.provider === 'aws';
+};
+
+/**
+ * Options used to configure the `@elastic/elasticsearch` client or the
+ * `@opensearch-project/opensearch` client, depending on the given config. It
+ * will be passed as an argument to the
+ * {@link ElasticSearchSearchEngine.newClient} method.
+ *
+ * @public
+ */
+export type ElasticSearchClientOptions =
+ | ElasticSearchElasticSearchClientOptions
+ | OpenSearchElasticSearchClientOptions;
+
+/**
+ * Options used to configure the `@opensearch-project/opensearch` client.
+ *
+ * They are drawn from the `ClientOptions` class of `@opensearch-project/opensearch`,
+ * but are maintained separately so that this interface is not coupled to it.
+ *
+ * @public
+ */
+export interface OpenSearchElasticSearchClientOptions
+ extends BaseElasticSearchClientOptions {
+ provider?: 'aws';
+ auth?: OpenSearchAuth;
+ connection?: OpenSearchConnectionConstructor;
+ node?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[];
+ nodes?: string | string[] | OpenSearchNodeOptions | OpenSearchNodeOptions[];
+}
+
+/**
+ * Options used to configure the `@elastic/elasticsearch` client.
+ *
+ * They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`,
+ * but are maintained separately so that this interface is not coupled to it.
+ *
+ * @public
+ */
+export interface ElasticSearchElasticSearchClientOptions
+ extends BaseElasticSearchClientOptions {
+ provider?: 'elastic';
+ auth?: ElasticSearchAuth;
+ Connection?: ElasticSearchConnectionConstructor;
node?:
| string
| string[]
@@ -37,8 +80,21 @@ export interface ElasticSearchClientOptions {
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
+ cloud?: {
+ id: string;
+ username?: string;
+ password?: string;
+ };
+}
+
+/**
+ * Base client options that are shared across `@opensearch-project/opensearch`
+ * and `@elastic/elasticsearch` clients.
+ *
+ * @public
+ */
+export interface BaseElasticSearchClientOptions {
Transport?: ElasticSearchTransportConstructor;
- Connection?: ElasticSearchConnectionConstructor;
maxRetries?: number;
requestTimeout?: number;
pingTimeout?: number;
@@ -56,25 +112,24 @@ export interface ElasticSearchClientOptions {
headers?: Record;
opaqueIdPrefix?: string;
name?: string | symbol;
- auth?: ElasticSearchAuth;
proxy?: string | URL;
enableMetaHeader?: boolean;
- cloud?: {
- id: string;
- username?: string;
- password?: string;
- };
disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor';
}
+/**
+ * @public
+ */
+export type OpenSearchAuth = {
+ username: string;
+ password: string;
+};
+
/**
* @public
*/
export type ElasticSearchAuth =
- | {
- username: string;
- password: string;
- }
+ | OpenSearchAuth
| {
apiKey:
| string
@@ -101,6 +156,22 @@ export interface ElasticSearchNodeOptions {
};
}
+/**
+ * @public
+ */
+export interface OpenSearchNodeOptions {
+ url: URL;
+ id?: string;
+ agent?: ElasticSearchAgentOptions;
+ ssl?: TLSConnectionOptions;
+ headers?: Record;
+ roles?: {
+ master: boolean;
+ data: boolean;
+ ingest: boolean;
+ };
+}
+
/**
* @public
*/
@@ -127,6 +198,23 @@ export interface ElasticSearchConnectionConstructor {
ML: string;
};
}
+
+/**
+ * @public
+ */
+export interface OpenSearchConnectionConstructor {
+ new (opts?: any): any;
+ statuses: {
+ ALIVE: string;
+ DEAD: string;
+ };
+ roles: {
+ MASTER: string;
+ DATA: string;
+ INGEST: string;
+ };
+}
+
/**
* @public
*/
@@ -139,3 +227,5 @@ export interface ElasticSearchTransportConstructor {
DEFAULT: string;
};
}
+
+// todo(iamEAP) implement canary types to ensure we remain compatible through upgrades.
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts
new file mode 100644
index 0000000000..5bd8bb30b0
--- /dev/null
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.test.ts
@@ -0,0 +1,337 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { Client as ElasticSearchClient } from '@elastic/elasticsearch';
+import { Client as OpenSearchClient } from '@opensearch-project/opensearch';
+import Mock from '@short.io/opensearch-mock';
+import { Readable } from 'stream';
+
+import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
+import {
+ createElasticSearchClientOptions,
+ ElasticSearchClientOptions,
+} from './ElasticSearchSearchEngine';
+
+jest.mock('@elastic/elasticsearch', () => ({
+ ...jest.requireActual('@elastic/elasticsearch'),
+ Client: jest.fn().mockReturnValue({
+ search: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ cat: {
+ aliases: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ },
+ helpers: {
+ bulk: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ },
+ indices: {
+ create: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ delete: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ exists: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ putIndexTemplate: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ updateAliases: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'es', args })),
+ },
+ }),
+}));
+
+jest.mock('@opensearch-project/opensearch', () => ({
+ ...jest.requireActual('@opensearch-project/opensearch'),
+ Client: jest.fn().mockReturnValue({
+ search: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ cat: {
+ aliases: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ },
+ helpers: {
+ bulk: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ },
+ indices: {
+ create: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ delete: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ exists: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ putIndexTemplate: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ updateAliases: jest
+ .fn()
+ .mockImplementation(async args => ({ client: 'os', args })),
+ },
+ }),
+}));
+
+describe('ElasticSearchClientWrapper', () => {
+ describe('elasticSearchClient', () => {
+ let esOptions: ElasticSearchClientOptions;
+
+ beforeEach(async () => {
+ esOptions = await createElasticSearchClientOptions(
+ new ConfigReader({
+ node: 'http://localhost:9200',
+ }),
+ );
+ jest.clearAllMocks();
+ });
+
+ it('instantiation', () => {
+ ElasticSearchClientWrapper.fromClientOptions(esOptions);
+ expect(ElasticSearchClient).toHaveBeenCalledWith(esOptions);
+ });
+
+ it('search', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const searchInput = { index: 'xyz', body: { eg: 'etc' } };
+ const result = (await wrapper.search(searchInput)) as any;
+
+ // Should call the ElasticSearch client's search with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual(searchInput);
+ });
+
+ it('bulk', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const bulkInput = {
+ datasource: Readable.from([{}]),
+ onDocument() {
+ return { index: { _index: 'xyz' } };
+ },
+ refreshCompletion: 'xyz',
+ };
+ const result = (await wrapper.bulk(bulkInput)) as any;
+
+ // Should call the ElasticSearch client's bulk helper with expected input
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual(bulkInput);
+ });
+
+ it('putIndexTemplate', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const indexTemplate = { name: 'xyz', body: { index_patterns: ['*'] } };
+ const result = (await wrapper.putIndexTemplate(indexTemplate)) as any;
+
+ // Should call the ElasticSearch client with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual(indexTemplate);
+ });
+
+ it('indexExists', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const input = { index: 'xyz' };
+ const result = (await wrapper.indexExists(input)) as any;
+
+ // Should call the ElasticSearch client with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual(input);
+ });
+
+ it('deleteIndex', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const input = { index: 'xyz' };
+ const result = (await wrapper.deleteIndex(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual(input);
+ });
+
+ it('createIndex', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const input = { index: 'xyz' };
+ const result = (await wrapper.createIndex(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual(input);
+ });
+
+ it('getAliases', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const input = { aliases: ['xyz'] };
+ const result = (await wrapper.getAliases(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual({
+ format: 'json',
+ name: input.aliases,
+ });
+ });
+
+ it('updateAliases', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(esOptions);
+
+ const input = { actions: [{ remove: { index: 'xyz', alias: 'abc' } }] };
+ const result = (await wrapper.updateAliases(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('es');
+ expect(result.args).toStrictEqual({
+ body: { actions: input.actions },
+ });
+ });
+ });
+
+ describe('openSearchClient', () => {
+ const mock = new Mock();
+ let osOptions: ElasticSearchClientOptions;
+
+ beforeEach(() => {
+ osOptions = {
+ provider: 'aws',
+ node: 'https://my-es-cluster.eu-west-1.es.amazonaws.com',
+ connection: mock.getConnection(),
+ };
+
+ jest.clearAllMocks();
+ });
+
+ it('instantiation', () => {
+ ElasticSearchClientWrapper.fromClientOptions(osOptions);
+ expect(OpenSearchClient).toHaveBeenCalledWith(osOptions);
+ });
+
+ it('search', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const searchInput = { index: 'xyz', body: { eg: 'etc' } };
+ const result = (await wrapper.search(searchInput)) as any;
+
+ // Should call the OpenSearch client's search with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual(searchInput);
+ });
+
+ it('bulk', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const bulkInput = {
+ datasource: Readable.from([{}]),
+ onDocument() {
+ return { index: { _index: 'xyz' } };
+ },
+ refreshCompletion: 'xyz',
+ };
+ const result = (await wrapper.bulk(bulkInput)) as any;
+
+ // Should call the OpenSearch client's bulk helper with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual(bulkInput);
+ });
+
+ it('putIndexTemplate', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const indexTemplate = { name: 'xyz', body: { index_patterns: ['*'] } };
+ const result = (await wrapper.putIndexTemplate(indexTemplate)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual(indexTemplate);
+ });
+
+ it('indexExists', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const input = { index: 'xyz' };
+ const result = (await wrapper.indexExists(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual(input);
+ });
+
+ it('deleteIndex', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const input = { index: 'xyz' };
+ const result = (await wrapper.deleteIndex(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual(input);
+ });
+
+ it('createIndex', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const input = { index: 'xyz' };
+ const result = (await wrapper.createIndex(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual(input);
+ });
+
+ it('getAliases', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const input = { aliases: ['xyz'] };
+ const result = (await wrapper.getAliases(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual({
+ format: 'json',
+ name: input.aliases,
+ });
+ });
+
+ it('updateAliases', async () => {
+ const wrapper = ElasticSearchClientWrapper.fromClientOptions(osOptions);
+
+ const input = { actions: [{ remove: { index: 'xyz', alias: 'abc' } }] };
+ const result = (await wrapper.updateAliases(input)) as any;
+
+ // Should call the OpenSearch client with expected input.
+ expect(result.client).toBe('os');
+ expect(result.args).toStrictEqual({
+ body: { actions: input.actions },
+ });
+ });
+ });
+});
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts
new file mode 100644
index 0000000000..bcf67e091c
--- /dev/null
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientWrapper.ts
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Client as ElasticSearchClient } from '@elastic/elasticsearch';
+import { Client as OpenSearchClient } from '@opensearch-project/opensearch';
+import { Readable } from 'stream';
+import {
+ ElasticSearchClientOptions,
+ isOpenSearchCompatible,
+} from './ElasticSearchClientOptions';
+import { ElasticSearchCustomIndexTemplate } from './types';
+
+/**
+ * @public
+ */
+export type ElasticSearchAliasAction =
+ | {
+ remove: { index: any; alias: any };
+ add?: undefined;
+ }
+ | {
+ add: { indices: any; alias: any; index?: undefined };
+ remove?: undefined;
+ }
+ | {
+ add: { index: any; alias: any; indices?: undefined };
+ remove?: undefined;
+ }
+ | undefined;
+
+/**
+ * @public
+ */
+export type ElasticSearchIndexAction = {
+ index: {
+ _index: string;
+ [key: string]: any;
+ };
+};
+
+/**
+ * A wrapper class that exposes logical methods that are conditionally fired
+ * against either a configured Elasticsearch client or a configured Opensearch
+ * client.
+ *
+ * This is necessary because, despite its intention to be API-compatible, the
+ * opensearch client does not support API key-based authentication. This is
+ * also the sanest way to accomplish this while making typescript happy.
+ *
+ * In the future, if the differences between implementations become
+ * unmaintainably divergent, we should split out the Opensearch and
+ * Elasticsearch search engine implementations.
+ *
+ * @public
+ */
+export class ElasticSearchClientWrapper {
+ private readonly elasticSearchClient: ElasticSearchClient | undefined;
+ private readonly openSearchClient: OpenSearchClient | undefined;
+
+ private constructor({
+ openSearchClient,
+ elasticSearchClient,
+ }: {
+ openSearchClient?: OpenSearchClient;
+ elasticSearchClient?: ElasticSearchClient;
+ }) {
+ this.openSearchClient = openSearchClient;
+ this.elasticSearchClient = elasticSearchClient;
+ }
+
+ static fromClientOptions(options: ElasticSearchClientOptions) {
+ if (isOpenSearchCompatible(options)) {
+ return new ElasticSearchClientWrapper({
+ openSearchClient: new OpenSearchClient(options),
+ });
+ }
+
+ return new ElasticSearchClientWrapper({
+ elasticSearchClient: new ElasticSearchClient(options),
+ });
+ }
+
+ search({ index, body }: { index: string | string[]; body: Object }) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.search({ index, body });
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.search({ index, body });
+ }
+
+ throw new Error('No client defined');
+ }
+
+ bulk(bulkOptions: {
+ datasource: Readable;
+ onDocument: () => ElasticSearchIndexAction;
+ refreshOnCompletion?: string | boolean;
+ }) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.helpers.bulk(bulkOptions);
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.helpers.bulk(bulkOptions);
+ }
+
+ throw new Error('No client defined');
+ }
+
+ putIndexTemplate(template: ElasticSearchCustomIndexTemplate) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.indices.putIndexTemplate(template);
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.indices.putIndexTemplate(template);
+ }
+
+ throw new Error('No client defined');
+ }
+
+ indexExists({ index }: { index: string | string[] }) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.indices.exists({ index });
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.indices.exists({ index });
+ }
+
+ throw new Error('No client defined');
+ }
+
+ deleteIndex({ index }: { index: string | string[] }) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.indices.delete({ index });
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.indices.delete({ index });
+ }
+
+ throw new Error('No client defined');
+ }
+
+ createIndex({ index }: { index: string }) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.indices.create({ index });
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.indices.create({ index });
+ }
+
+ throw new Error('No client defined');
+ }
+
+ getAliases({ aliases }: { aliases: string[] }) {
+ if (this.openSearchClient) {
+ return this.openSearchClient.cat.aliases({
+ format: 'json',
+ name: aliases,
+ });
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.cat.aliases({
+ format: 'json',
+ name: aliases,
+ });
+ }
+
+ throw new Error('No client defined');
+ }
+
+ updateAliases({ actions }: { actions: ElasticSearchAliasAction[] }) {
+ const filteredActions = actions.filter(Boolean);
+ if (this.openSearchClient) {
+ return this.openSearchClient.indices.updateAliases({
+ body: {
+ actions: filteredActions,
+ },
+ });
+ }
+
+ if (this.elasticSearchClient) {
+ return this.elasticSearchClient.indices.updateAliases({
+ body: {
+ actions: filteredActions,
+ },
+ });
+ }
+
+ throw new Error('No client defined');
+ }
+}
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts
index 7892b731ed..cc88f456b6 100644
--- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts
@@ -16,8 +16,9 @@
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
-import { Client, errors } from '@elastic/elasticsearch';
+import { errors } from '@elastic/elasticsearch';
import Mock from '@elastic/elasticsearch-mock';
+import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
import {
ElasticSearchConcreteQuery,
decodePageCursor,
@@ -70,7 +71,7 @@ const customIndexTemplate = {
describe('ElasticSearchSearchEngine', () => {
let testSearchEngine: ElasticSearchSearchEngine;
let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests;
- let client: Client;
+ let clientWrapper: ElasticSearchClientWrapper;
beforeEach(() => {
testSearchEngine = new ElasticSearchSearchEngine(
@@ -86,7 +87,7 @@ describe('ElasticSearchSearchEngine', () => {
getVoidLogger(),
);
// eslint-disable-next-line dot-notation
- client = testSearchEngine['elasticSearchClient'];
+ clientWrapper = testSearchEngine['elasticSearchClientWrapper'];
});
describe('custom index template', () => {
@@ -686,7 +687,7 @@ describe('ElasticSearchSearchEngine', () => {
});
it('should handle index/search type filtering correctly', async () => {
- const elasticSearchQuerySpy = jest.spyOn(client, 'search');
+ const elasticSearchQuerySpy = jest.spyOn(clientWrapper, 'search');
await testSearchEngine.query({
term: 'testTerm',
filters: {},
@@ -725,7 +726,7 @@ describe('ElasticSearchSearchEngine', () => {
});
it('should create matchAll query if no term defined', async () => {
- const elasticSearchQuerySpy = jest.spyOn(client, 'search');
+ const elasticSearchQuerySpy = jest.spyOn(clientWrapper, 'search');
await testSearchEngine.query({
term: '',
filters: {},
@@ -759,7 +760,7 @@ describe('ElasticSearchSearchEngine', () => {
});
it('should query only specified indices if defined', async () => {
- const elasticSearchQuerySpy = jest.spyOn(client, 'search');
+ const elasticSearchQuerySpy = jest.spyOn(clientWrapper, 'search');
await testSearchEngine.query({
term: '',
filters: {},
@@ -809,7 +810,7 @@ describe('ElasticSearchSearchEngine', () => {
type: 'test-index',
indexPrefix: '',
indexSeparator: '-index__',
- elasticSearchClient: client,
+ elasticSearchClientWrapper: clientWrapper,
}),
);
expect(indexerMock.on).toHaveBeenCalledWith(
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
index 31112e3446..4fa0d4e110 100644
--- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
@@ -14,10 +14,7 @@
* limitations under the License.
*/
-import {
- awsGetCredentials,
- createAWSConnection,
-} from '@acuris/aws-es-connection';
+import { awsGetCredentials, createAWSConnection } from 'aws-os-connection';
import { Config } from '@backstage/config';
import {
IndexableDocument,
@@ -26,47 +23,17 @@ import {
SearchEngine,
SearchQuery,
} from '@backstage/plugin-search-common';
-import { Client } from '@elastic/elasticsearch';
import esb from 'elastic-builder';
import { isEmpty, isNaN as nan, isNumber } from 'lodash';
import { v4 as uuid } from 'uuid';
import { Logger } from 'winston';
-import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
+import { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';
+import { ElasticSearchCustomIndexTemplate } from './types';
+import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
export type { ElasticSearchClientOptions };
-/**
- * Elasticsearch specific index template
- * @public
- */
-export type ElasticSearchCustomIndexTemplate = {
- name: string;
- body: ElasticSearchCustomIndexTemplateBody;
-};
-
-/**
- * Elasticsearch specific index template body
- * @public
- */
-export type ElasticSearchCustomIndexTemplateBody = {
- /**
- * Array of wildcard (*) expressions used to match the names of data streams and indices during creation.
- */
- index_patterns: string[];
- /**
- * An ordered list of component template names.
- * Component templates are merged in the order specified,
- * meaning that the last component template specified has the highest precedence.
- */
- composed_of?: string[];
- /**
- * See available properties of template
- * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body
- */
- template?: Record;
-};
-
/**
* Search query that the elasticsearch engine understands.
* @public
@@ -143,7 +110,7 @@ function isBlank(str: string) {
* @public
*/
export class ElasticSearchSearchEngine implements SearchEngine {
- private readonly elasticSearchClient: Client;
+ private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper;
private readonly highlightOptions: ElasticSearchHighlightConfig;
constructor(
@@ -153,7 +120,8 @@ export class ElasticSearchSearchEngine implements SearchEngine {
private readonly logger: Logger,
highlightOptions?: ElasticSearchHighlightOptions,
) {
- this.elasticSearchClient = this.newClient(options => new Client(options));
+ this.elasticSearchClientWrapper =
+ ElasticSearchClientWrapper.fromClientOptions(elasticSearchClientOptions);
const uuidTag = uuid();
this.highlightOptions = {
preTag: `<${uuidTag}>`,
@@ -177,7 +145,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
if (options.provider === 'elastic') {
logger.info('Initializing Elastic.co ElasticSearch search engine.');
} else if (options.provider === 'aws') {
- logger.info('Initializing AWS ElasticSearch search engine.');
+ logger.info('Initializing AWS OpenSearch search engine.');
} else {
logger.info('Initializing ElasticSearch search engine.');
}
@@ -269,7 +237,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
async setIndexTemplate(template: ElasticSearchCustomIndexTemplate) {
try {
- await this.elasticSearchClient.indices.putIndexTemplate(template);
+ await this.elasticSearchClientWrapper.putIndexTemplate(template);
this.logger.info('Custom index template set');
} catch (error) {
this.logger.error(`Unable to set custom index template: ${error}`);
@@ -284,7 +252,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
indexPrefix: this.indexPrefix,
indexSeparator: this.indexSeparator,
alias,
- elasticSearchClient: this.elasticSearchClient,
+ elasticSearchClientWrapper: this.elasticSearchClientWrapper,
logger: this.logger,
});
@@ -292,13 +260,13 @@ export class ElasticSearchSearchEngine implements SearchEngine {
indexer.on('error', async e => {
this.logger.error(`Failed to index documents for type ${type}`, e);
try {
- const response = await this.elasticSearchClient.indices.exists({
+ const response = await this.elasticSearchClientWrapper.indexExists({
index: indexer.indexName,
});
const indexCreated = response.body;
if (indexCreated) {
this.logger.info(`Removing created index ${indexer.indexName}`);
- await this.elasticSearchClient.indices.delete({
+ await this.elasticSearchClientWrapper.deleteIndex({
index: indexer.indexName,
});
}
@@ -319,7 +287,7 @@ export class ElasticSearchSearchEngine implements SearchEngine {
? documentTypes.map(it => this.constructSearchAlias(it))
: this.constructSearchAlias('*');
try {
- const result = await this.elasticSearchClient.search({
+ const result = await this.elasticSearchClientWrapper.search({
index: queryIndices,
body: elasticSearchQuery,
});
@@ -398,7 +366,7 @@ export function encodePageCursor({ page }: { page: number }): string {
return Buffer.from(`${page}`, 'utf-8').toString('base64');
}
-async function createElasticSearchClientOptions(
+export async function createElasticSearchClientOptions(
config?: Config,
): Promise {
if (!config) {
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts
index 4e0e8e229f..eff056c3f4 100644
--- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts
@@ -16,13 +16,13 @@
import { getVoidLogger } from '@backstage/backend-common';
import { TestPipeline } from '@backstage/plugin-search-backend-node';
-import { Client } from '@elastic/elasticsearch';
import Mock from '@elastic/elasticsearch-mock';
import { range } from 'lodash';
+import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';
const mock = new Mock();
-const client = new Client({
+const clientWrapper = ElasticSearchClientWrapper.fromClientOptions({
node: 'http://localhost:9200',
Connection: mock.getConnection(),
});
@@ -43,7 +43,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
indexSeparator: '-index__',
alias: 'some-type-index__search',
logger: getVoidLogger(),
- elasticSearchClient: client,
+ elasticSearchClientWrapper: clientWrapper,
});
// Set up all requisite Elastic mocks.
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts
index fd16d0c963..87a4472a15 100644
--- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts
+++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts
@@ -16,9 +16,9 @@
import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node';
import { IndexableDocument } from '@backstage/plugin-search-common';
-import { Client } from '@elastic/elasticsearch';
import { Readable } from 'stream';
import { Logger } from 'winston';
+import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
/**
* Options for instansiate ElasticSearchSearchEngineIndexer
@@ -30,7 +30,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
indexSeparator: string;
alias: string;
logger: Logger;
- elasticSearchClient: Client;
+ elasticSearchClientWrapper: ElasticSearchClientWrapper;
};
function duration(startTimestamp: [number, number]): string {
@@ -57,7 +57,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
private readonly removableAlias: string;
private readonly logger: Logger;
private readonly sourceStream: Readable;
- private readonly elasticSearchClient: Client;
+ private readonly elasticSearchClientWrapper: ElasticSearchClientWrapper;
private bulkResult: Promise;
constructor(options: ElasticSearchSearchEngineIndexerOptions) {
@@ -70,7 +70,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
this.indexName = this.constructIndexName(`${Date.now()}`);
this.alias = options.alias;
this.removableAlias = `${this.alias}_removable`;
- this.elasticSearchClient = options.elasticSearchClient;
+ this.elasticSearchClientWrapper = options.elasticSearchClientWrapper;
// The ES client bulk helper supports stream-based indexing, but we have to
// supply the stream directly to it at instantiation-time. We can't supply
@@ -83,7 +83,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
// Keep a reference to the ES Bulk helper so that we can know when all
// documents have been successfully written to ES.
- this.bulkResult = this.elasticSearchClient.helpers.bulk({
+ this.bulkResult = this.elasticSearchClientWrapper.bulk({
datasource: this.sourceStream,
onDocument() {
that.processed++;
@@ -98,16 +98,15 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
async initialize(): Promise {
this.logger.info(`Started indexing documents for index ${this.type}`);
- const aliases = await this.elasticSearchClient.cat.aliases({
- format: 'json',
- name: [this.alias, this.removableAlias],
+ const aliases = await this.elasticSearchClientWrapper.getAliases({
+ aliases: [this.alias, this.removableAlias],
});
this.removableIndices = [
...new Set(aliases.body.map((r: Record) => r.index)),
] as string[];
- await this.elasticSearchClient.indices.create({
+ await this.elasticSearchClientWrapper.createIndex({
index: this.indexName,
});
}
@@ -140,25 +139,23 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
)}`,
result,
);
- await this.elasticSearchClient.indices.updateAliases({
- body: {
- actions: [
- {
- remove: { index: this.constructIndexName('*'), alias: this.alias },
- },
- this.removableIndices.length
- ? {
- add: {
- indices: this.removableIndices,
- alias: this.removableAlias,
- },
- }
- : undefined,
- {
- add: { index: this.indexName, alias: this.alias },
- },
- ].filter(Boolean),
- },
+ await this.elasticSearchClientWrapper.updateAliases({
+ actions: [
+ {
+ remove: { index: this.constructIndexName('*'), alias: this.alias },
+ },
+ this.removableIndices.length
+ ? {
+ add: {
+ indices: this.removableIndices,
+ alias: this.removableAlias,
+ },
+ }
+ : undefined,
+ {
+ add: { index: this.indexName, alias: this.alias },
+ },
+ ].filter(Boolean),
});
// If any indices are removable, remove them. Do not bubble up this error,
@@ -166,7 +163,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
if (this.removableIndices.length) {
this.logger.info('Removing stale search indices', this.removableIndices);
try {
- await this.elasticSearchClient.indices.delete({
+ await this.elasticSearchClientWrapper.deleteIndex({
index: this.removableIndices,
});
} catch (e) {
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts
index 8ac86970a1..8c3b9e3c6d 100644
--- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts
+++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts
@@ -15,13 +15,25 @@
*/
export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine';
+export { isOpenSearchCompatible } from './ElasticSearchClientOptions';
export type {
+ BaseElasticSearchClientOptions,
ElasticSearchAgentOptions,
ElasticSearchConnectionConstructor,
+ ElasticSearchElasticSearchClientOptions,
ElasticSearchTransportConstructor,
ElasticSearchNodeOptions,
ElasticSearchAuth,
+ OpenSearchAuth,
+ OpenSearchConnectionConstructor,
+ OpenSearchElasticSearchClientOptions,
+ OpenSearchNodeOptions,
} from './ElasticSearchClientOptions';
+export type {
+ ElasticSearchAliasAction,
+ ElasticSearchClientWrapper,
+ ElasticSearchIndexAction,
+} from './ElasticSearchClientWrapper';
export type {
ElasticSearchConcreteQuery,
ElasticSearchClientOptions,
@@ -30,10 +42,12 @@ export type {
ElasticSearchQueryTranslator,
ElasticSearchQueryTranslatorOptions,
ElasticSearchOptions,
- ElasticSearchCustomIndexTemplate,
- ElasticSearchCustomIndexTemplateBody,
} from './ElasticSearchSearchEngine';
export type {
ElasticSearchSearchEngineIndexer,
ElasticSearchSearchEngineIndexerOptions,
} from './ElasticSearchSearchEngineIndexer';
+export type {
+ ElasticSearchCustomIndexTemplate,
+ ElasticSearchCustomIndexTemplateBody,
+} from './types';
diff --git a/plugins/search-backend-module-elasticsearch/src/engines/types.ts b/plugins/search-backend-module-elasticsearch/src/engines/types.ts
new file mode 100644
index 0000000000..a83e8b09b6
--- /dev/null
+++ b/plugins/search-backend-module-elasticsearch/src/engines/types.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Elasticsearch specific index template
+ * @public
+ */
+export type ElasticSearchCustomIndexTemplate = {
+ name: string;
+ body: ElasticSearchCustomIndexTemplateBody;
+};
+
+/**
+ * Elasticsearch specific index template body
+ * @public
+ */
+export type ElasticSearchCustomIndexTemplateBody = {
+ /**
+ * Array of wildcard (*) expressions used to match the names of data streams and indices during creation.
+ */
+ index_patterns: string[];
+ /**
+ * An ordered list of component template names.
+ * Component templates are merged in the order specified,
+ * meaning that the last component template specified has the highest precedence.
+ */
+ composed_of?: string[];
+ /**
+ * See available properties of template
+ * https://www.elastic.co/guide/en/elasticsearch/reference/7.15/indices-put-template.html#put-index-template-api-request-body
+ */
+ template?: Record;
+};
diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts
index adf7fa43f2..cb4f62098a 100644
--- a/plugins/search-backend-module-elasticsearch/src/index.ts
+++ b/plugins/search-backend-module-elasticsearch/src/index.ts
@@ -20,13 +20,18 @@
* @packageDocumentation
*/
-export { ElasticSearchSearchEngine } from './engines';
+export { ElasticSearchSearchEngine, isOpenSearchCompatible } from './engines';
export type {
+ BaseElasticSearchClientOptions,
ElasticSearchAgentOptions,
+ ElasticSearchAliasAction,
+ ElasticSearchClientWrapper,
ElasticSearchConcreteQuery,
ElasticSearchClientOptions,
+ ElasticSearchElasticSearchClientOptions,
ElasticSearchHighlightConfig,
ElasticSearchHighlightOptions,
+ ElasticSearchIndexAction,
ElasticSearchQueryTranslator,
ElasticSearchQueryTranslatorOptions,
ElasticSearchConnectionConstructor,
@@ -38,4 +43,8 @@ export type {
ElasticSearchSearchEngineIndexerOptions,
ElasticSearchCustomIndexTemplate,
ElasticSearchCustomIndexTemplateBody,
+ OpenSearchAuth,
+ OpenSearchConnectionConstructor,
+ OpenSearchElasticSearchClientOptions,
+ OpenSearchNodeOptions,
} from './engines';
diff --git a/yarn.lock b/yarn.lock
index 345aa3d510..015ab547a1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,13 +2,6 @@
# yarn lockfile v1
-"@acuris/aws-es-connection@^2.2.0":
- version "2.3.0"
- resolved "https://registry.npmjs.org/@acuris/aws-es-connection/-/aws-es-connection-2.3.0.tgz#bfa736163bdfe2e7fc12f8529eead949fb4e73fe"
- integrity sha512-k8jSkdNwpSPty7oautGaJXoAzzPjP7WKbmaAE9i+vJrKWg9n2UtjesHeDhOAPzV9MG8EzXofGozYjaLdeTP+Ug==
- dependencies:
- aws4 "^1.11.0"
-
"@ampproject/remapping@^2.1.0":
version "2.1.2"
resolved "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.1.2.tgz#4edca94973ded9630d20101cd8559cedb8d8bd34"
@@ -2079,10 +2072,10 @@
find-my-way "^4.3.3"
into-stream "^6.0.0"
-"@elastic/elasticsearch@7.13.0":
- version "7.13.0"
- resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.13.0.tgz#6dcf511dfa91187e22c81e54f41f4bd0fd96b4d6"
- integrity sha512-WgwLWo2p9P2tdqzBGX9fHeG8p5IOTXprXNTECQG2mJ7z9n93N5AFBJpEw4d35tWWeCWi9jI13A2wzQZH7XZ/xw==
+"@elastic/elasticsearch@^7.13.0":
+ version "7.17.0"
+ resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.17.0.tgz#589fb219234cf1b0da23744e82b1d25e2fe9a797"
+ integrity sha512-5QLPCjd0uLmLj1lSuKSThjNpq39f6NmlTy9ROLFwG5gjyTgpwSqufDeYG/Fm43Xs05uF7WcscoO7eguI3HuuYA==
dependencies:
debug "^4.3.1"
hpagent "^0.1.1"
@@ -5127,6 +5120,16 @@
rxjs "7.5.5"
tslib "2.0.3"
+"@opensearch-project/opensearch@^1.1.0":
+ version "1.1.0"
+ resolved "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7"
+ integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ==
+ dependencies:
+ debug "^4.3.1"
+ hpagent "^0.1.1"
+ ms "^2.1.3"
+ secure-json-parse "^2.4.0"
+
"@opentelemetry/api@^1.0.1":
version "1.0.4"
resolved "https://registry.npmjs.org/@opentelemetry/api/-/api-1.0.4.tgz#a167e46c10d05a07ab299fc518793b0cff8f6924"
@@ -5420,6 +5423,15 @@
dependencies:
any-observable "^0.3.0"
+"@short.io/opensearch-mock@^0.3.1":
+ version "0.3.1"
+ resolved "https://registry.npmjs.org/@short.io/opensearch-mock/-/opensearch-mock-0.3.1.tgz#41678404b94342ac60263a8df590f9b1efa80622"
+ integrity sha512-gwLTzxJrZNGVQNn+FHLhpw7at3jHb6tNczq2hvFPJ3C3nLaCBRkbP2IQHrIDzklLBVFV83Ug0Xgs73A5o9eLtg==
+ dependencies:
+ fast-deep-equal "^3.1.3"
+ find-my-way "^4.3.3"
+ into-stream "^6.0.0"
+
"@sideway/address@^4.1.0":
version "4.1.1"
resolved "https://registry.npmjs.org/@sideway/address/-/address-4.1.1.tgz#9e321e74310963fdf8eebfbee09c7bd69972de4d"
@@ -8250,6 +8262,13 @@ available-typed-arrays@^1.0.2:
dependencies:
array-filter "^1.0.0"
+aws-os-connection@^0.1.0:
+ version "0.1.0"
+ resolved "https://registry.npmjs.org/aws-os-connection/-/aws-os-connection-0.1.0.tgz#02c1be08cf63448c5bdc9d47b03b7ad3222b662c"
+ integrity sha512-WBN2S/5zgzfPpEiOTEW7IPJX+hlq+AQeAVKK4AL/Th0SfRRKu6a46i4aZKovQslBkKnmcSFvy+mHSJ299N8YWA==
+ dependencies:
+ aws4 "^1.11.0"
+
aws-sdk-mock@^5.2.1:
version "5.7.0"
resolved "https://registry.npmjs.org/aws-sdk-mock/-/aws-sdk-mock-5.7.0.tgz#b2a9454c78d008186c3f1a460b79cdb9cc9dfdc4"