From 006df4a58152be772fbbc9acc312195625fcd83a Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Wed, 27 Sep 2023 22:02:47 -0500 Subject: [PATCH 1/9] Support AWS OpenSearch Serverless skipping _refresh call as it is not supported Signed-off-by: Andrew Ochsner --- .changeset/mighty-humans-shave.md | 5 +++ .../src/engines/ElasticSearchSearchEngine.ts | 13 +++++- .../ElasticSearchSearchEngineIndexer.test.ts | 40 ++++++++++++++++++- .../ElasticSearchSearchEngineIndexer.ts | 3 +- 4 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 .changeset/mighty-humans-shave.md diff --git a/.changeset/mighty-humans-shave.md b/.changeset/mighty-humans-shave.md new file mode 100644 index 0000000000..ba6d599983 --- /dev/null +++ b/.changeset/mighty-humans-shave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Support AWS OpenSearch Serverless search backend. Does not support \_refresh endpoint. diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 9b085b52b0..55690db25e 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -26,7 +26,10 @@ import { isEmpty, isNumber, isNaN as nan } from 'lodash'; import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws'; import { RequestSigner } from 'aws4'; import { Config } from '@backstage/config'; -import { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; +import { + ElasticSearchClientOptions, + OpenSearchElasticSearchClientOptions, +} from './ElasticSearchClientOptions'; import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { ElasticSearchCustomIndexTemplate } from './types'; import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; @@ -162,6 +165,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { logger.info('Initializing Elastic.co ElasticSearch search engine.'); } else if (clientOptions.provider === 'aws') { logger.info('Initializing AWS OpenSearch search engine.'); + logger.info(JSON.stringify(clientOptions)); } else if (clientOptions.provider === 'opensearch') { logger.info('Initializing OpenSearch search engine.'); } else { @@ -302,6 +306,11 @@ export class ElasticSearchSearchEngine implements SearchEngine { elasticSearchClientWrapper: this.elasticSearchClientWrapper, logger: indexerLogger, batchSize: this.batchSize, + skipRefresh: + ( + this + .elasticSearchClientOptions as OpenSearchElasticSearchClientOptions + )?.service === 'aoss', }); // Attempt cleanup upon failure. @@ -473,6 +482,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { return { provider: 'aws', node: config.getString('node'), + region: config.getOptionalString('region'), + service, ...(sslConfig ? { ssl: { diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 94b59fcf42..453a825574 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -34,6 +34,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { let createSpy: jest.Mock; let aliasesSpy: jest.Mock; let deleteSpy: jest.Mock; + let refreshSpy: jest.Mock; beforeEach(() => { // Instantiate the indexer to be tested. @@ -45,6 +46,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { logger: getVoidLogger(), elasticSearchClientWrapper: clientWrapper, batchSize: 1000, + skipRefresh: false, }); // Set up all requisite Elastic mocks. @@ -57,12 +59,13 @@ describe('ElasticSearchSearchEngineIndexer', () => { }, bulkSpy, ); + refreshSpy = jest.fn().mockReturnValue({}); mock.add( { method: 'GET', path: '/:index/_refresh', }, - jest.fn().mockReturnValue({}), + refreshSpy, ); catSpy = jest.fn().mockReturnValue([ @@ -212,6 +215,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { // Ensure multiple bulk requests were made. expect(bulkSpy).toHaveBeenCalledTimes(2); + expect(refreshSpy).toHaveBeenCalledTimes(1); // Ensure the first and last documents were included in the payloads. const docLocations: string[] = [ @@ -269,6 +273,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { logger: getVoidLogger(), elasticSearchClientWrapper: mockClientWrapper, batchSize: 1000, + skipRefresh: false, }); // When the indexer is run in the test pipeline @@ -279,4 +284,37 @@ describe('ElasticSearchSearchEngineIndexer', () => { // Then the pipeline should have received the expected error expect(error).toBe(expectedError); }); + + it('indexes documents, skip refresh', async () => { + // Instantiate the indexer to be tested. + indexer = new ElasticSearchSearchEngineIndexer({ + type: 'some-type', + indexPrefix: '', + indexSeparator: '-index__', + alias: 'some-type-index__search', + logger: getVoidLogger(), + elasticSearchClientWrapper: clientWrapper, + batchSize: 1000, + skipRefresh: true, + }); + + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + { + title: 'Another test', + text: 'Some more text', + location: 'test/location/2', + }, + ]; + + await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute(); + + // Ensure bulk called but refresh not + expect(bulkSpy).toHaveBeenCalledTimes(1); + expect(refreshSpy).toHaveBeenCalledTimes(0); + }); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index b742216d52..55782e1ead 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -33,6 +33,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; + skipRefresh: boolean; }; function duration(startTimestamp: [number, number]): string { @@ -95,7 +96,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { index: { _index: that.indexName }, }; }, - refreshOnCompletion: that.indexName, + refreshOnCompletion: !options.skipRefresh && that.indexName, }); // Safely catch errors thrown by the bulk helper client, e.g. HTTP timeouts From 622d841ab3007561846aca5ffafe41ea39bf7f41 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Wed, 27 Sep 2023 22:14:11 -0500 Subject: [PATCH 2/9] clean up changelog Signed-off-by: Andrew Ochsner --- .changeset/mighty-humans-shave.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/mighty-humans-shave.md b/.changeset/mighty-humans-shave.md index ba6d599983..7a4971e021 100644 --- a/.changeset/mighty-humans-shave.md +++ b/.changeset/mighty-humans-shave.md @@ -2,4 +2,4 @@ '@backstage/plugin-search-backend-module-elasticsearch': patch --- -Support AWS OpenSearch Serverless search backend. Does not support \_refresh endpoint. +Support AWS OpenSearch Serverless search backend. Does not support `_refresh` endpoint. From 9a90db82269cae8e9ca403cb22c682772f642d40 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Wed, 27 Sep 2023 22:18:29 -0500 Subject: [PATCH 3/9] add api-report Signed-off-by: Andrew Ochsner --- plugins/search-backend-module-elasticsearch/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 408ec288d0..fb1b73a729 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -370,6 +370,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; + skipRefresh: boolean; }; // @public (undocumented) From 634d685605c846355eaa437fd374ae6da834c8ab Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Wed, 27 Sep 2023 22:31:20 -0500 Subject: [PATCH 4/9] remove log message not needed Signed-off-by: Andrew Ochsner --- .../src/engines/ElasticSearchSearchEngine.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 55690db25e..8ccbe9a4ec 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -165,7 +165,6 @@ export class ElasticSearchSearchEngine implements SearchEngine { logger.info('Initializing Elastic.co ElasticSearch search engine.'); } else if (clientOptions.provider === 'aws') { logger.info('Initializing AWS OpenSearch search engine.'); - logger.info(JSON.stringify(clientOptions)); } else if (clientOptions.provider === 'opensearch') { logger.info('Initializing OpenSearch search engine.'); } else { From b8ce15b1e5bfa2f5184dc61c96af6c45addcd1aa Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Thu, 26 Oct 2023 14:43:35 -0500 Subject: [PATCH 5/9] make skipRefresh optional Signed-off-by: Andrew Ochsner --- plugins/search-backend-module-elasticsearch/api-report.md | 2 +- .../src/engines/ElasticSearchSearchEngineIndexer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index fb1b73a729..7de3a1857b 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -370,7 +370,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; - skipRefresh: boolean; + skipRefresh?: boolean; }; // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 55782e1ead..55859ab9e6 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -33,7 +33,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; - skipRefresh: boolean; + skipRefresh?: boolean; }; function duration(startTimestamp: [number, number]): string { From 6fc10ed06e820e2ca398d11a90b881b59e149d39 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Sun, 29 Oct 2023 13:56:37 -0500 Subject: [PATCH 6/9] missed a file Signed-off-by: Andrew Ochsner --- .../src/engines/ElasticSearchSearchEngineIndexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 55859ab9e6..4c165886a3 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -96,7 +96,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { index: { _index: that.indexName }, }; }, - refreshOnCompletion: !options.skipRefresh && that.indexName, + refreshOnCompletion: options.skipRefresh !== true && that.indexName, }); // Safely catch errors thrown by the bulk helper client, e.g. HTTP timeouts From 1a9270c405b6a37e9114ac6603d1d1f8b9f6438d Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Sun, 29 Oct 2023 14:07:55 -0500 Subject: [PATCH 7/9] add optinal region to config schema Signed-off-by: Andrew Ochsner --- plugins/search-backend-module-elasticsearch/config.d.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 95f1588fb4..fa05378259 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -115,6 +115,12 @@ export interface Config { * Eg. https://my-es-cluster.eu-west-1.es.amazonaws.com */ node: string; + + /** + * The AWS region. + * Only needed if using a custom DNS record + */ + region?: string; } /** From 8ede5296539ac4371e6e7fe6fdb9ce2b47a6d8b8 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Sun, 29 Oct 2023 14:12:19 -0500 Subject: [PATCH 8/9] pass true or false not index name Signed-off-by: Andrew Ochsner --- .../src/engines/ElasticSearchSearchEngineIndexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index 4c165886a3..4b1307114a 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -96,7 +96,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { index: { _index: that.indexName }, }; }, - refreshOnCompletion: options.skipRefresh !== true && that.indexName, + refreshOnCompletion: options.skipRefresh !== true, }); // Safely catch errors thrown by the bulk helper client, e.g. HTTP timeouts From 5ef8b6aa39990f5e35706b54c77af5607afd94b5 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Sun, 29 Oct 2023 14:35:52 -0500 Subject: [PATCH 9/9] add 'service' to config schema Signed-off-by: Andrew Ochsner --- plugins/search-backend-module-elasticsearch/config.d.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index fa05378259..b798240cef 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -118,9 +118,16 @@ export interface Config { /** * The AWS region. - * Only needed if using a custom DNS record + * Only needed if using a custom DNS record. */ region?: string; + + /** + * The AWS service used for request signature. + * Either 'es' for "Managed Clusters" or 'aoss' for "Serverless". + * Only needed if using a custom DNS record. + */ + service?: 'es' | 'aoss'; } /**