diff --git a/.changeset/mighty-humans-shave.md b/.changeset/mighty-humans-shave.md new file mode 100644 index 0000000000..7a4971e021 --- /dev/null +++ b/.changeset/mighty-humans-shave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': patch +--- + +Support AWS OpenSearch Serverless search backend. Does not support `_refresh` endpoint. diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 408ec288d0..7de3a1857b 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -370,6 +370,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; + skipRefresh?: boolean; }; // @public (undocumented) diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts index 95f1588fb4..b798240cef 100644 --- a/plugins/search-backend-module-elasticsearch/config.d.ts +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -115,6 +115,19 @@ export interface Config { * Eg. https://my-es-cluster.eu-west-1.es.amazonaws.com */ node: string; + + /** + * The AWS region. + * Only needed if using a custom DNS record. + */ + region?: string; + + /** + * The AWS service used for request signature. + * Either 'es' for "Managed Clusters" or 'aoss' for "Serverless". + * Only needed if using a custom DNS record. + */ + service?: 'es' | 'aoss'; } /** diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 9b085b52b0..8ccbe9a4ec 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -26,7 +26,10 @@ import { isEmpty, isNumber, isNaN as nan } from 'lodash'; import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws'; import { RequestSigner } from 'aws4'; import { Config } from '@backstage/config'; -import { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; +import { + ElasticSearchClientOptions, + OpenSearchElasticSearchClientOptions, +} from './ElasticSearchClientOptions'; import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper'; import { ElasticSearchCustomIndexTemplate } from './types'; import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; @@ -302,6 +305,11 @@ export class ElasticSearchSearchEngine implements SearchEngine { elasticSearchClientWrapper: this.elasticSearchClientWrapper, logger: indexerLogger, batchSize: this.batchSize, + skipRefresh: + ( + this + .elasticSearchClientOptions as OpenSearchElasticSearchClientOptions + )?.service === 'aoss', }); // Attempt cleanup upon failure. @@ -473,6 +481,8 @@ export class ElasticSearchSearchEngine implements SearchEngine { return { provider: 'aws', node: config.getString('node'), + region: config.getOptionalString('region'), + service, ...(sslConfig ? { ssl: { diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts index 94b59fcf42..453a825574 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -34,6 +34,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { let createSpy: jest.Mock; let aliasesSpy: jest.Mock; let deleteSpy: jest.Mock; + let refreshSpy: jest.Mock; beforeEach(() => { // Instantiate the indexer to be tested. @@ -45,6 +46,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { logger: getVoidLogger(), elasticSearchClientWrapper: clientWrapper, batchSize: 1000, + skipRefresh: false, }); // Set up all requisite Elastic mocks. @@ -57,12 +59,13 @@ describe('ElasticSearchSearchEngineIndexer', () => { }, bulkSpy, ); + refreshSpy = jest.fn().mockReturnValue({}); mock.add( { method: 'GET', path: '/:index/_refresh', }, - jest.fn().mockReturnValue({}), + refreshSpy, ); catSpy = jest.fn().mockReturnValue([ @@ -212,6 +215,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { // Ensure multiple bulk requests were made. expect(bulkSpy).toHaveBeenCalledTimes(2); + expect(refreshSpy).toHaveBeenCalledTimes(1); // Ensure the first and last documents were included in the payloads. const docLocations: string[] = [ @@ -269,6 +273,7 @@ describe('ElasticSearchSearchEngineIndexer', () => { logger: getVoidLogger(), elasticSearchClientWrapper: mockClientWrapper, batchSize: 1000, + skipRefresh: false, }); // When the indexer is run in the test pipeline @@ -279,4 +284,37 @@ describe('ElasticSearchSearchEngineIndexer', () => { // Then the pipeline should have received the expected error expect(error).toBe(expectedError); }); + + it('indexes documents, skip refresh', async () => { + // Instantiate the indexer to be tested. + indexer = new ElasticSearchSearchEngineIndexer({ + type: 'some-type', + indexPrefix: '', + indexSeparator: '-index__', + alias: 'some-type-index__search', + logger: getVoidLogger(), + elasticSearchClientWrapper: clientWrapper, + batchSize: 1000, + skipRefresh: true, + }); + + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + { + title: 'Another test', + text: 'Some more text', + location: 'test/location/2', + }, + ]; + + await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute(); + + // Ensure bulk called but refresh not + expect(bulkSpy).toHaveBeenCalledTimes(1); + expect(refreshSpy).toHaveBeenCalledTimes(0); + }); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts index b742216d52..4b1307114a 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -33,6 +33,7 @@ export type ElasticSearchSearchEngineIndexerOptions = { logger: Logger | LoggerService; elasticSearchClientWrapper: ElasticSearchClientWrapper; batchSize: number; + skipRefresh?: boolean; }; function duration(startTimestamp: [number, number]): string { @@ -95,7 +96,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { index: { _index: that.indexName }, }; }, - refreshOnCompletion: that.indexName, + refreshOnCompletion: options.skipRefresh !== true, }); // Safely catch errors thrown by the bulk helper client, e.g. HTTP timeouts