Merge pull request #20212 from aochsner/feature/support-aoss

Support AWS OpenSearch Serverless

.changeset/mighty-humans-shave.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-search-backend-module-elasticsearch': patch
+---
+
+Support AWS OpenSearch Serverless search backend. Does not support `_refresh` endpoint.

plugins/search-backend-module-elasticsearch/api-report.md

@@ -370,6 +370,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
   logger: Logger | LoggerService;
   elasticSearchClientWrapper: ElasticSearchClientWrapper;
   batchSize: number;
+  skipRefresh?: boolean;
 };
 
 // @public (undocumented)

plugins/search-backend-module-elasticsearch/config.d.ts

@@ -115,6 +115,19 @@ export interface Config {
              * Eg. https://my-es-cluster.eu-west-1.es.amazonaws.com
              */
             node: string;
+
+            /**
+             * The AWS region.
+             * Only needed if using a custom DNS record.
+             */
+            region?: string;
+
+            /**
+             * The AWS service used for request signature.
+             * Either 'es' for "Managed Clusters" or 'aoss' for "Serverless".
+             * Only needed if using a custom DNS record.
+             */
+            service?: 'es' | 'aoss';
           }
 
         /**

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts

@@ -26,7 +26,10 @@ import { isEmpty, isNumber, isNaN as nan } from 'lodash';
 import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws';
 import { RequestSigner } from 'aws4';
 import { Config } from '@backstage/config';
-import { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
+import {
+  ElasticSearchClientOptions,
+  OpenSearchElasticSearchClientOptions,
+} from './ElasticSearchClientOptions';
 import { ElasticSearchClientWrapper } from './ElasticSearchClientWrapper';
 import { ElasticSearchCustomIndexTemplate } from './types';
 import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer';
@@ -302,6 +305,11 @@ export class ElasticSearchSearchEngine implements SearchEngine {
       elasticSearchClientWrapper: this.elasticSearchClientWrapper,
       logger: indexerLogger,
       batchSize: this.batchSize,
+      skipRefresh:
+        (
+          this
+            .elasticSearchClientOptions as OpenSearchElasticSearchClientOptions
+        )?.service === 'aoss',
     });
 
     // Attempt cleanup upon failure.
@@ -473,6 +481,8 @@ export class ElasticSearchSearchEngine implements SearchEngine {
       return {
         provider: 'aws',
         node: config.getString('node'),
+        region: config.getOptionalString('region'),
+        service,
         ...(sslConfig
           ? {
               ssl: {

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts

@@ -34,6 +34,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
   let createSpy: jest.Mock;
   let aliasesSpy: jest.Mock;
   let deleteSpy: jest.Mock;
+  let refreshSpy: jest.Mock;
 
   beforeEach(() => {
     // Instantiate the indexer to be tested.
@@ -45,6 +46,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
       logger: getVoidLogger(),
       elasticSearchClientWrapper: clientWrapper,
       batchSize: 1000,
+      skipRefresh: false,
     });
 
     // Set up all requisite Elastic mocks.
@@ -57,12 +59,13 @@ describe('ElasticSearchSearchEngineIndexer', () => {
       },
       bulkSpy,
     );
+    refreshSpy = jest.fn().mockReturnValue({});
     mock.add(
       {
         method: 'GET',
         path: '/:index/_refresh',
       },
-      jest.fn().mockReturnValue({}),
+      refreshSpy,
     );
 
     catSpy = jest.fn().mockReturnValue([
@@ -212,6 +215,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
 
     // Ensure multiple bulk requests were made.
     expect(bulkSpy).toHaveBeenCalledTimes(2);
+    expect(refreshSpy).toHaveBeenCalledTimes(1);
 
     // Ensure the first and last documents were included in the payloads.
     const docLocations: string[] = [
@@ -269,6 +273,7 @@ describe('ElasticSearchSearchEngineIndexer', () => {
       logger: getVoidLogger(),
       elasticSearchClientWrapper: mockClientWrapper,
       batchSize: 1000,
+      skipRefresh: false,
     });
 
     // When the indexer is run in the test pipeline
@@ -279,4 +284,37 @@ describe('ElasticSearchSearchEngineIndexer', () => {
     // Then the pipeline should have received the expected error
     expect(error).toBe(expectedError);
   });
+
+  it('indexes documents, skip refresh', async () => {
+    // Instantiate the indexer to be tested.
+    indexer = new ElasticSearchSearchEngineIndexer({
+      type: 'some-type',
+      indexPrefix: '',
+      indexSeparator: '-index__',
+      alias: 'some-type-index__search',
+      logger: getVoidLogger(),
+      elasticSearchClientWrapper: clientWrapper,
+      batchSize: 1000,
+      skipRefresh: true,
+    });
+
+    const documents = [
+      {
+        title: 'testTerm',
+        text: 'testText',
+        location: 'test/location',
+      },
+      {
+        title: 'Another test',
+        text: 'Some more text',
+        location: 'test/location/2',
+      },
+    ];
+
+    await TestPipeline.fromIndexer(indexer).withDocuments(documents).execute();
+
+    // Ensure bulk called but refresh not
+    expect(bulkSpy).toHaveBeenCalledTimes(1);
+    expect(refreshSpy).toHaveBeenCalledTimes(0);
+  });
 });

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts

@@ -33,6 +33,7 @@ export type ElasticSearchSearchEngineIndexerOptions = {
   logger: Logger | LoggerService;
   elasticSearchClientWrapper: ElasticSearchClientWrapper;
   batchSize: number;
+  skipRefresh?: boolean;
 };
 
 function duration(startTimestamp: [number, number]): string {
@@ -95,7 +96,7 @@ export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer {
           index: { _index: that.indexName },
         };
       },
-      refreshOnCompletion: that.indexName,
+      refreshOnCompletion: options.skipRefresh !== true,
     });
 
     // Safely catch errors thrown by the bulk helper client, e.g. HTTP timeouts