paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Merge pull request #8147 from cowboyd/elastic-search-new-client

Browse Source 
Add ability to re-use search cluster configuration from ElasticSearchSearchEngine
This commit is contained in:
Camila Belo
2021-12-15 15:03:11 +01:00
committed by GitHub
parent 22354eef67 c47959856a
commit 11151655a0
8 changed files with 356 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/add-new-client-method-to-elastic-search-egine.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/plugin-search-backend-module-elasticsearch': patch
---
Add `newClient()` method to re-use the configuration of the elastic search
engine with custom clients
docs/features/search/search-engines.md
+12
View File
@@ -90,6 +90,18 @@ within your instance. The configuration options are documented in the
The underlying functionality is using official ElasticSearch client version 7.x,
meaning that ElasticSearch version 7 is the only one confirmed to be supported.
Should you need to create your own bespoke search experiences that require more
than just a query translator (such as faceted search or Relay pagination), you
can access the configuration of the search engine in order to create new elastic
search clients. The version of the client need not be the same as one used
internally by the elastic search engine plugin. For example:
```typescript
import { Client } from '@elastic/elastic-search';
const client = searchEngine.newClient(options => new Client(options));
```
## Example configurations
### AWS
plugins/search-backend-module-elasticsearch/api-report.md
+94 -5
View File
@@ -3,18 +3,103 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Client } from '@elastic/elasticsearch';
/// <reference types="node" />
import { Config } from '@backstage/config';
import type { ConnectionOptions } from 'tls';
import { IndexableDocument } from '@backstage/search-common';
import { Logger as Logger_2 } from 'winston';
import { SearchEngine } from '@backstage/search-common';
import { SearchQuery } from '@backstage/search-common';
import { SearchResultSet } from '@backstage/search-common';
// Warning: (ae-missing-release-tag) "ElasticSearchClientOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
// Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "@backstage/plugin-search-backend-module-elasticsearch" does not have an export "ElasticSearchEngine"
//
// @public
export interface ElasticSearchClientOptions {
// Warning: (ae-forgotten-export) The symbol "ElasticSearchAgentOptions" needs to be exported by the entry point index.d.ts
//
// (undocumented)
agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchAuth" needs to be exported by the entry point index.d.ts
//
// (undocumented)
auth?: ElasticSearchAuth;
// (undocumented)
cloud?: {
id: string;
username?: string;
password?: string;
};
// (undocumented)
compression?: 'gzip';
// Warning: (ae-forgotten-export) The symbol "ElasticSearchConnectionConstructor" needs to be exported by the entry point index.d.ts
//
// (undocumented)
Connection?: ElasticSearchConnectionConstructor;
// (undocumented)
disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor';
// (undocumented)
enableMetaHeader?: boolean;
// (undocumented)
headers?: Record<string, any>;
// (undocumented)
maxRetries?: number;
// (undocumented)
name?: string | symbol;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchNodeOptions" needs to be exported by the entry point index.d.ts
//
// (undocumented)
node?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
// (undocumented)
nodeFilter?: (connection: any) => boolean;
// (undocumented)
nodes?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
// (undocumented)
nodeSelector?: ((connections: any[]) => any) | string;
// (undocumented)
opaqueIdPrefix?: string;
// (undocumented)
pingTimeout?: number;
// (undocumented)
provider?: 'aws' | 'elastic';
// (undocumented)
proxy?: string | URL;
// (undocumented)
requestTimeout?: number;
// (undocumented)
resurrectStrategy?: 'ping' | 'optimistic' | 'none';
// (undocumented)
sniffEndpoint?: string;
// (undocumented)
sniffInterval?: number | boolean;
// (undocumented)
sniffOnConnectionFault?: boolean;
// (undocumented)
sniffOnStart?: boolean;
// (undocumented)
ssl?: ConnectionOptions;
// (undocumented)
suggestCompression?: boolean;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchTransportConstructor" needs to be exported by the entry point index.d.ts
//
// (undocumented)
Transport?: ElasticSearchTransportConstructor;
}
// @public (undocumented)
export class ElasticSearchSearchEngine implements SearchEngine {
constructor(
elasticSearchClient: Client,
elasticSearchClientOptions: ElasticSearchClientOptions,
aliasPostfix: string,
indexPrefix: string,
logger: Logger_2,
@@ -22,11 +107,15 @@ export class ElasticSearchSearchEngine implements SearchEngine {
// Warning: (ae-forgotten-export) The symbol "ElasticSearchOptions" needs to be exported by the entry point index.d.ts
//
// (undocumented)
static fromConfig(
options: ElasticSearchOptions,
): Promise<ElasticSearchSearchEngine>;
static fromConfig({
logger,
config,
aliasPostfix,
indexPrefix,
}: ElasticSearchOptions): Promise<ElasticSearchSearchEngine>;
// (undocumented)
index(type: string, documents: IndexableDocument[]): Promise<void>;
newClient<T>(create: (options: ElasticSearchClientOptions) => T): T;
// (undocumented)
query(query: SearchQuery): Promise<SearchResultSet>;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchQueryTranslator" needs to be exported by the entry point index.d.ts
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts
+125
View File
@@ -0,0 +1,125 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { ConnectionOptions as TLSConnectionOptions } from 'tls';
/**
* Options used to configure the `@elastic/elasticsearch` client and
* are what will be passed as an argument to the
* {@link ElasticSearchEngine.newClient} method
*
* They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`,
* but are maintained separately so that this interface is not coupled to
*/
export interface ElasticSearchClientOptions {
provider?: 'aws' | 'elastic';
node?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
nodes?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
Transport?: ElasticSearchTransportConstructor;
Connection?: ElasticSearchConnectionConstructor;
maxRetries?: number;
requestTimeout?: number;
pingTimeout?: number;
sniffInterval?: number | boolean;
sniffOnStart?: boolean;
sniffEndpoint?: string;
sniffOnConnectionFault?: boolean;
resurrectStrategy?: 'ping' | 'optimistic' | 'none';
suggestCompression?: boolean;
compression?: 'gzip';
ssl?: TLSConnectionOptions;
agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false;
nodeFilter?: (connection: any) => boolean;
nodeSelector?: ((connections: any[]) => any) | string;
headers?: Record<string, any>;
opaqueIdPrefix?: string;
name?: string | symbol;
auth?: ElasticSearchAuth;
proxy?: string | URL;
enableMetaHeader?: boolean;
cloud?: {
id: string;
username?: string;
password?: string;
};
disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor';
}
export type ElasticSearchAuth =
| {
username: string;
password: string;
}
| {
apiKey:
| string
| {
id: string;
api_key: string;
};
};
export interface ElasticSearchNodeOptions {
url: URL;
id?: string;
agent?: ElasticSearchAgentOptions;
ssl?: TLSConnectionOptions;
headers?: Record<string, any>;
roles?: {
master: boolean;
data: boolean;
ingest: boolean;
ml: boolean;
};
}
export interface ElasticSearchAgentOptions {
keepAlive?: boolean;
keepAliveMsecs?: number;
maxSockets?: number;
maxFreeSockets?: number;
}
export interface ElasticSearchConnectionConstructor {
new (opts?: any): any;
statuses: {
ALIVE: string;
DEAD: string;
};
roles: {
MASTER: string;
DATA: string;
INGEST: string;
ML: string;
};
}
export interface ElasticSearchTransportConstructor {
new (opts?: any): any;
sniffReasons: {
SNIFF_ON_START: string;
SNIFF_INTERVAL: string;
SNIFF_ON_CONNECTION_FAULT: string;
DEFAULT: string;
};
}
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts
+8 -6
View File
@@ -15,7 +15,6 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { SearchEngine } from '@backstage/search-common';
import { Client } from '@elastic/elasticsearch';
import Mock from '@elastic/elasticsearch-mock';
import {
@@ -33,28 +32,31 @@ class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEng
}
const mock = new Mock();
const client = new Client({
const options = {
node: 'http://localhost:9200',
Connection: mock.getConnection(),
});
};
describe('ElasticSearchSearchEngine', () => {
let testSearchEngine: SearchEngine;
let testSearchEngine: ElasticSearchSearchEngine;
let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests;
let client: Client;
beforeEach(() => {
testSearchEngine = new ElasticSearchSearchEngine(
client,
options,
'search',
'',
getVoidLogger(),
);
inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests(
client,
options,
'search',
'',
getVoidLogger(),
);
// eslint-disable-next-line dot-notation
client = testSearchEngine['elasticSearchClient'];
});
describe('queryTranslator', () => {
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
+106 -86
View File
@@ -30,6 +30,10 @@ import esb from 'elastic-builder';
import { isEmpty, isNaN as nan, isNumber } from 'lodash';
import { Logger } from 'winston';
import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
export type { ElasticSearchClientOptions };
export type ConcreteElasticSearchQuery = {
documentTypes?: string[];
elasticSearchQuery: Object;
@@ -68,105 +72,49 @@ function isBlank(str: string) {
* @public
*/
export class ElasticSearchSearchEngine implements SearchEngine {
private readonly elasticSearchClient: Client;
constructor(
private readonly elasticSearchClient: Client,
private readonly elasticSearchClientOptions: ElasticSearchClientOptions,
private readonly aliasPostfix: string,
private readonly indexPrefix: string,
private readonly logger: Logger,
) {}
) {
this.elasticSearchClient = this.newClient(options => new Client(options));
}
static async fromConfig(options: ElasticSearchOptions) {
const {
logger,
config,
aliasPostfix = `search`,
indexPrefix = ``,
} = options;
static async fromConfig({
logger,
config,
aliasPostfix = `search`,
indexPrefix = ``,
}: ElasticSearchOptions) {
const options = await createElasticSearchClientOptions(
config.getConfig('search.elasticsearch'),
);
if (options.provider === 'elastic') {
logger.info('Initializing Elastic.co ElasticSearch search engine.');
} else if (options.provider === 'aws') {
logger.info('Initializing AWS ElasticSearch search engine.');
} else {
logger.info('Initializing ElasticSearch search engine.');
}
return new ElasticSearchSearchEngine(
await ElasticSearchSearchEngine.constructElasticSearchClient(
logger,
config.getConfig('search.elasticsearch'),
),
options,
aliasPostfix,
indexPrefix,
logger,
);
}
private static async constructElasticSearchClient(
logger: Logger,
config?: Config,
) {
if (!config) {
throw new Error('No elastic search config found');
}
const clientOptionsConfig = config.getOptionalConfig('clientOptions');
const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');
if (config.getOptionalString('provider') === 'elastic') {
logger.info('Initializing Elastic.co ElasticSearch search engine.');
const authConfig = config.getConfig('auth');
return new Client({
cloud: {
id: config.getString('cloudId'),
},
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
});
}
if (config.getOptionalString('provider') === 'aws') {
logger.info('Initializing AWS ElasticSearch search engine.');
const awsCredentials = await awsGetCredentials();
const AWSConnection = createAWSConnection(awsCredentials);
return new Client({
node: config.getString('node'),
...AWSConnection,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
});
}
logger.info('Initializing ElasticSearch search engine.');
const authConfig = config.getOptionalConfig('auth');
const auth =
authConfig &&
(authConfig.has('apiKey')
? {
apiKey: authConfig.getString('apiKey'),
}
: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
});
return new Client({
node: config.getString('node'),
auth,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
});
/**
* Create a custom search client from the derived elastic search
* configuration. This need not be the same client that the engine uses
* internally.
*/
newClient<T>(create: (options: ElasticSearchClientOptions) => T): T {
return create(this.elasticSearchClientOptions);
}
protected translator(query: SearchQuery): ConcreteElasticSearchQuery {
@@ -349,3 +297,75 @@ export function decodePageCursor(pageCursor?: string): { page: number } {
export function encodePageCursor({ page }: { page: number }): string {
return Buffer.from(`${page}`, 'utf-8').toString('base64');
}
async function createElasticSearchClientOptions(
config?: Config,
): Promise<ElasticSearchClientOptions> {
if (!config) {
throw new Error('No elastic search config found');
}
const clientOptionsConfig = config.getOptionalConfig('clientOptions');
const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');
if (config.getOptionalString('provider') === 'elastic') {
const authConfig = config.getConfig('auth');
return {
provider: 'elastic',
cloud: {
id: config.getString('cloudId'),
},
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
};
}
if (config.getOptionalString('provider') === 'aws') {
const awsCredentials = await awsGetCredentials();
const AWSConnection = createAWSConnection(awsCredentials);
return {
provider: 'aws',
node: config.getString('node'),
...AWSConnection,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
};
}
const authConfig = config.getOptionalConfig('auth');
const auth =
authConfig &&
(authConfig.has('apiKey')
? {
apiKey: authConfig.getString('apiKey'),
}
: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
});
return {
node: config.getString('node'),
auth,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
};
}
plugins/search-backend-module-elasticsearch/src/engines/index.ts
+4 -1
View File
@@ -15,4 +15,7 @@
*/
export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine';
export type { ConcreteElasticSearchQuery } from './ElasticSearchSearchEngine';
export type {
ConcreteElasticSearchQuery,
ElasticSearchClientOptions,
} from './ElasticSearchSearchEngine';
plugins/search-backend-module-elasticsearch/src/index.ts
+1
View File
@@ -21,3 +21,4 @@
*/
export { ElasticSearchSearchEngine } from './engines';
export type { ElasticSearchClientOptions } from './engines';