paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #8147 from cowboyd/elastic-search-new-client

Browse Source 
Add ability to re-use search cluster configuration from ElasticSearchSearchEngine
This commit is contained in:
Camila Belo
2021-12-15 15:03:11 +01:00
committed by GitHub
parent 22354eef67 c47959856a
commit 11151655a0
8 changed files with 356 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/search-backend-module-elasticsearch/api-report.md
+94 -5
View File
@@ -3,18 +3,103 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Client } from '@elastic/elasticsearch';
/// <reference types="node" />
import { Config } from '@backstage/config';
import type { ConnectionOptions } from 'tls';
import { IndexableDocument } from '@backstage/search-common';
import { Logger as Logger_2 } from 'winston';
import { SearchEngine } from '@backstage/search-common';
import { SearchQuery } from '@backstage/search-common';
import { SearchResultSet } from '@backstage/search-common';
// Warning: (ae-missing-release-tag) "ElasticSearchClientOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
// Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "@backstage/plugin-search-backend-module-elasticsearch" does not have an export "ElasticSearchEngine"
//
// @public
export interface ElasticSearchClientOptions {
// Warning: (ae-forgotten-export) The symbol "ElasticSearchAgentOptions" needs to be exported by the entry point index.d.ts
//
// (undocumented)
agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchAuth" needs to be exported by the entry point index.d.ts
//
// (undocumented)
auth?: ElasticSearchAuth;
// (undocumented)
cloud?: {
id: string;
username?: string;
password?: string;
};
// (undocumented)
compression?: 'gzip';
// Warning: (ae-forgotten-export) The symbol "ElasticSearchConnectionConstructor" needs to be exported by the entry point index.d.ts
//
// (undocumented)
Connection?: ElasticSearchConnectionConstructor;
// (undocumented)
disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor';
// (undocumented)
enableMetaHeader?: boolean;
// (undocumented)
headers?: Record<string, any>;
// (undocumented)
maxRetries?: number;
// (undocumented)
name?: string | symbol;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchNodeOptions" needs to be exported by the entry point index.d.ts
//
// (undocumented)
node?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
// (undocumented)
nodeFilter?: (connection: any) => boolean;
// (undocumented)
nodes?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
// (undocumented)
nodeSelector?: ((connections: any[]) => any) | string;
// (undocumented)
opaqueIdPrefix?: string;
// (undocumented)
pingTimeout?: number;
// (undocumented)
provider?: 'aws' | 'elastic';
// (undocumented)
proxy?: string | URL;
// (undocumented)
requestTimeout?: number;
// (undocumented)
resurrectStrategy?: 'ping' | 'optimistic' | 'none';
// (undocumented)
sniffEndpoint?: string;
// (undocumented)
sniffInterval?: number | boolean;
// (undocumented)
sniffOnConnectionFault?: boolean;
// (undocumented)
sniffOnStart?: boolean;
// (undocumented)
ssl?: ConnectionOptions;
// (undocumented)
suggestCompression?: boolean;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchTransportConstructor" needs to be exported by the entry point index.d.ts
//
// (undocumented)
Transport?: ElasticSearchTransportConstructor;
}
// @public (undocumented)
export class ElasticSearchSearchEngine implements SearchEngine {
constructor(
elasticSearchClient: Client,
elasticSearchClientOptions: ElasticSearchClientOptions,
aliasPostfix: string,
indexPrefix: string,
logger: Logger_2,
@@ -22,11 +107,15 @@ export class ElasticSearchSearchEngine implements SearchEngine {
// Warning: (ae-forgotten-export) The symbol "ElasticSearchOptions" needs to be exported by the entry point index.d.ts
//
// (undocumented)
static fromConfig(
options: ElasticSearchOptions,
): Promise<ElasticSearchSearchEngine>;
static fromConfig({
logger,
config,
aliasPostfix,
indexPrefix,
}: ElasticSearchOptions): Promise<ElasticSearchSearchEngine>;
// (undocumented)
index(type: string, documents: IndexableDocument[]): Promise<void>;
newClient<T>(create: (options: ElasticSearchClientOptions) => T): T;
// (undocumented)
query(query: SearchQuery): Promise<SearchResultSet>;
// Warning: (ae-forgotten-export) The symbol "ElasticSearchQueryTranslator" needs to be exported by the entry point index.d.ts
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchClientOptions.ts
+125
View File
@@ -0,0 +1,125 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { ConnectionOptions as TLSConnectionOptions } from 'tls';
/**
* Options used to configure the `@elastic/elasticsearch` client and
* are what will be passed as an argument to the
* {@link ElasticSearchEngine.newClient} method
*
* They are drawn from the `ClientOptions` class of `@elastic/elasticsearch`,
* but are maintained separately so that this interface is not coupled to
*/
export interface ElasticSearchClientOptions {
provider?: 'aws' | 'elastic';
node?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
nodes?:
| string
| string[]
| ElasticSearchNodeOptions
| ElasticSearchNodeOptions[];
Transport?: ElasticSearchTransportConstructor;
Connection?: ElasticSearchConnectionConstructor;
maxRetries?: number;
requestTimeout?: number;
pingTimeout?: number;
sniffInterval?: number | boolean;
sniffOnStart?: boolean;
sniffEndpoint?: string;
sniffOnConnectionFault?: boolean;
resurrectStrategy?: 'ping' | 'optimistic' | 'none';
suggestCompression?: boolean;
compression?: 'gzip';
ssl?: TLSConnectionOptions;
agent?: ElasticSearchAgentOptions | ((opts?: any) => unknown) | false;
nodeFilter?: (connection: any) => boolean;
nodeSelector?: ((connections: any[]) => any) | string;
headers?: Record<string, any>;
opaqueIdPrefix?: string;
name?: string | symbol;
auth?: ElasticSearchAuth;
proxy?: string | URL;
enableMetaHeader?: boolean;
cloud?: {
id: string;
username?: string;
password?: string;
};
disablePrototypePoisoningProtection?: boolean | 'proto' | 'constructor';
}
export type ElasticSearchAuth =
| {
username: string;
password: string;
}
| {
apiKey:
| string
| {
id: string;
api_key: string;
};
};
export interface ElasticSearchNodeOptions {
url: URL;
id?: string;
agent?: ElasticSearchAgentOptions;
ssl?: TLSConnectionOptions;
headers?: Record<string, any>;
roles?: {
master: boolean;
data: boolean;
ingest: boolean;
ml: boolean;
};
}
export interface ElasticSearchAgentOptions {
keepAlive?: boolean;
keepAliveMsecs?: number;
maxSockets?: number;
maxFreeSockets?: number;
}
export interface ElasticSearchConnectionConstructor {
new (opts?: any): any;
statuses: {
ALIVE: string;
DEAD: string;
};
roles: {
MASTER: string;
DATA: string;
INGEST: string;
ML: string;
};
}
export interface ElasticSearchTransportConstructor {
new (opts?: any): any;
sniffReasons: {
SNIFF_ON_START: string;
SNIFF_INTERVAL: string;
SNIFF_ON_CONNECTION_FAULT: string;
DEFAULT: string;
};
}
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts
+8 -6
View File
@@ -15,7 +15,6 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { SearchEngine } from '@backstage/search-common';
import { Client } from '@elastic/elasticsearch';
import Mock from '@elastic/elasticsearch-mock';
import {
@@ -33,28 +32,31 @@ class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEng
}
const mock = new Mock();
const client = new Client({
const options = {
node: 'http://localhost:9200',
Connection: mock.getConnection(),
});
};
describe('ElasticSearchSearchEngine', () => {
let testSearchEngine: SearchEngine;
let testSearchEngine: ElasticSearchSearchEngine;
let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests;
let client: Client;
beforeEach(() => {
testSearchEngine = new ElasticSearchSearchEngine(
client,
options,
'search',
'',
getVoidLogger(),
);
inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests(
client,
options,
'search',
'',
getVoidLogger(),
);
// eslint-disable-next-line dot-notation
client = testSearchEngine['elasticSearchClient'];
});
describe('queryTranslator', () => {
plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts
+106 -86
View File
@@ -30,6 +30,10 @@ import esb from 'elastic-builder';
import { isEmpty, isNaN as nan, isNumber } from 'lodash';
import { Logger } from 'winston';
import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions';
export type { ElasticSearchClientOptions };
export type ConcreteElasticSearchQuery = {
documentTypes?: string[];
elasticSearchQuery: Object;
@@ -68,105 +72,49 @@ function isBlank(str: string) {
* @public
*/
export class ElasticSearchSearchEngine implements SearchEngine {
private readonly elasticSearchClient: Client;
constructor(
private readonly elasticSearchClient: Client,
private readonly elasticSearchClientOptions: ElasticSearchClientOptions,
private readonly aliasPostfix: string,
private readonly indexPrefix: string,
private readonly logger: Logger,
) {}
) {
this.elasticSearchClient = this.newClient(options => new Client(options));
}
static async fromConfig(options: ElasticSearchOptions) {
const {
logger,
config,
aliasPostfix = `search`,
indexPrefix = ``,
} = options;
static async fromConfig({
logger,
config,
aliasPostfix = `search`,
indexPrefix = ``,
}: ElasticSearchOptions) {
const options = await createElasticSearchClientOptions(
config.getConfig('search.elasticsearch'),
);
if (options.provider === 'elastic') {
logger.info('Initializing Elastic.co ElasticSearch search engine.');
} else if (options.provider === 'aws') {
logger.info('Initializing AWS ElasticSearch search engine.');
} else {
logger.info('Initializing ElasticSearch search engine.');
}
return new ElasticSearchSearchEngine(
await ElasticSearchSearchEngine.constructElasticSearchClient(
logger,
config.getConfig('search.elasticsearch'),
),
options,
aliasPostfix,
indexPrefix,
logger,
);
}
private static async constructElasticSearchClient(
logger: Logger,
config?: Config,
) {
if (!config) {
throw new Error('No elastic search config found');
}
const clientOptionsConfig = config.getOptionalConfig('clientOptions');
const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');
if (config.getOptionalString('provider') === 'elastic') {
logger.info('Initializing Elastic.co ElasticSearch search engine.');
const authConfig = config.getConfig('auth');
return new Client({
cloud: {
id: config.getString('cloudId'),
},
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
});
}
if (config.getOptionalString('provider') === 'aws') {
logger.info('Initializing AWS ElasticSearch search engine.');
const awsCredentials = await awsGetCredentials();
const AWSConnection = createAWSConnection(awsCredentials);
return new Client({
node: config.getString('node'),
...AWSConnection,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
});
}
logger.info('Initializing ElasticSearch search engine.');
const authConfig = config.getOptionalConfig('auth');
const auth =
authConfig &&
(authConfig.has('apiKey')
? {
apiKey: authConfig.getString('apiKey'),
}
: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
});
return new Client({
node: config.getString('node'),
auth,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
});
/**
* Create a custom search client from the derived elastic search
* configuration. This need not be the same client that the engine uses
* internally.
*/
newClient<T>(create: (options: ElasticSearchClientOptions) => T): T {
return create(this.elasticSearchClientOptions);
}
protected translator(query: SearchQuery): ConcreteElasticSearchQuery {
@@ -349,3 +297,75 @@ export function decodePageCursor(pageCursor?: string): { page: number } {
export function encodePageCursor({ page }: { page: number }): string {
return Buffer.from(`${page}`, 'utf-8').toString('base64');
}
async function createElasticSearchClientOptions(
config?: Config,
): Promise<ElasticSearchClientOptions> {
if (!config) {
throw new Error('No elastic search config found');
}
const clientOptionsConfig = config.getOptionalConfig('clientOptions');
const sslConfig = clientOptionsConfig?.getOptionalConfig('ssl');
if (config.getOptionalString('provider') === 'elastic') {
const authConfig = config.getConfig('auth');
return {
provider: 'elastic',
cloud: {
id: config.getString('cloudId'),
},
auth: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
},
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
};
}
if (config.getOptionalString('provider') === 'aws') {
const awsCredentials = await awsGetCredentials();
const AWSConnection = createAWSConnection(awsCredentials);
return {
provider: 'aws',
node: config.getString('node'),
...AWSConnection,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
};
}
const authConfig = config.getOptionalConfig('auth');
const auth =
authConfig &&
(authConfig.has('apiKey')
? {
apiKey: authConfig.getString('apiKey'),
}
: {
username: authConfig.getString('username'),
password: authConfig.getString('password'),
});
return {
node: config.getString('node'),
auth,
...(sslConfig
? {
ssl: {
rejectUnauthorized:
sslConfig?.getOptionalBoolean('rejectUnauthorized'),
},
}
: {}),
};
}
plugins/search-backend-module-elasticsearch/src/engines/index.ts
+4 -1
View File
@@ -15,4 +15,7 @@
*/
export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine';
export type { ConcreteElasticSearchQuery } from './ElasticSearchSearchEngine';
export type {
ConcreteElasticSearchQuery,
ElasticSearchClientOptions,
} from './ElasticSearchSearchEngine';
plugins/search-backend-module-elasticsearch/src/index.ts
+1
View File
@@ -21,3 +21,4 @@
*/
export { ElasticSearchSearchEngine } from './engines';
export type { ElasticSearchClientOptions } from './engines';