From d9c13d535be453c4e84bb28fd7180dee033b6843 Mon Sep 17 00:00:00 2001 From: Jussi Hallila Date: Wed, 21 Jul 2021 16:45:16 +0200 Subject: [PATCH] Implement ElasticSearch search engine * Adding indexing, searching and default translator for ElasticSearch engine * Modifying default backend to use ES if it is enabled * Adding configuration schema to configure ElasticSearch 3 different ways * Elastic.co hosted solution * AWS hosted ElasticSearch Service * Custom, using standard ElasticSearch URL and auth info * Add and modify some of the documentation regarding search Signed-off-by: Jussi Hallila --- .changeset/two-ravens-warn.md | 72 +++ .github/styles/vocab.txt | 1 + docs/features/search/getting-started.md | 10 +- docs/features/search/search-engines.md | 141 ++++++ packages/backend/package.json | 1 + packages/backend/src/plugins/search.ts | 9 +- packages/search-common/api-report.md | 14 + packages/search-common/src/types.ts | 28 ++ .../.eslintrc.js | 3 + .../README.md | 9 + .../api-report.md | 52 +++ .../config.d.ts | 104 +++++ .../package.json | 44 ++ .../engines/ElasticSearchSearchEngine.test.ts | 432 ++++++++++++++++++ .../src/engines/ElasticSearchSearchEngine.ts | 274 +++++++++++ .../src/engines/index.ts | 18 + .../src/index.ts | 17 + plugins/search-backend-node/api-report.md | 13 +- .../search-backend-node/src/IndexBuilder.ts | 2 +- .../src/engines/LunrSearchEngine.test.ts | 2 +- .../src/engines/LunrSearchEngine.ts | 3 +- plugins/search-backend-node/src/index.ts | 6 +- plugins/search-backend-node/src/types.ts | 36 +- yarn.lock | 152 +++++- 24 files changed, 1385 insertions(+), 58 deletions(-) create mode 100644 .changeset/two-ravens-warn.md create mode 100644 docs/features/search/search-engines.md create mode 100644 plugins/search-backend-module-elasticsearch/.eslintrc.js create mode 100644 plugins/search-backend-module-elasticsearch/README.md create mode 100644 plugins/search-backend-module-elasticsearch/api-report.md create mode 100644 plugins/search-backend-module-elasticsearch/config.d.ts create mode 100644 plugins/search-backend-module-elasticsearch/package.json create mode 100644 plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts create mode 100644 plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts create mode 100644 plugins/search-backend-module-elasticsearch/src/engines/index.ts create mode 100644 plugins/search-backend-module-elasticsearch/src/index.ts diff --git a/.changeset/two-ravens-warn.md b/.changeset/two-ravens-warn.md new file mode 100644 index 0000000000..348d4aff21 --- /dev/null +++ b/.changeset/two-ravens-warn.md @@ -0,0 +1,72 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +'@backstage/search-common': patch +'@backstage/plugin-search-backend-node': patch +--- + +Implements configuration and indexing functionality for ElasticSearch search engine. Adds indexing, searching and default translator for ElasticSearch and modifies default backend example-app to use ES if it is configured. + +## Example configurations: + +### AWS + +Using AWS hosted ElasticSearch the only configuration options needed is the URL to the ElasticSearch service. The implementation assumes +that environment variables for AWS access key id and secret access key are defined in accordance to the [default AWS credential chain.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html). + +```yaml +search: + elasticSearch: + provider: aws + node: https://my-backstage-search-asdfqwerty.eu-west-1.es.amazonaws.com +``` + +### Elastic.co + +Elastic Cloud hosted ElasticSearch uses a Cloud ID to determine the instance of hosted ElasticSearch to connect to. Additionally, username and password needs to be provided either directly or using environment variables like defined in [Backstage documentation.](https://backstage.io/docs/conf/writing#includes-and-dynamic-data) + +```yaml +search: + elasticSearch: + provider: elastic + cloudId: backstage-elastic:asdfqwertyasdfqwertyasdfqwertyasdfqwerty== + auth: + username: elastic + password: changeme +``` + +### Others + +Other ElasticSearch instances can be connected to by using standard ElasticSearch authentication methods and exposed URL, provided that the cluster supports that. The configuration options needed are the URL to the node and authentication information. Authentication can be handled by either providing username/password or and API key or a bearer token. In case both username/password combination and one of the tokens are provided, token takes precedence. For more information how to create an API key, see [Elastic documentation on API keys](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html) and how to create a bearer token, see [Elastic documentation on tokens.](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html) + +#### Configuration examples + +##### With username and password + +```yaml +search: + elasticSearch: + node: http://localhost:9200 + auth: + username: elastic + password: changeme +``` + +##### With bearer token + +```yaml +search: + elasticSearch: + node: http://localhost:9200 + auth: + bearer: token +``` + +##### With API key + +```yaml +search: + elasticSearch: + node: http://localhost:9200 + auth: + apiKey: base64EncodedKey +``` diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 48740ef367..f098788ae6 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -75,6 +75,7 @@ dockerode Docusaurus env Env +elasticsearch esbuild eslint etag diff --git a/docs/features/search/getting-started.md b/docs/features/search/getting-started.md index 2bd26faf8b..79ef267ea5 100644 --- a/docs/features/search/getting-started.md +++ b/docs/features/search/getting-started.md @@ -252,13 +252,9 @@ an example: Backstage Search isn't a search engine itself, rather, it provides an interface between your Backstage instance and a [Search Engine](./concepts.md#search-engines) of your choice. Currently, we only -support one, an in-memory search Engine called Lunr. It can be instantiated like -this: - -```typescript -const searchEngine = new LunrSearchEngine({ logger }); -const indexBuilder = new IndexBuilder({ logger, searchEngine }); -``` +support two engines, an in-memory search Engine called Lunr and ElasticSearch. +See [Search Engines](./search-engines.md) documentation for more information how +to configure these in your Backstage instance. Backstage Search can be used to power search of anything! Plugins like the Catalog offer default [collators](./concepts.md#collators) (e.g. diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md new file mode 100644 index 0000000000..0cb0f3507d --- /dev/null +++ b/docs/features/search/search-engines.md @@ -0,0 +1,141 @@ +--- +id: search-engines +title: Search Engines +description: Choosing and configuring your search engine for Backstage +--- + +# Search Engines + +Backstage supports 2 search engines by default, an in-memory engine called Lunr +and ElasticSearch. You can configure your own search engines by implementing the +provided interface as mentioned in the +[search backend documentation.](./getting-started.md#Backend) + +Provided search engine implementations have their own way of constructing +queries, which may be something you want to modify. Alterations to the querying +logic of a search engine can be made by providing your own implementation of a +QueryTranslator interface. This modification can be done without touching +provided search engines by using the exposed setter to set the modified query +translator into the instance. + +``` +const searchEngine = new LunrSearchEngine({ logger }); +searchEngine.setTranslator(new MyNewAndBetterQueryTranslator()); +``` + +## Lunr + +Lunr search engine is enabled by default for your backstage instance if you have +not done additional changes to the scaffolded app. + +Lunr can be instantiated like this: + +```typescript +// app/backend/src/plugins/search.ts +const searchEngine = new LunrSearchEngine({ logger }); +const indexBuilder = new IndexBuilder({ logger, searchEngine }); +``` + +## ElasticSearch + +Backstage supports ElasticSearch search engine connections, indexing and +querying out of the box. Available configuration options enable usage of either +AWS or Elastic.co hosted solutions, or a custom self-hosted solution. + +Similarly to Lunr above, ElasticSearch can be set up like this: + +```typescript +// app/backend/src/plugins/search.ts +const searchEngine = await ElasticSearchSearchEngine.initialize({ + logger, + config, +}); +const indexBuilder = new IndexBuilder({ logger, searchEngine }); +``` + +For the engine to be available, your backend package needs a dependency into +package `@backstage/plugin-search-backend-module-elasticsearch`. + +ElasticSearch needs some additional configuration before it is ready to use +within your instance. The configuration options are documented in the +[configuration schema definition file.](https://github.com/backstage/backstage/blob/master/plugins/search-backend-module-elasticsearch/config.d.ts) + +## Example configurations + +### AWS + +Using AWS hosted ElasticSearch the only configuration option needed is the URL +to the ElasticSearch service. The implementation assumes that environment +variables for AWS access key id and secret access key are defined in accordance +to the +[default AWS credential chain.](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html). + +```yaml +search: + elasticSearch: + provider: aws + node: https://my-backstage-search-asdfqwerty.eu-west-1.es.amazonaws.com +``` + +### Elastic.co + +Elastic Cloud hosted ElasticSearch uses a Cloud ID to determine the instance of +hosted ElasticSearch to connect to. Additionally, username and password needs to +be provided either directly or using environment variables like defined in +[Backstage documentation.](https://backstage.io/docs/conf/writing#includes-and-dynamic-data) + +```yaml +search: + elasticSearch: + provider: elastic + cloudId: backstage-elastic:asdfqwertyasdfqwertyasdfqwertyasdfqwerty== + auth: + username: elastic + password: changeme +``` + +### Others + +Other ElasticSearch instances can be connected to by using standard +ElasticSearch authentication methods and exposed URL, provided that the cluster +supports that. The configuration options needed are the URL to the node and +authentication information. Authentication can be handled by either providing +username/password or an API key or a bearer token. In case both +username/password combination and one of the tokens are provided, token takes +precedence. For more information how to create an API key, see +[Elastic documentation on API keys](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html), +and how to create a bearer token see +[Elastic documentation on tokens.](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html) + +#### Configuration examples + +##### With username and password + +```yaml +search: + elasticSearch: + node: http://localhost:9200 + auth: + username: elastic + password: changeme +``` + +##### With bearer token + +```yaml +search: + elasticSearch: + node: http://localhost:9200 + auth: + bearer: token +``` + +##### With API key + +```yaml +search: + elasticSearch: + node: http://localhost:9200 + auth: + apiKey: base64EncodedKey +``` diff --git a/packages/backend/package.json b/packages/backend/package.json index 9fc455801b..5d4ff60d68 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -47,6 +47,7 @@ "@backstage/plugin-scaffolder-backend-module-rails": "^0.1.3", "@backstage/plugin-search-backend": "^0.2.3", "@backstage/plugin-search-backend-node": "^0.4.0", + "@backstage/plugin-search-backend-module-elasticsearch": "^0.0.1", "@backstage/plugin-techdocs-backend": "^0.9.0", "@backstage/plugin-todo-backend": "^0.1.8", "@gitbeaker/node": "^30.2.0", diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 7e7f6ae400..eb13bc9df6 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -22,13 +22,20 @@ import { import { PluginEnvironment } from '../types'; import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; export default async function createPlugin({ logger, discovery, + config, }: PluginEnvironment) { // Initialize a connection to a search engine. - const searchEngine = new LunrSearchEngine({ logger }); + const searchEngine = config.has('search.elasticSearch') + ? await ElasticSearchSearchEngine.fromConfig({ + logger, + config, + }) + : new LunrSearchEngine({ logger }); const indexBuilder = new IndexBuilder({ logger, searchEngine }); // Collators are responsible for gathering documents known to plugins. This diff --git a/packages/search-common/api-report.md b/packages/search-common/api-report.md index 504f9afb9c..055f2d152f 100644 --- a/packages/search-common/api-report.md +++ b/packages/search-common/api-report.md @@ -32,6 +32,20 @@ export interface IndexableDocument { title: string; } +// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export type QueryTranslator = (query: SearchQuery) => unknown; + +// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface SearchEngine { + index(type: string, documents: IndexableDocument[]): Promise; + query(query: SearchQuery): Promise; + setTranslator(translator: QueryTranslator): void; +} + // Warning: (ae-missing-release-tag) "SearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 8a5e16b6b5..2a4447efac 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -79,3 +79,31 @@ export interface DocumentDecorator { readonly types?: string[]; execute(documents: IndexableDocument[]): Promise; } + +/** + * A type of function responsible for translating an abstract search query into + * a concrete query relevant to a particular search engine. + */ +export type QueryTranslator = (query: SearchQuery) => unknown; + +/** + * Interface that must be implemented by specific search engines, responsible + * for performing indexing and querying and translating abstract queries into + * concrete, search engine-specific queries. + */ +export interface SearchEngine { + /** + * Override the default translator provided by the SearchEngine. + */ + setTranslator(translator: QueryTranslator): void; + + /** + * Add the given documents to the SearchEngine index of the given type. + */ + index(type: string, documents: IndexableDocument[]): Promise; + + /** + * Perform a search query against the SearchEngine. + */ + query(query: SearchQuery): Promise; +} diff --git a/plugins/search-backend-module-elasticsearch/.eslintrc.js b/plugins/search-backend-module-elasticsearch/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/search-backend-module-elasticsearch/README.md b/plugins/search-backend-module-elasticsearch/README.md new file mode 100644 index 0000000000..fc9f3a4f99 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/README.md @@ -0,0 +1,9 @@ +# search-backend-module-elasticsearch + +This is an extension to module to search-backend-node plugin, which provides basic types, interfaces and functionality to implement search process within Backstage. + +This module provides functionality to index and implement querying using ElasticSearch engine. The module exposes configuration options to connect Backstage to your ElasticSearch cluster to be used as search engine. + +## Getting started + +See [Backstage documentation](https://backstage.io/docs/features/search/getting-started) for details on how to install and configure ElasticSearch for your Backstage instance. diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md new file mode 100644 index 0000000000..eac76a30b9 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -0,0 +1,52 @@ +## API Report File for "@backstage/plugin-search-backend-module-elasticsearch" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { Client } from '@elastic/elasticsearch'; +import { Config } from '@backstage/config'; +import { IndexableDocument } from '@backstage/search-common'; +import { Logger as Logger_2 } from 'winston'; +import { SearchEngine } from '@backstage/search-common'; +import { SearchQuery } from '@backstage/search-common'; +import { SearchResultSet } from '@backstage/search-common'; + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class ElasticSearchSearchEngine implements SearchEngine { + constructor( + elasticSearchClient: Client, + aliasPostfix: string, + indexPrefix: string, + logger: Logger_2, + ); + // Warning: (ae-forgotten-export) The symbol "ElasticSearchOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static fromConfig({ + logger, + config, + aliasPostfix, + indexPrefix, + }: ElasticSearchOptions): Promise; + // (undocumented) + index(type: string, documents: IndexableDocument[]): Promise; + // (undocumented) + query(query: SearchQuery): Promise; + // Warning: (ae-forgotten-export) The symbol "ElasticSearchQueryTranslator" needs to be exported by the entry point index.d.ts + // + // (undocumented) + setTranslator(translator: ElasticSearchQueryTranslator): void; + // Warning: (ae-forgotten-export) The symbol "ConcreteElasticSearchQuery" needs to be exported by the entry point index.d.ts + // + // (undocumented) + protected translator({ + term, + filters, + types, + }: SearchQuery): ConcreteElasticSearchQuery; +} + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/search-backend-module-elasticsearch/config.d.ts b/plugins/search-backend-module-elasticsearch/config.d.ts new file mode 100644 index 0000000000..d808166574 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/config.d.ts @@ -0,0 +1,104 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** Configuration options for the search plugin */ + search?: { + /** + * Options for ElasticSearch + */ + elasticSearch?: + | // elastic = Elastic.co ElasticSearch provider + { + provider: 'elastic'; + + /** + * Elastic.co CloudID + * See: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#authentication + */ + cloudId: string; + + auth: { + username: string; + + /** + * @visibility secret + */ + password: string; + }; + } + + /** + * AWS = Amazon Elasticsearch Service provider + * + * Authentication is handled using the default AWS credentials provider chain + */ + | { + provider: 'aws'; + + /** + * Node configuration. + * URL AWS ES endpoint to connect to. + * Eg. https://my-es-cluster.eu-west-1.es.amazonaws.com + */ + node: string; + } + + /** + * Standard ElasticSearch + * + * Includes self-hosted clusters and others that provide direct connection via an endpoint + * and authentication method (see possible authentication options below) + */ + | { + /** + * Node configuration. + * URL/URLS to ElasticSearch node to connect to. + * Either direct URL like 'https://localhost:9200' or with credentials like 'https://username:password@localhost:9200' + */ + node: string | string[]; + + /** + * Authentication credentials for ElasticSearch + * If both ApiKey/Bearer token and username+password is provided, tokens take precedence + */ + auth: { + username?: string; + + /** + * @visibility secret + */ + password?: string; + + /** + * Base64 Encoded API key to be used to connect to the cluster. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html + * + * @visibility secret + */ + apiKey?: string; + + /** + * Bearer authentication token to connect to the cluster. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html + * + * @visibility secret + */ + bearer?: string; + }; + }; + }; +} diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json new file mode 100644 index 0000000000..dd9ca7703c --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -0,0 +1,44 @@ +{ + "name": "@backstage/plugin-search-backend-module-elasticsearch", + "version": "0.0.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/config": "^0.1.5", + "@backstage/search-common": "^0.1.2", + "@elastic/elasticsearch": "^7.13.0", + "@acuris/aws-es-connection": "^2.2.0", + "aws-sdk": "^2.948.0", + "elastic-builder": "^2.16.0", + "lodash": "^4.17.21", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-common": "^0.8.6", + "@backstage/cli": "^0.7.4", + "@elastic/elasticsearch-mock": "^0.3.0" + }, + "files": [ + "dist", + "config.d.ts" + ], + "jest": { + "testEnvironment": "node" + }, + "configSchema": "config.d.ts" +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts new file mode 100644 index 0000000000..b4bba9de9c --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -0,0 +1,432 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { SearchEngine } from '@backstage/search-common'; +import { + ConcreteElasticSearchQuery, + ElasticSearchSearchEngine, +} from './ElasticSearchSearchEngine'; +import { Client } from '@elastic/elasticsearch'; +import Mock from '@elastic/elasticsearch-mock'; + +class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEngine { + getTranslator() { + return this.translator; + } +} + +const mock = new Mock(); +const client = new Client({ + node: 'http://localhost:9200', + Connection: mock.getConnection(), +}); + +describe('ElasticSearchSearchEngine', () => { + let testSearchEngine: SearchEngine; + let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; + + beforeEach(() => { + testSearchEngine = new ElasticSearchSearchEngine( + client, + 'search', + '', + getVoidLogger(), + ); + inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests( + client, + 'search', + '', + getVoidLogger(), + ); + }); + + describe('queryTranslator', () => { + beforeAll(() => { + mock.clearAll(); + mock.add( + { + method: 'POST', + path: '/*__search/_search', + }, + () => ({ + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }), + ); + }); + it('should invoke the query translator', async () => { + const translatorSpy = jest.fn().mockReturnValue({ + elasticSearchQuery: () => ({ + toJSON: () => + JSON.stringify({ + query: { + match_all: {}, + }, + }), + }), + documentTypes: [], + }); + testSearchEngine.setTranslator(translatorSpy); + + await testSearchEngine.query({ + term: 'testTerm', + filters: {}, + pageCursor: '', + }); + + expect(translatorSpy).toHaveBeenCalledWith({ + term: 'testTerm', + filters: {}, + pageCursor: '', + }); + }); + + it('should return translated query with 1 filter', async () => { + const translatorUnderTest = inspectableSearchEngine.getTranslator(); + + const actualTranslatedQuery = translatorUnderTest({ + types: ['indexName'], + term: 'testTerm', + filters: { kind: 'testKind' }, + pageCursor: '', + }) as ConcreteElasticSearchQuery; + + expect(actualTranslatedQuery).toMatchObject({ + documentTypes: ['indexName'], + elasticSearchQuery: expect.any(Object), + }); + + const queryBody = actualTranslatedQuery.elasticSearchQuery; + + expect(queryBody).toEqual({ + query: { + bool: { + must: { + multi_match: { + query: 'testTerm', + fields: ['*'], + fuzziness: 'auto', + minimum_should_match: 1, + }, + }, + filter: { + match: { + kind: 'testKind', + }, + }, + }, + }, + size: 100, + }); + }); + + it('should return translated query with multiple filters', async () => { + const translatorUnderTest = inspectableSearchEngine.getTranslator(); + + const actualTranslatedQuery = translatorUnderTest({ + types: ['indexName'], + term: 'testTerm', + filters: { kind: 'testKind', namespace: 'testNameSpace' }, + pageCursor: '', + }) as ConcreteElasticSearchQuery; + + expect(actualTranslatedQuery).toMatchObject({ + documentTypes: ['indexName'], + elasticSearchQuery: expect.any(Object), + }); + + const queryBody = actualTranslatedQuery.elasticSearchQuery; + + expect(queryBody).toEqual({ + query: { + bool: { + must: { + multi_match: { + query: 'testTerm', + fields: ['*'], + fuzziness: 'auto', + minimum_should_match: 1, + }, + }, + filter: [ + { + match: { + kind: 'testKind', + }, + }, + { + match: { + namespace: 'testNameSpace', + }, + }, + ], + }, + }, + size: 100, + }); + }); + + it('should return translated query with filter with multiple values', async () => { + const translatorUnderTest = inspectableSearchEngine.getTranslator(); + + const actualTranslatedQuery = translatorUnderTest({ + types: ['indexName'], + term: 'testTerm', + filters: { kind: ['testKind', 'kastTeint'] }, + pageCursor: '', + }) as ConcreteElasticSearchQuery; + + expect(actualTranslatedQuery).toMatchObject({ + documentTypes: ['indexName'], + elasticSearchQuery: expect.any(Object), + }); + + const queryBody = actualTranslatedQuery.elasticSearchQuery; + + expect(queryBody).toEqual({ + query: { + bool: { + must: { + multi_match: { + query: 'testTerm', + fields: ['*'], + fuzziness: 'auto', + minimum_should_match: 1, + }, + }, + filter: { + bool: { + should: [ + { + match: { + kind: 'testKind', + }, + }, + { + match: { + kind: 'kastTeint', + }, + }, + ], + }, + }, + }, + }, + size: 100, + }); + }); + + it('should throw if unsupported filter shapes passed in', async () => { + const translatorUnderTest = inspectableSearchEngine.getTranslator(); + const actualTranslatedQuery = () => + translatorUnderTest({ + types: ['indexName'], + term: 'testTerm', + filters: { kind: { a: 'b' } }, + pageCursor: '', + }) as ConcreteElasticSearchQuery; + expect(actualTranslatedQuery).toThrow(); + }); + }); + + describe('query functionality', () => { + beforeEach(() => { + mock.clearAll(); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/test-index__search', + }, + () => [ + { + alias: 'test-index__search', + index: 'test-index-index__1626850643538', + filter: '-', + 'routing.index': '-', + 'routing.search': '-', + is_write_index: '-', + }, + ], + ); + mock.add( + { + method: 'POST', + path: ['/_bulk'], + }, + () => ({ + took: 30, + errors: false, + items: [ + { + index: { + _index: 'test', + _type: '_doc', + _id: '1', + _version: 1, + result: 'created', + _shards: { + total: 2, + successful: 1, + failed: 0, + }, + status: 201, + _seq_no: 0, + _primary_term: 1, + }, + }, + ], + }), + ); + + mock.add( + { + method: 'POST', + path: '/*__search/_search', + }, + () => ({ + hits: { + total: { value: 0, relation: 'eq' }, + hits: [], + }, + }), + ); + }); + + // Mostly useless test since we are more or less testing the mock, runs through the whole flow though + // We might want to spin up ES test container to run against the real engine. + // That container eats GBs of memory so opting out of that for now... + it('should perform search query and return 0 results on empty index', async () => { + const mockedSearchResult = await testSearchEngine.query({ + term: 'testTerm', + filters: {}, + pageCursor: '', + }); + + // Should return 0 results as nothing is indexed here + expect(mockedSearchResult).toMatchObject({ results: [] }); + }); + + it('should handle index/search type filtering correctly', async () => { + const elasticSearchQuerySpy = jest.spyOn(client, 'search'); + await testSearchEngine.query({ + term: 'testTerm', + filters: {}, + pageCursor: '', + }); + + expect(elasticSearchQuerySpy).toHaveBeenCalled(); + expect(elasticSearchQuerySpy).toHaveBeenCalledWith({ + body: { + query: { + bool: { + must: { + multi_match: { + query: 'testTerm', + fields: ['*'], + fuzziness: 'auto', + minimum_should_match: 1, + }, + }, + filter: [], + }, + }, + size: 100, + }, + index: '*__search', + }); + + elasticSearchQuerySpy.mockClear(); + }); + + it('should create matchAll query if no term defined', async () => { + const elasticSearchQuerySpy = jest.spyOn(client, 'search'); + await testSearchEngine.query({ + term: '', + filters: {}, + pageCursor: '', + }); + + expect(elasticSearchQuerySpy).toHaveBeenCalled(); + expect(elasticSearchQuerySpy).toHaveBeenCalledWith({ + body: { + query: { + bool: { + must: { + match_all: {}, + }, + filter: [], + }, + }, + size: 100, + }, + index: '*__search', + }); + + elasticSearchQuerySpy.mockClear(); + }); + + it('should query only specified indices if defined', async () => { + const elasticSearchQuerySpy = jest.spyOn(client, 'search'); + await testSearchEngine.query({ + term: '', + filters: {}, + pageCursor: '', + types: ['test-type'], + }); + + expect(elasticSearchQuerySpy).toHaveBeenCalled(); + expect(elasticSearchQuerySpy).toHaveBeenCalledWith({ + body: { + query: { + bool: { + must: { + match_all: {}, + }, + filter: [], + }, + }, + size: 100, + }, + index: ['test-type__search'], + }); + + elasticSearchQuerySpy.mockClear(); + }); + }); + + describe('index', () => { + it('should index document', async () => { + const indexSpy = jest.spyOn(testSearchEngine, 'index'); + const mockDocuments = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + // call index func and ensure the index func was invoked. + await testSearchEngine.index('test-index', mockDocuments); + expect(indexSpy).toHaveBeenCalled(); + expect(indexSpy).toHaveBeenCalledWith('test-index', [ + { title: 'testTerm', text: 'testText', location: 'test/location' }, + ]); + }); + }); +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts new file mode 100644 index 0000000000..b068df666a --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -0,0 +1,274 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + IndexableDocument, + SearchQuery, + SearchResultSet, + SearchEngine, +} from '@backstage/search-common'; +import { Logger } from 'winston'; +import esb from 'elastic-builder'; +import { Client } from '@elastic/elasticsearch'; +import { Config } from '@backstage/config'; +import { + createAWSConnection, + awsGetCredentials, +} from '@acuris/aws-es-connection'; +import { isEmpty, isNaN as nan, isNumber } from 'lodash'; + +export type ConcreteElasticSearchQuery = { + documentTypes?: string[]; + elasticSearchQuery: Object; +}; + +type ElasticConfigAuth = { + username: string; + password: string; + apiKey: string; + bearer: string; +}; + +type ElasticSearchQueryTranslator = ( + query: SearchQuery, +) => ConcreteElasticSearchQuery; + +type ElasticSearchOptions = { + logger: Logger; + config: Config; + aliasPostfix?: string; + indexPrefix?: string; +}; + +type ElasticSearchResult = { + _index: string; + _type: string; + _score: number; + _source: IndexableDocument; +}; + +function duration(startTimestamp: [number, number]): string { + const delta = process.hrtime(startTimestamp); + const seconds = delta[0] + delta[1] / 1e9; + return `${seconds.toFixed(1)}s`; +} + +function isBlank(str: string) { + return (isEmpty(str) && !isNumber(str)) || nan(str); +} + +export class ElasticSearchSearchEngine implements SearchEngine { + constructor( + private readonly elasticSearchClient: Client, + private readonly aliasPostfix: string, + private readonly indexPrefix: string, + private readonly logger: Logger, + ) {} + + static async fromConfig({ + logger, + config, + aliasPostfix = `search`, + indexPrefix = ``, + }: ElasticSearchOptions) { + return new ElasticSearchSearchEngine( + await ElasticSearchSearchEngine.constructElasticSearchClient( + logger, + config.getConfig('search.elasticSearch'), + ), + aliasPostfix, + indexPrefix, + logger, + ); + } + + private static async constructElasticSearchClient( + logger: Logger, + config?: Config, + ) { + if (!config) { + throw new Error('No elastic search config found'); + } + + if (config.getOptionalString('provider') === 'elastic') { + logger.info('Initializing Elastic.co ElasticSearch search engine.'); + return new Client({ + cloud: { + id: config.getString('cloudId'), + }, + auth: config.get('auth'), + }); + } + if (config.getOptionalString('provider') === 'aws') { + logger.info('Initializing AWS ElasticSearch search engine.'); + const awsCredentials = await awsGetCredentials(); + const AWSConnection = createAWSConnection(awsCredentials); + return new Client({ + node: config.getString('node'), + ...AWSConnection, + }); + } + logger.info('Initializing ElasticSearch search engine.'); + return new Client({ + node: config.getString('node'), + auth: config.get('auth'), + }); + } + + protected translator({ + term, + filters = {}, + types, + }: SearchQuery): ConcreteElasticSearchQuery { + const filter = Object.entries(filters) + .filter(([_, value]) => Boolean(value)) + .map(([key, value]: [key: string, value: any]) => { + if (['string', 'number', 'boolean'].includes(typeof value)) { + return esb.matchQuery(key, value.toString()); + } + if (Array.isArray(value)) { + return esb + .boolQuery() + .should(value.map(it => esb.matchQuery(key, it.toString()))); + } + this.logger.error( + 'Failed to query, unrecognized filter type', + key, + value, + ); + throw new Error( + 'Failed to add filters to query. Unrecognized filter type', + ); + }); + const query = isBlank(term) + ? esb.matchAllQuery() + : esb + .multiMatchQuery(['*'], term) + .fuzziness('auto') + .minimumShouldMatch(1); + + return { + elasticSearchQuery: esb + .requestBodySearch() + .query(esb.boolQuery().filter(filter).must([query])) + // TODO: Replace size limit with page cursor after pagination approach decided + // See: https://github.com/backstage/backstage/issues/6062 + .size(100) + .toJSON(), + documentTypes: types, + }; + } + + setTranslator(translator: ElasticSearchQueryTranslator) { + this.translator = translator; + } + + async index(type: string, documents: IndexableDocument[]): Promise { + this.logger.info( + `Started indexing ${documents.length} documents for index ${type}`, + ); + const startTimestamp = process.hrtime(); + const alias = this.constructSearchAlias(type); + const index = this.constructIndexName(type, `${Date.now()}`); + try { + const aliases = await this.elasticSearchClient.cat.aliases({ + format: 'json', + name: alias, + }); + const removableIndices = aliases.body.map( + (r: Record) => r.index, + ); + + await this.elasticSearchClient.indices.create({ + index, + }); + const result = await this.elasticSearchClient.helpers.bulk({ + datasource: documents, + onDocument() { + return { + index: { _index: index }, + }; + }, + refreshOnCompletion: index, + }); + + this.logger.info( + `Indexing completed for index ${type} in ${duration(startTimestamp)}`, + result, + ); + await this.elasticSearchClient.indices.updateAliases({ + body: { + actions: [ + { remove: { index: this.constructIndexName(type, '*'), alias } }, + { add: { index, alias } }, + ], + }, + }); + + this.logger.info('Removing stale search indices', removableIndices); + if (removableIndices.length) { + await this.elasticSearchClient.indices.delete({ + index: removableIndices, + }); + } + } catch (e) { + this.logger.error(`Failed to index documents for type ${type}`, e); + const response = await this.elasticSearchClient.indices.exists({ + index, + }); + const indexCreated = response.body; + if (indexCreated) { + this.logger.info(`Removing created index ${index}`); + await this.elasticSearchClient.indices.delete({ + index, + }); + } + } + } + + async query(query: SearchQuery): Promise { + const { elasticSearchQuery, documentTypes } = this.translator(query); + const queryIndices = documentTypes + ? documentTypes.map(it => this.constructSearchAlias(it)) + : this.constructSearchAlias('*'); + try { + const result = await this.elasticSearchClient.search({ + index: queryIndices, + body: elasticSearchQuery, + }); + return { + results: result.body.hits.hits.map((d: ElasticSearchResult) => ({ + type: d._index.split('__')[0], + document: d._source, + })), + }; + } catch (e) { + this.logger.error( + `Failed to query documents for indices ${queryIndices}`, + e, + ); + return Promise.reject({ results: [] }); + } + } + + private constructIndexName(type: string, postFix: string) { + return `${this.indexPrefix}${type}-index__${postFix}`; + } + + private constructSearchAlias(type: string) { + return `${this.indexPrefix}${type}__${this.aliasPostfix}`; + } +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts new file mode 100644 index 0000000000..2f3641c114 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine'; +export type { ConcreteElasticSearchQuery } from './ElasticSearchSearchEngine'; diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts new file mode 100644 index 0000000000..4cfeec6598 --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ElasticSearchSearchEngine } from './engines'; diff --git a/plugins/search-backend-node/api-report.md b/plugins/search-backend-node/api-report.md index 4418c4c590..beb1234e6a 100644 --- a/plugins/search-backend-node/api-report.md +++ b/plugins/search-backend-node/api-report.md @@ -8,6 +8,8 @@ import { DocumentDecorator } from '@backstage/search-common'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; import { default as lunr_2 } from 'lunr'; +import { QueryTranslator } from '@backstage/search-common'; +import { SearchEngine } from '@backstage/search-common'; import { SearchQuery } from '@backstage/search-common'; import { SearchResultSet } from '@backstage/search-common'; @@ -50,8 +52,6 @@ export class LunrSearchEngine implements SearchEngine { // // (undocumented) setTranslator(translator: LunrQueryTranslator): void; - // Warning: (ae-forgotten-export) The symbol "QueryTranslator" needs to be exported by the entry point index.d.ts - // // (undocumented) protected translator: QueryTranslator; } @@ -66,14 +66,7 @@ export class Scheduler { stop(): void; } -// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export interface SearchEngine { - index(type: string, documents: IndexableDocument[]): Promise; - query(query: SearchQuery): Promise; - setTranslator(translator: QueryTranslator): void; -} +export { SearchEngine }; // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 9bbbd73067..5268caf58b 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -18,13 +18,13 @@ import { DocumentCollator, DocumentDecorator, IndexableDocument, + SearchEngine, } from '@backstage/search-common'; import { Logger } from 'winston'; import { Scheduler } from './index'; import { RegisterCollatorParameters, RegisterDecoratorParameters, - SearchEngine, } from './types'; interface CollatorEnvelope { diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index 3d59ba4b6f..3233c3adae 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -16,7 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import lunr from 'lunr'; -import { SearchEngine } from '../types'; +import { SearchEngine } from '@backstage/search-common'; import { ConcreteLunrQuery, LunrSearchEngine } from './LunrSearchEngine'; /** diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index e832e49f85..0cb4fd8336 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -18,10 +18,11 @@ import { IndexableDocument, SearchQuery, SearchResultSet, + QueryTranslator, + SearchEngine, } from '@backstage/search-common'; import lunr from 'lunr'; import { Logger } from 'winston'; -import { QueryTranslator, SearchEngine } from '../types'; export type ConcreteLunrQuery = { lunrQueryBuilder: lunr.Index.QueryBuilder; diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index b52a1ef20c..a0a1109211 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -17,4 +17,8 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export { LunrSearchEngine } from './engines'; -export type { SearchEngine } from './types'; + +/** + * @deprecated Import from @backstage/search-common instead + */ +export type { SearchEngine } from '@backstage/search-common'; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index 28dc1237ba..df83357f6c 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -14,13 +14,7 @@ * limitations under the License. */ -import { - DocumentCollator, - DocumentDecorator, - IndexableDocument, - SearchQuery, - SearchResultSet, -} from '@backstage/search-common'; +import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; /** * Parameters required to register a collator. @@ -46,31 +40,3 @@ export interface RegisterDecoratorParameters { */ decorator: DocumentDecorator; } - -/** - * A type of function responsible for translating an abstract search query into - * a concrete query relevant to a particular search engine. - */ -export type QueryTranslator = (query: SearchQuery) => unknown; - -/** - * Interface that must be implemented by specific search engines, responsible - * for performing indexing and querying and translating abstract queries into - * concrete, search engine-specific queries. - */ -export interface SearchEngine { - /** - * Override the default translator provided by the SearchEngine. - */ - setTranslator(translator: QueryTranslator): void; - - /** - * Add the given documents to the SearchEngine index of the given type. - */ - index(type: string, documents: IndexableDocument[]): Promise; - - /** - * Perform a search query against the SearchEngine. - */ - query(query: SearchQuery): Promise; -} diff --git a/yarn.lock b/yarn.lock index 1be3dedc16..5739be4711 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,13 @@ # yarn lockfile v1 +"@acuris/aws-es-connection@^2.2.0": + version "2.2.0" + resolved "https://registry.npmjs.org/@acuris/aws-es-connection/-/aws-es-connection-2.2.0.tgz#43f7d6f3d15de0231642647a45eb84a8108e7d3e" + integrity sha512-xstECUJiWhj3kUK3aBpidoeHojXV611dcUewBwMG0hDRrRkIS+aj3xOhlfXcqYql7ITG84jgTfAidFqr8F+dnQ== + dependencies: + aws4 "^1.8.0" + "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.0.6" resolved "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz#5d9000a3ac1fd25404da886da6b266adcd99cf1c" @@ -1811,6 +1818,25 @@ dependencies: "@date-io/core" "^2.10.11" +"@elastic/elasticsearch-mock@^0.3.0": + version "0.3.0" + resolved "https://registry.npmjs.org/@elastic/elasticsearch-mock/-/elasticsearch-mock-0.3.0.tgz#6b1d8448aad3ca20f760fa01c0206b733c9c1e54" + integrity sha512-hZYRjPgRE1M0wCqdsgaDtwxrgQEXDZya1gQ3gnpc8pB8mHUfPoO+9ye7GbDPUkWbuGGGZ4/p6OKmAbt/ME+CDQ== + dependencies: + fast-deep-equal "^3.1.1" + find-my-way "^2.2.2" + into-stream "^5.1.1" + +"@elastic/elasticsearch@^7.13.0": + version "7.13.0" + resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.13.0.tgz#6dcf511dfa91187e22c81e54f41f4bd0fd96b4d6" + integrity sha512-WgwLWo2p9P2tdqzBGX9fHeG8p5IOTXprXNTECQG2mJ7z9n93N5AFBJpEw4d35tWWeCWi9jI13A2wzQZH7XZ/xw== + dependencies: + debug "^4.3.1" + hpagent "^0.1.1" + ms "^2.1.3" + secure-json-parse "^2.4.0" + "@emotion/cache@^10.0.27": version "10.0.29" resolved "https://registry.npmjs.org/@emotion/cache/-/cache-10.0.29.tgz#87e7e64f412c060102d589fe7c6dc042e6f9d1e0" @@ -8129,6 +8155,21 @@ aws-sdk@^2.928.0: uuid "3.3.2" xml2js "0.4.19" +aws-sdk@^2.948.0: + version "2.948.0" + resolved "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.948.0.tgz#0c974c351af97dbc66dbd96bd6c20928baf10415" + integrity sha512-UJaCwccNaNNFtbhlvg+BmcaVWNI7RPonZA16nca0s3O+UnHm5y5H/nN6XpuJp+NUrxrLgTFaztPvjmBp5q6p+g== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -11912,6 +11953,21 @@ ejs@^3.1.2: dependencies: jake "^10.6.1" +elastic-builder@^2.16.0: + version "2.16.0" + resolved "https://registry.npmjs.org/elastic-builder/-/elastic-builder-2.16.0.tgz#684757ab9e6a4214653d23d84cec5ab8d185892f" + integrity sha512-5EXFxTAOPQFW7uYe59lZ5pqHBoyILQ8U3x1GgZN921EfAsLNdA2kMV0bgK8/rwJOd9JM0F40WpGxCPzHRtCG1Q== + dependencies: + babel-runtime "^6.26.0" + lodash.has "^4.5.2" + lodash.hasin "^4.5.2" + lodash.head "^4.0.1" + lodash.isempty "^4.4.0" + lodash.isnil "^4.0.0" + lodash.isobject "^3.0.2" + lodash.isstring "^4.0.1" + lodash.omit "^4.5.0" + electron-to-chromium@^1.3.378, electron-to-chromium@^1.3.564, electron-to-chromium@^1.3.723: version "1.3.739" resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.739.tgz#f07756aa92cabd5a6eec6f491525a64fe62f98b9" @@ -12932,6 +12988,11 @@ extsprintf@^1.2.0: resolved "https://registry.npmjs.org/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= +fast-decode-uri-component@^1.0.0: + version "1.0.1" + resolved "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz#46f8b6c22b30ff7a81357d4f59abfae938202543" + integrity sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg== + fast-deep-equal@2.0.1, fast-deep-equal@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" @@ -13221,6 +13282,15 @@ find-cache-dir@^3.3.1: make-dir "^3.0.2" pkg-dir "^4.1.0" +find-my-way@^2.2.2: + version "2.2.5" + resolved "https://registry.npmjs.org/find-my-way/-/find-my-way-2.2.5.tgz#86ce825266fa28cd962e538a45ec2aaa84c3d514" + integrity sha512-GjRZZlGcGmTh9t+6Xrj5K0YprpoAFCAiCPgmAH9Kb09O4oX6hYuckDfnDipYj+Q7B1GtYWSzDI5HEecNYscLQg== + dependencies: + fast-decode-uri-component "^1.0.0" + safe-regex2 "^2.0.0" + semver-store "^0.3.0" + find-root@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz#abcfc8ba76f708c42a97b3d685b7e9450bfb9ce4" @@ -13449,7 +13519,7 @@ fresh@0.5.2: resolved "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= -from2@^2.1.0: +from2@^2.1.0, from2@^2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" integrity sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8= @@ -14659,6 +14729,11 @@ hpack.js@^2.1.6: readable-stream "^2.0.1" wbuf "^1.1.0" +hpagent@^0.1.1: + version "0.1.2" + resolved "https://registry.npmjs.org/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9" + integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== + hsl-regex@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/hsl-regex/-/hsl-regex-1.0.0.tgz#d49330c789ed819e276a4c0d272dffa30b18fe6e" @@ -15301,6 +15376,14 @@ interpret@^2.0.0, interpret@^2.2.0: resolved "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9" integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw== +into-stream@^5.1.1: + version "5.1.1" + resolved "https://registry.npmjs.org/into-stream/-/into-stream-5.1.1.tgz#f9a20a348a11f3c13face22763f2d02e127f4db8" + integrity sha512-krrAJ7McQxGGmvaYbB7Q1mcA+cRwg9Ij2RfWIeVesNBgVDZmzY/Fa4IpZUT3bmdRzMzdf/mzltCG2Dq99IZGBA== + dependencies: + from2 "^2.3.0" + p-is-promise "^3.0.0" + invariant@^2.0.0, invariant@^2.2.2, invariant@^2.2.3, invariant@^2.2.4: version "2.2.4" resolved "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" @@ -17667,6 +17750,21 @@ lodash.get@^4, lodash.get@^4.0.0, lodash.get@^4.4.2: resolved "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk= +lodash.has@^4.5.2: + version "4.5.2" + resolved "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862" + integrity sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI= + +lodash.hasin@^4.5.2: + version "4.5.2" + resolved "https://registry.npmjs.org/lodash.hasin/-/lodash.hasin-4.5.2.tgz#f91e352378d21ef7090b9e7687c2ca35c5b4d52a" + integrity sha1-+R41I3jSHvcJC552h8LKNcW01So= + +lodash.head@^4.0.1: + version "4.0.1" + resolved "https://registry.npmjs.org/lodash.head/-/lodash.head-4.0.1.tgz#e2aa322d3ec40cd6aae186082977d993b354ed9c" + integrity sha1-4qoyLT7EDNaq4YYIKXfZk7NU7Zw= + lodash.includes@^4.3.0: version "4.3.0" resolved "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" @@ -17677,6 +17775,11 @@ lodash.isboolean@^3.0.3: resolved "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" integrity sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY= +lodash.isempty@^4.4.0: + version "4.4.0" + resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" + integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4= + lodash.isequal@^4.0.0: version "4.5.0" resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" @@ -17692,11 +17795,21 @@ lodash.ismatch@^4.4.0: resolved "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz#756cb5150ca3ba6f11085a78849645f188f85f37" integrity sha1-dWy1FQyjum8RCFp4hJZF8Yj4Xzc= +lodash.isnil@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz#49e28cd559013458c814c5479d3c663a21bfaa6c" + integrity sha1-SeKM1VkBNFjIFMVHnTxmOiG/qmw= + lodash.isnumber@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc" integrity sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w= +lodash.isobject@^3.0.2: + version "3.0.2" + resolved "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-3.0.2.tgz#3c8fb8d5b5bf4bf90ae06e14f2a530a4ed935e1d" + integrity sha1-PI+41bW/S/kK4G4U8qUwpO2TXh0= + lodash.isplainobject@^4.0.6: version "4.0.6" resolved "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" @@ -17717,6 +17830,11 @@ lodash.merge@^4.6.2: resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== +lodash.omit@^4.5.0: + version "4.5.0" + resolved "https://registry.npmjs.org/lodash.omit/-/lodash.omit-4.5.0.tgz#6eb19ae5a1ee1dd9df0b969e66ce0b7fa30b5e60" + integrity sha1-brGa5aHuHdnfC5aeZs4Lf6MLXmA= + lodash.once@^4.0.0, lodash.once@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" @@ -18807,6 +18925,11 @@ ms@2.1.2, ms@^2.0.0, ms@^2.1.1: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + msal@^1.0.2: version "1.4.4" resolved "https://registry.npmjs.org/msal/-/msal-1.4.4.tgz#3f9b5a4442aa711c12ab8e88b8ed89b293f99711" @@ -19821,6 +19944,11 @@ p-finally@^2.0.0: resolved "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz#bd6fcaa9c559a096b680806f4d657b3f0f240561" integrity sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw== +p-is-promise@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971" + integrity sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ== + p-limit@3.1.0, p-limit@^3.0.1, p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" @@ -22895,6 +23023,11 @@ ret@~0.1.10: resolved "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== +ret@~0.2.0: + version "0.2.2" + resolved "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz#b6861782a1f4762dce43402a71eb7a283f44573c" + integrity sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ== + retry-request@^4.0.0: version "4.1.3" resolved "https://registry.npmjs.org/retry-request/-/retry-request-4.1.3.tgz#d5f74daf261372cff58d08b0a1979b4d7cab0fde" @@ -23110,6 +23243,13 @@ safe-json-stringify@~1: resolved "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz#356e44bc98f1f93ce45df14bcd7c01cda86e0afd" integrity sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg== +safe-regex2@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/safe-regex2/-/safe-regex2-2.0.0.tgz#b287524c397c7a2994470367e0185e1916b1f5b9" + integrity sha512-PaUSFsUaNNuKwkBijoAPHAK6/eM6VirvyPWlZ7BAQy4D+hCvh4B6lIG+nPdhbFfIbP+gTGBcrdsOaUs0F+ZBOQ== + dependencies: + ret "~0.2.0" + safe-regex@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e" @@ -23208,6 +23348,11 @@ scuid@^1.1.0: resolved "https://registry.npmjs.org/scuid/-/scuid-1.1.0.tgz#d3f9f920956e737a60f72d0e4ad280bf324d5dab" integrity sha512-MuCAyrGZcTLfQoH2XoBlQ8C6bzwN88XT/0slOGz0pn8+gIP85BOAfYa44ZXQUTOwRwPU0QvgU+V+OSajl/59Xg== +secure-json-parse@^2.4.0: + version "2.4.0" + resolved "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" + integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -23237,6 +23382,11 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" +semver-store@^0.3.0: + version "0.3.0" + resolved "https://registry.npmjs.org/semver-store/-/semver-store-0.3.0.tgz#ce602ff07df37080ec9f4fb40b29576547befbe9" + integrity sha512-TcZvGMMy9vodEFSse30lWinkj+JgOBvPn8wRItpQRSayhc+4ssDs335uklkfvQQJgL/WvmHLVj4Ycv2s7QCQMg== + "semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.4.1, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: version "5.7.1" resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"