Merge pull request #6569 from RoadieHQ/gh4447-elasticseach-searchengine

Implement ElasticSearch search engine

plugins/search-backend-module-elasticsearch/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/search-backend-module-elasticsearch/README.md

@@ -0,0 +1,9 @@
+# search-backend-module-elasticsearch
+
+This is an extension to module to search-backend-node plugin, which provides basic types, interfaces and functionality to implement search process within Backstage.
+
+This module provides functionality to index and implement querying using ElasticSearch engine. The module exposes configuration options to connect Backstage to your ElasticSearch cluster to be used as search engine.
+
+## Getting started
+
+See [Backstage documentation](https://backstage.io/docs/features/search/getting-started) for details on how to install and configure ElasticSearch for your Backstage instance.

plugins/search-backend-module-elasticsearch/api-report.md

@@ -0,0 +1,52 @@
+## API Report File for "@backstage/plugin-search-backend-module-elasticsearch"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { Client } from '@elastic/elasticsearch';
+import { Config } from '@backstage/config';
+import { IndexableDocument } from '@backstage/search-common';
+import { Logger as Logger_2 } from 'winston';
+import { SearchEngine } from '@backstage/search-common';
+import { SearchQuery } from '@backstage/search-common';
+import { SearchResultSet } from '@backstage/search-common';
+
+// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class ElasticSearchSearchEngine implements SearchEngine {
+  constructor(
+    elasticSearchClient: Client,
+    aliasPostfix: string,
+    indexPrefix: string,
+    logger: Logger_2,
+  );
+  // Warning: (ae-forgotten-export) The symbol "ElasticSearchOptions" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  static fromConfig({
+    logger,
+    config,
+    aliasPostfix,
+    indexPrefix,
+  }: ElasticSearchOptions): Promise<ElasticSearchSearchEngine>;
+  // (undocumented)
+  index(type: string, documents: IndexableDocument[]): Promise<void>;
+  // (undocumented)
+  query(query: SearchQuery): Promise<SearchResultSet>;
+  // Warning: (ae-forgotten-export) The symbol "ElasticSearchQueryTranslator" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  setTranslator(translator: ElasticSearchQueryTranslator): void;
+  // Warning: (ae-forgotten-export) The symbol "ConcreteElasticSearchQuery" needs to be exported by the entry point index.d.ts
+  //
+  // (undocumented)
+  protected translator({
+    term,
+    filters,
+    types,
+  }: SearchQuery): ConcreteElasticSearchQuery;
+}
+
+// (No @packageDocumentation comment for this package)
+```

plugins/search-backend-module-elasticsearch/config.d.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface Config {
+  /** Configuration options for the search plugin */
+  search?: {
+    /**
+     * Options for ElasticSearch
+     */
+    elasticsearch?:
+      | // elastic = Elastic.co ElasticSearch provider
+      {
+          provider: 'elastic';
+
+          /**
+           * Elastic.co CloudID
+           * See: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-connecting.html#authentication
+           */
+          cloudId: string;
+
+          auth: {
+            username: string;
+
+            /**
+             * @visibility secret
+             */
+            password: string;
+          };
+        }
+
+      /**
+       *  AWS = Amazon Elasticsearch Service provider
+       *
+       *  Authentication is handled using the default AWS credentials provider chain
+       */
+      | {
+          provider: 'aws';
+
+          /**
+           * Node configuration.
+           * URL AWS ES endpoint to connect to.
+           * Eg. https://my-es-cluster.eu-west-1.es.amazonaws.com
+           */
+          node: string;
+        }
+
+      /**
+       * Standard ElasticSearch
+       *
+       * Includes self-hosted clusters and others that provide direct connection via an endpoint
+       * and authentication method (see possible authentication options below)
+       */
+      | {
+          /**
+           * Node configuration.
+           * URL/URLS to ElasticSearch node to connect to.
+           * Either direct URL like 'https://localhost:9200' or with credentials like 'https://username:password@localhost:9200'
+           */
+          node: string | string[];
+
+          /**
+           * Authentication credentials for ElasticSearch
+           * If both ApiKey/Bearer token and username+password is provided, tokens take precedence
+           */
+          auth?: {
+            username?: string;
+
+            /**
+             * @visibility secret
+             */
+            password?: string;
+
+            /**
+             * Base64 Encoded API key to be used to connect to the cluster.
+             * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html
+             *
+             * @visibility secret
+             */
+            apiKey?: string;
+
+            /**
+             * Bearer authentication token to connect to the cluster.
+             * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-service-token.html
+             *
+             * @visibility secret
+             */
+            bearer?: string;
+          };
+        };
+  };
+}

plugins/search-backend-module-elasticsearch/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@backstage/plugin-search-backend-module-elasticsearch",
+  "version": "0.0.1",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "scripts": {
+    "start": "backstage-cli backend:dev",
+    "build": "backstage-cli backend:build",
+    "lint": "backstage-cli lint",
+    "test": "backstage-cli test",
+    "prepack": "backstage-cli prepack",
+    "postpack": "backstage-cli postpack",
+    "clean": "backstage-cli clean"
+  },
+  "dependencies": {
+    "@backstage/config": "^0.1.5",
+    "@backstage/search-common": "^0.1.2",
+    "@elastic/elasticsearch": "^7.13.0",
+    "@acuris/aws-es-connection": "^2.2.0",
+    "aws-sdk": "^2.948.0",
+    "elastic-builder": "^2.16.0",
+    "lodash": "^4.17.21",
+    "winston": "^3.2.1"
+  },
+  "devDependencies": {
+    "@backstage/backend-common": "^0.8.6",
+    "@backstage/cli": "^0.7.4",
+    "@elastic/elasticsearch-mock": "^0.3.0"
+  },
+  "files": [
+    "dist",
+    "config.d.ts"
+  ],
+  "jest": {
+    "testEnvironment": "node"
+  },
+  "configSchema": "config.d.ts"
+}

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts

@@ -0,0 +1,432 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { SearchEngine } from '@backstage/search-common';
+import {
+  ConcreteElasticSearchQuery,
+  ElasticSearchSearchEngine,
+} from './ElasticSearchSearchEngine';
+import { Client } from '@elastic/elasticsearch';
+import Mock from '@elastic/elasticsearch-mock';
+
+class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEngine {
+  getTranslator() {
+    return this.translator;
+  }
+}
+
+const mock = new Mock();
+const client = new Client({
+  node: 'http://localhost:9200',
+  Connection: mock.getConnection(),
+});
+
+describe('ElasticSearchSearchEngine', () => {
+  let testSearchEngine: SearchEngine;
+  let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests;
+
+  beforeEach(() => {
+    testSearchEngine = new ElasticSearchSearchEngine(
+      client,
+      'search',
+      '',
+      getVoidLogger(),
+    );
+    inspectableSearchEngine = new ElasticSearchSearchEngineForTranslatorTests(
+      client,
+      'search',
+      '',
+      getVoidLogger(),
+    );
+  });
+
+  describe('queryTranslator', () => {
+    beforeAll(() => {
+      mock.clearAll();
+      mock.add(
+        {
+          method: 'POST',
+          path: '/*__search/_search',
+        },
+        () => ({
+          hits: {
+            total: { value: 0, relation: 'eq' },
+            hits: [],
+          },
+        }),
+      );
+    });
+    it('should invoke the query translator', async () => {
+      const translatorSpy = jest.fn().mockReturnValue({
+        elasticSearchQuery: () => ({
+          toJSON: () =>
+            JSON.stringify({
+              query: {
+                match_all: {},
+              },
+            }),
+        }),
+        documentTypes: [],
+      });
+      testSearchEngine.setTranslator(translatorSpy);
+
+      await testSearchEngine.query({
+        term: 'testTerm',
+        filters: {},
+        pageCursor: '',
+      });
+
+      expect(translatorSpy).toHaveBeenCalledWith({
+        term: 'testTerm',
+        filters: {},
+        pageCursor: '',
+      });
+    });
+
+    it('should return translated query with 1 filter', async () => {
+      const translatorUnderTest = inspectableSearchEngine.getTranslator();
+
+      const actualTranslatedQuery = translatorUnderTest({
+        types: ['indexName'],
+        term: 'testTerm',
+        filters: { kind: 'testKind' },
+        pageCursor: '',
+      }) as ConcreteElasticSearchQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        documentTypes: ['indexName'],
+        elasticSearchQuery: expect.any(Object),
+      });
+
+      const queryBody = actualTranslatedQuery.elasticSearchQuery;
+
+      expect(queryBody).toEqual({
+        query: {
+          bool: {
+            must: {
+              multi_match: {
+                query: 'testTerm',
+                fields: ['*'],
+                fuzziness: 'auto',
+                minimum_should_match: 1,
+              },
+            },
+            filter: {
+              match: {
+                kind: 'testKind',
+              },
+            },
+          },
+        },
+        size: 100,
+      });
+    });
+
+    it('should return translated query with multiple filters', async () => {
+      const translatorUnderTest = inspectableSearchEngine.getTranslator();
+
+      const actualTranslatedQuery = translatorUnderTest({
+        types: ['indexName'],
+        term: 'testTerm',
+        filters: { kind: 'testKind', namespace: 'testNameSpace' },
+        pageCursor: '',
+      }) as ConcreteElasticSearchQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        documentTypes: ['indexName'],
+        elasticSearchQuery: expect.any(Object),
+      });
+
+      const queryBody = actualTranslatedQuery.elasticSearchQuery;
+
+      expect(queryBody).toEqual({
+        query: {
+          bool: {
+            must: {
+              multi_match: {
+                query: 'testTerm',
+                fields: ['*'],
+                fuzziness: 'auto',
+                minimum_should_match: 1,
+              },
+            },
+            filter: [
+              {
+                match: {
+                  kind: 'testKind',
+                },
+              },
+              {
+                match: {
+                  namespace: 'testNameSpace',
+                },
+              },
+            ],
+          },
+        },
+        size: 100,
+      });
+    });
+
+    it('should return translated query with filter with multiple values', async () => {
+      const translatorUnderTest = inspectableSearchEngine.getTranslator();
+
+      const actualTranslatedQuery = translatorUnderTest({
+        types: ['indexName'],
+        term: 'testTerm',
+        filters: { kind: ['testKind', 'kastTeint'] },
+        pageCursor: '',
+      }) as ConcreteElasticSearchQuery;
+
+      expect(actualTranslatedQuery).toMatchObject({
+        documentTypes: ['indexName'],
+        elasticSearchQuery: expect.any(Object),
+      });
+
+      const queryBody = actualTranslatedQuery.elasticSearchQuery;
+
+      expect(queryBody).toEqual({
+        query: {
+          bool: {
+            must: {
+              multi_match: {
+                query: 'testTerm',
+                fields: ['*'],
+                fuzziness: 'auto',
+                minimum_should_match: 1,
+              },
+            },
+            filter: {
+              bool: {
+                should: [
+                  {
+                    match: {
+                      kind: 'testKind',
+                    },
+                  },
+                  {
+                    match: {
+                      kind: 'kastTeint',
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        },
+        size: 100,
+      });
+    });
+
+    it('should throw if unsupported filter shapes passed in', async () => {
+      const translatorUnderTest = inspectableSearchEngine.getTranslator();
+      const actualTranslatedQuery = () =>
+        translatorUnderTest({
+          types: ['indexName'],
+          term: 'testTerm',
+          filters: { kind: { a: 'b' } },
+          pageCursor: '',
+        }) as ConcreteElasticSearchQuery;
+      expect(actualTranslatedQuery).toThrow();
+    });
+  });
+
+  describe('query functionality', () => {
+    beforeEach(() => {
+      mock.clearAll();
+      mock.add(
+        {
+          method: 'GET',
+          path: '/_cat/aliases/test-index__search',
+        },
+        () => [
+          {
+            alias: 'test-index__search',
+            index: 'test-index-index__1626850643538',
+            filter: '-',
+            'routing.index': '-',
+            'routing.search': '-',
+            is_write_index: '-',
+          },
+        ],
+      );
+      mock.add(
+        {
+          method: 'POST',
+          path: ['/_bulk'],
+        },
+        () => ({
+          took: 30,
+          errors: false,
+          items: [
+            {
+              index: {
+                _index: 'test',
+                _type: '_doc',
+                _id: '1',
+                _version: 1,
+                result: 'created',
+                _shards: {
+                  total: 2,
+                  successful: 1,
+                  failed: 0,
+                },
+                status: 201,
+                _seq_no: 0,
+                _primary_term: 1,
+              },
+            },
+          ],
+        }),
+      );
+
+      mock.add(
+        {
+          method: 'POST',
+          path: '/*__search/_search',
+        },
+        () => ({
+          hits: {
+            total: { value: 0, relation: 'eq' },
+            hits: [],
+          },
+        }),
+      );
+    });
+
+    // Mostly useless test since we are more or less testing the mock, runs through the whole flow though
+    // We might want to spin up ES test container to run against the real engine.
+    // That container eats GBs of memory so opting out of that for now...
+    it('should perform search query and return 0 results on empty index', async () => {
+      const mockedSearchResult = await testSearchEngine.query({
+        term: 'testTerm',
+        filters: {},
+        pageCursor: '',
+      });
+
+      // Should return 0 results as nothing is indexed here
+      expect(mockedSearchResult).toMatchObject({ results: [] });
+    });
+
+    it('should handle index/search type filtering correctly', async () => {
+      const elasticSearchQuerySpy = jest.spyOn(client, 'search');
+      await testSearchEngine.query({
+        term: 'testTerm',
+        filters: {},
+        pageCursor: '',
+      });
+
+      expect(elasticSearchQuerySpy).toHaveBeenCalled();
+      expect(elasticSearchQuerySpy).toHaveBeenCalledWith({
+        body: {
+          query: {
+            bool: {
+              must: {
+                multi_match: {
+                  query: 'testTerm',
+                  fields: ['*'],
+                  fuzziness: 'auto',
+                  minimum_should_match: 1,
+                },
+              },
+              filter: [],
+            },
+          },
+          size: 100,
+        },
+        index: '*__search',
+      });
+
+      elasticSearchQuerySpy.mockClear();
+    });
+
+    it('should create matchAll query if no term defined', async () => {
+      const elasticSearchQuerySpy = jest.spyOn(client, 'search');
+      await testSearchEngine.query({
+        term: '',
+        filters: {},
+        pageCursor: '',
+      });
+
+      expect(elasticSearchQuerySpy).toHaveBeenCalled();
+      expect(elasticSearchQuerySpy).toHaveBeenCalledWith({
+        body: {
+          query: {
+            bool: {
+              must: {
+                match_all: {},
+              },
+              filter: [],
+            },
+          },
+          size: 100,
+        },
+        index: '*__search',
+      });
+
+      elasticSearchQuerySpy.mockClear();
+    });
+
+    it('should query only specified indices if defined', async () => {
+      const elasticSearchQuerySpy = jest.spyOn(client, 'search');
+      await testSearchEngine.query({
+        term: '',
+        filters: {},
+        pageCursor: '',
+        types: ['test-type'],
+      });
+
+      expect(elasticSearchQuerySpy).toHaveBeenCalled();
+      expect(elasticSearchQuerySpy).toHaveBeenCalledWith({
+        body: {
+          query: {
+            bool: {
+              must: {
+                match_all: {},
+              },
+              filter: [],
+            },
+          },
+          size: 100,
+        },
+        index: ['test-type__search'],
+      });
+
+      elasticSearchQuerySpy.mockClear();
+    });
+  });
+
+  describe('index', () => {
+    it('should index document', async () => {
+      const indexSpy = jest.spyOn(testSearchEngine, 'index');
+      const mockDocuments = [
+        {
+          title: 'testTerm',
+          text: 'testText',
+          location: 'test/location',
+        },
+      ];
+
+      // call index func and ensure the index func was invoked.
+      await testSearchEngine.index('test-index', mockDocuments);
+      expect(indexSpy).toHaveBeenCalled();
+      expect(indexSpy).toHaveBeenCalledWith('test-index', [
+        { title: 'testTerm', text: 'testText', location: 'test/location' },
+      ]);
+    });
+  });
+});

plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts

@@ -0,0 +1,283 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  IndexableDocument,
+  SearchQuery,
+  SearchResultSet,
+  SearchEngine,
+} from '@backstage/search-common';
+import { Logger } from 'winston';
+import esb from 'elastic-builder';
+import { Client } from '@elastic/elasticsearch';
+import { Config } from '@backstage/config';
+import {
+  createAWSConnection,
+  awsGetCredentials,
+} from '@acuris/aws-es-connection';
+import { isEmpty, isNaN as nan, isNumber } from 'lodash';
+
+export type ConcreteElasticSearchQuery = {
+  documentTypes?: string[];
+  elasticSearchQuery: Object;
+};
+
+type ElasticConfigAuth = {
+  username: string;
+  password: string;
+  apiKey: string;
+  bearer: string;
+};
+
+type ElasticSearchQueryTranslator = (
+  query: SearchQuery,
+) => ConcreteElasticSearchQuery;
+
+type ElasticSearchOptions = {
+  logger: Logger;
+  config: Config;
+  aliasPostfix?: string;
+  indexPrefix?: string;
+};
+
+type ElasticSearchResult = {
+  _index: string;
+  _type: string;
+  _score: number;
+  _source: IndexableDocument;
+};
+
+function duration(startTimestamp: [number, number]): string {
+  const delta = process.hrtime(startTimestamp);
+  const seconds = delta[0] + delta[1] / 1e9;
+  return `${seconds.toFixed(1)}s`;
+}
+
+function isBlank(str: string) {
+  return (isEmpty(str) && !isNumber(str)) || nan(str);
+}
+
+export class ElasticSearchSearchEngine implements SearchEngine {
+  constructor(
+    private readonly elasticSearchClient: Client,
+    private readonly aliasPostfix: string,
+    private readonly indexPrefix: string,
+    private readonly logger: Logger,
+  ) {}
+
+  static async fromConfig({
+    logger,
+    config,
+    aliasPostfix = `search`,
+    indexPrefix = ``,
+  }: ElasticSearchOptions) {
+    return new ElasticSearchSearchEngine(
+      await ElasticSearchSearchEngine.constructElasticSearchClient(
+        logger,
+        config.getConfig('search.elasticsearch'),
+      ),
+      aliasPostfix,
+      indexPrefix,
+      logger,
+    );
+  }
+
+  private static async constructElasticSearchClient(
+    logger: Logger,
+    config?: Config,
+  ) {
+    if (!config) {
+      throw new Error('No elastic search config found');
+    }
+
+    if (config.getOptionalString('provider') === 'elastic') {
+      logger.info('Initializing Elastic.co ElasticSearch search engine.');
+      return new Client({
+        cloud: {
+          id: config.getString('cloudId'),
+        },
+        auth: config.get<ElasticConfigAuth>('auth'),
+      });
+    }
+    if (config.getOptionalString('provider') === 'aws') {
+      logger.info('Initializing AWS ElasticSearch search engine.');
+      const awsCredentials = await awsGetCredentials();
+      const AWSConnection = createAWSConnection(awsCredentials);
+      return new Client({
+        node: config.getString('node'),
+        ...AWSConnection,
+      });
+    }
+    logger.info('Initializing ElasticSearch search engine.');
+    return new Client({
+      node: config.getString('node'),
+      auth: config.get<ElasticConfigAuth>('auth'),
+    });
+  }
+
+  protected translator({
+    term,
+    filters = {},
+    types,
+  }: SearchQuery): ConcreteElasticSearchQuery {
+    const filter = Object.entries(filters)
+      .filter(([_, value]) => Boolean(value))
+      .map(([key, value]: [key: string, value: any]) => {
+        if (['string', 'number', 'boolean'].includes(typeof value)) {
+          return esb.matchQuery(key, value.toString());
+        }
+        if (Array.isArray(value)) {
+          return esb
+            .boolQuery()
+            .should(value.map(it => esb.matchQuery(key, it.toString())));
+        }
+        this.logger.error(
+          'Failed to query, unrecognized filter type',
+          key,
+          value,
+        );
+        throw new Error(
+          'Failed to add filters to query. Unrecognized filter type',
+        );
+      });
+    const query = isBlank(term)
+      ? esb.matchAllQuery()
+      : esb
+          .multiMatchQuery(['*'], term)
+          .fuzziness('auto')
+          .minimumShouldMatch(1);
+
+    return {
+      elasticSearchQuery: esb
+        .requestBodySearch()
+        .query(esb.boolQuery().filter(filter).must([query]))
+        // TODO: Replace size limit with page cursor after pagination approach decided
+        // See: https://github.com/backstage/backstage/issues/6062
+        .size(100)
+        .toJSON(),
+      documentTypes: types,
+    };
+  }
+
+  setTranslator(translator: ElasticSearchQueryTranslator) {
+    this.translator = translator;
+  }
+
+  async index(type: string, documents: IndexableDocument[]): Promise<void> {
+    this.logger.info(
+      `Started indexing ${documents.length} documents for index ${type}`,
+    );
+    const startTimestamp = process.hrtime();
+    const alias = this.constructSearchAlias(type);
+    const index = this.constructIndexName(type, `${Date.now()}`);
+    try {
+      const aliases = await this.elasticSearchClient.cat.aliases({
+        format: 'json',
+        name: alias,
+      });
+      const removableIndices = aliases.body.map(
+        (r: Record<string, any>) => r.index,
+      );
+
+      await this.elasticSearchClient.indices.create({
+        index,
+      });
+      const result = await this.elasticSearchClient.helpers.bulk({
+        datasource: documents,
+        onDocument() {
+          return {
+            index: { _index: index },
+          };
+        },
+        refreshOnCompletion: index,
+      });
+
+      this.logger.info(
+        `Indexing completed for index ${type} in ${duration(startTimestamp)}`,
+        result,
+      );
+      await this.elasticSearchClient.indices.updateAliases({
+        body: {
+          actions: [
+            { remove: { index: this.constructIndexName(type, '*'), alias } },
+            { add: { index, alias } },
+          ],
+        },
+      });
+
+      this.logger.info('Removing stale search indices', removableIndices);
+      if (removableIndices.length) {
+        await this.elasticSearchClient.indices.delete({
+          index: removableIndices,
+        });
+      }
+    } catch (e) {
+      this.logger.error(`Failed to index documents for type ${type}`, e);
+      const response = await this.elasticSearchClient.indices.exists({
+        index,
+      });
+      const indexCreated = response.body;
+      if (indexCreated) {
+        this.logger.info(`Removing created index ${index}`);
+        await this.elasticSearchClient.indices.delete({
+          index,
+        });
+      }
+    }
+  }
+
+  async query(query: SearchQuery): Promise<SearchResultSet> {
+    const { elasticSearchQuery, documentTypes } = this.translator(query);
+    const queryIndices = documentTypes
+      ? documentTypes.map(it => this.constructSearchAlias(it))
+      : this.constructSearchAlias('*');
+    try {
+      const result = await this.elasticSearchClient.search({
+        index: queryIndices,
+        body: elasticSearchQuery,
+      });
+      return {
+        results: result.body.hits.hits.map((d: ElasticSearchResult) => ({
+          type: this.getTypeFromIndex(d._index),
+          document: d._source,
+        })),
+      };
+    } catch (e) {
+      this.logger.error(
+        `Failed to query documents for indices ${queryIndices}`,
+        e,
+      );
+      return Promise.reject({ results: [] });
+    }
+  }
+
+  private readonly indexSeparator = '-index__';
+
+  private constructIndexName(type: string, postFix: string) {
+    return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`;
+  }
+
+  private getTypeFromIndex(index: string) {
+    return index
+      .substring(this.indexPrefix.length)
+      .split(this.indexSeparator)[0];
+  }
+
+  private constructSearchAlias(type: string) {
+    const postFix = this.aliasPostfix ? `__${this.aliasPostfix}` : '';
+    return `${this.indexPrefix}${type}${postFix}`;
+  }
+}

plugins/search-backend-module-elasticsearch/src/engines/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { ElasticSearchSearchEngine } from './ElasticSearchSearchEngine';
+export type { ConcreteElasticSearchQuery } from './ElasticSearchSearchEngine';

plugins/search-backend-module-elasticsearch/src/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { ElasticSearchSearchEngine } from './engines';

plugins/search-backend-node/api-report.md

@@ -8,6 +8,8 @@ import { DocumentDecorator } from '@backstage/search-common';
 import { IndexableDocument } from '@backstage/search-common';
 import { Logger as Logger_2 } from 'winston';
 import { default as lunr_2 } from 'lunr';
+import { QueryTranslator } from '@backstage/search-common';
+import { SearchEngine } from '@backstage/search-common';
 import { SearchQuery } from '@backstage/search-common';
 import { SearchResultSet } from '@backstage/search-common';
 
@@ -50,8 +52,6 @@ export class LunrSearchEngine implements SearchEngine {
   //
   // (undocumented)
   setTranslator(translator: LunrQueryTranslator): void;
-  // Warning: (ae-forgotten-export) The symbol "QueryTranslator" needs to be exported by the entry point index.d.ts
-  //
   // (undocumented)
   protected translator: QueryTranslator;
 }
@@ -66,14 +66,7 @@ export class Scheduler {
   stop(): void;
 }
 
-// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public
-export interface SearchEngine {
-  index(type: string, documents: IndexableDocument[]): Promise<void>;
-  query(query: SearchQuery): Promise<SearchResultSet>;
-  setTranslator(translator: QueryTranslator): void;
-}
+export { SearchEngine };
 
 // (No @packageDocumentation comment for this package)
 ```

plugins/search-backend-node/src/IndexBuilder.ts

@@ -18,13 +18,13 @@ import {
   DocumentCollator,
   DocumentDecorator,
   IndexableDocument,
+  SearchEngine,
 } from '@backstage/search-common';
 import { Logger } from 'winston';
 import { Scheduler } from './index';
 import {
   RegisterCollatorParameters,
   RegisterDecoratorParameters,
-  SearchEngine,
 } from './types';
 
 interface CollatorEnvelope {

plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts

@@ -16,7 +16,7 @@
 
 import { getVoidLogger } from '@backstage/backend-common';
 import lunr from 'lunr';
-import { SearchEngine } from '../types';
+import { SearchEngine } from '@backstage/search-common';
 import { ConcreteLunrQuery, LunrSearchEngine } from './LunrSearchEngine';
 
 /**

plugins/search-backend-node/src/engines/LunrSearchEngine.ts

@@ -18,10 +18,11 @@ import {
   IndexableDocument,
   SearchQuery,
   SearchResultSet,
+  QueryTranslator,
+  SearchEngine,
 } from '@backstage/search-common';
 import lunr from 'lunr';
 import { Logger } from 'winston';
-import { QueryTranslator, SearchEngine } from '../types';
 
 export type ConcreteLunrQuery = {
   lunrQueryBuilder: lunr.Index.QueryBuilder;

plugins/search-backend-node/src/index.ts

@@ -17,4 +17,8 @@
 export { IndexBuilder } from './IndexBuilder';
 export { Scheduler } from './Scheduler';
 export { LunrSearchEngine } from './engines';
-export type { SearchEngine } from './types';
+
+/**
+ * @deprecated Import from @backstage/search-common instead
+ */
+export type { SearchEngine } from '@backstage/search-common';

plugins/search-backend-node/src/types.ts

@@ -14,13 +14,7 @@
  * limitations under the License.
  */
 
-import {
-  DocumentCollator,
-  DocumentDecorator,
-  IndexableDocument,
-  SearchQuery,
-  SearchResultSet,
-} from '@backstage/search-common';
+import { DocumentCollator, DocumentDecorator } from '@backstage/search-common';
 
 /**
  * Parameters required to register a collator.
@@ -46,31 +40,3 @@ export interface RegisterDecoratorParameters {
    */
   decorator: DocumentDecorator;
 }
-
-/**
- * A type of function responsible for translating an abstract search query into
- * a concrete query relevant to a particular search engine.
- */
-export type QueryTranslator = (query: SearchQuery) => unknown;
-
-/**
- * Interface that must be implemented by specific search engines, responsible
- * for performing indexing and querying and translating abstract queries into
- * concrete, search engine-specific queries.
- */
-export interface SearchEngine {
-  /**
-   * Override the default translator provided by the SearchEngine.
-   */
-  setTranslator(translator: QueryTranslator): void;
-
-  /**
-   * Add the given documents to the SearchEngine index of the given type.
-   */
-  index(type: string, documents: IndexableDocument[]): Promise<void>;
-
-  /**
-   * Perform a search query against the SearchEngine.
-   */
-  query(query: SearchQuery): Promise<SearchResultSet>;
-}